mirror of
https://github.com/lz4/lz4.git
synced 2024-12-11 10:49:23 +08:00
Cyan4973
commit
9d8e1f58a0
@ -4,3 +4,11 @@ script: make test
|
||||
before_install:
|
||||
- sudo apt-get update -qq
|
||||
- sudo apt-get install -qq gcc-multilib
|
||||
- sudo apt-get install -qq valgrind
|
||||
|
||||
env:
|
||||
- LZ4_TRAVIS_CI_ENV=-m32
|
||||
- LZ4_TRAVIS_CI_ENV=-m64
|
||||
|
||||
matrix:
|
||||
fast_finish: true
|
||||
|
||||
25
Makefile
25
Makefile
@ -30,20 +30,21 @@
|
||||
# - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c
|
||||
# ################################################################
|
||||
|
||||
export RELEASE=r117
|
||||
# Version numbers
|
||||
export RELEASE=r118
|
||||
LIBVER_MAJOR=1
|
||||
LIBVER_MINOR=0
|
||||
LIBVER_MINOR=2
|
||||
LIBVER_PATCH=0
|
||||
LIBVER=$(LIBVER_MAJOR).$(LIBVER_MINOR).$(LIBVER_PATCH)
|
||||
|
||||
DESTDIR=
|
||||
PREFIX=/usr
|
||||
CC:=$(CC)
|
||||
PREFIX = /usr
|
||||
CC := $(CC)
|
||||
CFLAGS+= -I. -std=c99 -O3 -Wall -W -Wundef -DLZ4_VERSION=\"$(RELEASE)\"
|
||||
|
||||
LIBDIR=$(PREFIX)/lib
|
||||
LIBDIR?= $(PREFIX)/lib
|
||||
INCLUDEDIR=$(PREFIX)/include
|
||||
PRGDIR=programs
|
||||
PRGDIR = programs
|
||||
DISTRIBNAME=lz4-$(RELEASE).tar.gz
|
||||
|
||||
|
||||
@ -92,7 +93,7 @@ lz4programs: lz4.c lz4hc.c
|
||||
liblz4: lz4.c lz4hc.c
|
||||
@echo compiling static library
|
||||
@$(CC) $(CFLAGS) -c $^
|
||||
@ar rcs liblz4.a lz4.o lz4hc.o
|
||||
@$(AR) rcs liblz4.a lz4.o lz4hc.o
|
||||
@echo compiling dynamic library
|
||||
@$(CC) $(CFLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER)
|
||||
@echo creating versioned links
|
||||
@ -105,25 +106,25 @@ clean:
|
||||
@echo Cleaning completed
|
||||
|
||||
|
||||
#make install option is reserved to Linux & OSX targets
|
||||
#make install option is designed for Linux & OSX targets only
|
||||
ifneq (,$(filter $(shell uname),Linux Darwin))
|
||||
|
||||
install: liblz4
|
||||
@install -d -m 755 $(DESTDIR)$(LIBDIR)/ $(DESTDIR)$(INCLUDEDIR)/
|
||||
@install -m 755 liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a
|
||||
@install -m 755 liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER)
|
||||
@cp -a liblz4.$(SHARED_EXT_MAJOR) $(DESTDIR)$(LIBDIR)
|
||||
@cp -a liblz4.$(SHARED_EXT) $(DESTDIR)$(LIBDIR)
|
||||
@install -m 755 lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h
|
||||
@install -m 755 lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
|
||||
@install -m 644 liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a
|
||||
@install -m 644 lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h
|
||||
@install -m 644 lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
|
||||
@echo lz4 static and shared library installed
|
||||
@cd $(PRGDIR); $(MAKE) -e install
|
||||
|
||||
uninstall:
|
||||
[ -x $(DESTDIR)$(LIBDIR)/liblz4.a ] && rm -f $(DESTDIR)$(LIBDIR)/liblz4.a
|
||||
rm -f $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT)
|
||||
rm -f $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR)
|
||||
[ -x $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER) ] && rm -f $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER)
|
||||
[ -f $(DESTDIR)$(LIBDIR)/liblz4.a ] && rm -f $(DESTDIR)$(LIBDIR)/liblz4.a
|
||||
[ -f $(DESTDIR)$(INCLUDEDIR)/lz4.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/lz4.h
|
||||
[ -f $(DESTDIR)$(INCLUDEDIR)/lz4hc.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
|
||||
@echo lz4 libraries successfully uninstalled
|
||||
|
||||
@ -4,6 +4,11 @@ LZ4 - Extremely fast compression
|
||||
LZ4 is lossless compression algorithm, providing compression speed at 400 MB/s per core, scalable with multi-cores CPU. It also features an extremely fast decoder, with speed in multiple GB/s per core, typically reaching RAM speed limits on multi-core systems.
|
||||
A high compression derivative, called LZ4_HC, is also provided. It trades CPU time for compression ratio.
|
||||
|
||||
|Branch |Status |
|
||||
|------------|---------|
|
||||
|master | [](https://travis-ci.org/Cyan4973/lz4) |
|
||||
|dev | [](https://travis-ci.org/Cyan4973/lz4) |
|
||||
|
||||
This is an official mirror of LZ4 project, [hosted on Google Code](http://code.google.com/p/lz4/).
|
||||
The intention is to offer github's capabilities to lz4 users, such as cloning, branch, or source download.
|
||||
|
||||
|
||||
256
lz4.h
256
lz4.h
@ -28,8 +28,8 @@
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
|
||||
- LZ4 source repository : http://code.google.com/p/lz4/
|
||||
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
@ -42,16 +42,21 @@ extern "C" {
|
||||
Version
|
||||
**************************************/
|
||||
#define LZ4_VERSION_MAJOR 1 /* for major interface/format changes */
|
||||
#define LZ4_VERSION_MINOR 1 /* for minor interface/format changes */
|
||||
#define LZ4_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */
|
||||
#define LZ4_VERSION_MINOR 2 /* for minor interface/format changes */
|
||||
#define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */
|
||||
|
||||
|
||||
/**************************************
|
||||
Compiler Options
|
||||
Tuning parameter
|
||||
**************************************/
|
||||
#if (defined(__GNUC__) && defined(__STRICT_ANSI__)) || (defined(_MSC_VER) && !defined(__cplusplus)) /* Visual Studio */
|
||||
# define inline __inline /* Visual C is not C99, but supports some kind of inline */
|
||||
#endif
|
||||
/*
|
||||
* LZ4_MEMORY_USAGE :
|
||||
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
|
||||
* Increasing memory usage improves compression ratio
|
||||
* Reduced memory usage can improve speed, due to cache effect
|
||||
* Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
|
||||
*/
|
||||
#define LZ4_MEMORY_USAGE 14
|
||||
|
||||
|
||||
/**************************************
|
||||
@ -59,7 +64,7 @@ extern "C" {
|
||||
**************************************/
|
||||
|
||||
int LZ4_compress (const char* source, char* dest, int inputSize);
|
||||
int LZ4_decompress_safe (const char* source, char* dest, int inputSize, int maxOutputSize);
|
||||
int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxOutputSize);
|
||||
|
||||
/*
|
||||
LZ4_compress() :
|
||||
@ -72,10 +77,21 @@ LZ4_compress() :
|
||||
or 0 if the compression fails
|
||||
|
||||
LZ4_decompress_safe() :
|
||||
maxOutputSize : is the size of the destination buffer (which must be already allocated)
|
||||
compressedSize : is obviously the source size
|
||||
maxOutputSize : is the size of the destination buffer, which must be already allocated.
|
||||
return : the number of bytes decoded in the destination buffer (necessarily <= maxOutputSize)
|
||||
If the destination buffer is not large enough, decoding will stop and output an error code (<0).
|
||||
If the source stream is detected malformed, the function will stop decoding and return a negative result.
|
||||
This function is protected against buffer overflow exploits (never writes outside of output buffer, and never reads outside of input buffer). Therefore, it is protected against malicious data packets
|
||||
This function is protected against buffer overflow exploits :
|
||||
it never writes outside of output buffer, and never reads outside of input buffer.
|
||||
Therefore, it is protected against malicious data packets.
|
||||
*/
|
||||
|
||||
|
||||
/*
|
||||
Note :
|
||||
Should you prefer to explicitly allocate compression-table memory using your own allocation method,
|
||||
use the streaming functions provided below, simply reset the memory area between each call to LZ4_compress_continue()
|
||||
*/
|
||||
|
||||
|
||||
@ -89,7 +105,6 @@ LZ4_decompress_safe() :
|
||||
LZ4_compressBound() :
|
||||
Provides the maximum size that LZ4 may output in a "worst case" scenario (input data not compressible)
|
||||
primarily useful for memory allocation of output buffer.
|
||||
inline function is recommended for the general case,
|
||||
macro is also provided when result needs to be evaluated at compilation (such as stack memory allocation).
|
||||
|
||||
isize : is the input size. Max supported value is LZ4_MAX_INPUT_SIZE
|
||||
@ -118,17 +133,18 @@ LZ4_decompress_fast() :
|
||||
originalSize : is the original and therefore uncompressed size
|
||||
return : the number of bytes read from the source buffer (in other words, the compressed size)
|
||||
If the source stream is malformed, the function will stop decoding and return a negative result.
|
||||
Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes.
|
||||
note : This function is a bit faster than LZ4_decompress_safe()
|
||||
This function never writes outside of output buffers, but may read beyond input buffer in case of malicious data packet.
|
||||
Use this function preferably into a trusted environment (data to decode comes from a trusted source).
|
||||
Destination buffer must be already allocated. Its size must be a minimum of 'outputSize' bytes.
|
||||
It provides fast decompression and fully respect memory boundaries for properly formed compressed data.
|
||||
It does not provide full protection against intentionnally modified data stream.
|
||||
Use this function in a trusted environment (data to decode comes from a trusted source).
|
||||
*/
|
||||
int LZ4_decompress_fast (const char* source, char* dest, int originalSize);
|
||||
|
||||
|
||||
/*
|
||||
LZ4_decompress_safe_partial() :
|
||||
This function decompress a compressed block of size 'inputSize' at position 'source'
|
||||
This function decompress a compressed block of size 'compressedSize' at position 'source'
|
||||
into output buffer 'dest' of size 'maxOutputSize'.
|
||||
The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached,
|
||||
reducing decompression time.
|
||||
@ -138,110 +154,152 @@ LZ4_decompress_safe_partial() :
|
||||
If the source stream is detected malformed, the function will stop decoding and return a negative result.
|
||||
This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets
|
||||
*/
|
||||
int LZ4_decompress_safe_partial (const char* source, char* dest, int inputSize, int targetOutputSize, int maxOutputSize);
|
||||
int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxOutputSize);
|
||||
|
||||
|
||||
/***********************************************
|
||||
Experimental Streaming Compression Functions
|
||||
***********************************************/
|
||||
|
||||
#define LZ4_STREAMSIZE_U32 ((1 << (LZ4_MEMORY_USAGE-2)) + 8)
|
||||
#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U32 * sizeof(unsigned int))
|
||||
/*
|
||||
* LZ4_stream_t
|
||||
* information structure to track an LZ4 stream.
|
||||
* important : set this structure content to zero before first use !
|
||||
*/
|
||||
typedef struct { unsigned int table[LZ4_STREAMSIZE_U32]; } LZ4_stream_t;
|
||||
|
||||
/*
|
||||
* If you prefer dynamic allocation methods,
|
||||
* LZ4_createStream
|
||||
* provides a pointer (void*) towards an initialized LZ4_stream_t structure.
|
||||
* LZ4_free just frees it.
|
||||
*/
|
||||
void* LZ4_createStream();
|
||||
int LZ4_free (void* LZ4_stream);
|
||||
|
||||
|
||||
/*
|
||||
These functions are provided should you prefer to allocate memory for compression tables with your own allocation methods.
|
||||
To know how much memory must be allocated for the compression tables, use :
|
||||
int LZ4_sizeofState();
|
||||
|
||||
Note that tables must be aligned on 4-bytes boundaries, otherwise compression will fail (return code 0).
|
||||
|
||||
The allocated memory can be provided to the compressions functions using 'void* state' parameter.
|
||||
LZ4_compress_withState() and LZ4_compress_limitedOutput_withState() are equivalent to previously described functions.
|
||||
They just use the externally allocated memory area instead of allocating their own (on stack, or on heap).
|
||||
*/
|
||||
int LZ4_sizeofState(void);
|
||||
int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
|
||||
int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
|
||||
|
||||
|
||||
/**************************************
|
||||
Streaming Functions
|
||||
**************************************/
|
||||
void* LZ4_create (const char* inputBuffer);
|
||||
int LZ4_compress_continue (void* LZ4_Data, const char* source, char* dest, int inputSize);
|
||||
int LZ4_compress_limitedOutput_continue (void* LZ4_Data, const char* source, char* dest, int inputSize, int maxOutputSize);
|
||||
char* LZ4_slideInputBuffer (void* LZ4_Data);
|
||||
int LZ4_free (void* LZ4_Data);
|
||||
* LZ4_loadDict
|
||||
* Use this function to load a static dictionary into LZ4_stream.
|
||||
* Any previous data will be forgotten, only 'dictionary' will remain in memory.
|
||||
* Loading a size of 0 is allowed (same effect as init).
|
||||
* Return : 1 if OK, 0 if error
|
||||
*/
|
||||
int LZ4_loadDict (void* LZ4_stream, const char* dictionary, int dictSize);
|
||||
|
||||
/*
|
||||
These functions allow the compression of dependent blocks, where each block benefits from prior 64 KB within preceding blocks.
|
||||
In order to achieve this, it is necessary to start creating the LZ4 Data Structure, thanks to the function :
|
||||
|
||||
void* LZ4_create (const char* inputBuffer);
|
||||
The result of the function is the (void*) pointer on the LZ4 Data Structure.
|
||||
This pointer will be needed in all other functions.
|
||||
If the pointer returned is NULL, then the allocation has failed, and compression must be aborted.
|
||||
The only parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
|
||||
The input buffer must be already allocated, and size at least 192KB.
|
||||
'inputBuffer' will also be the 'const char* source' of the first block.
|
||||
|
||||
All blocks are expected to lay next to each other within the input buffer, starting from 'inputBuffer'.
|
||||
To compress each block, use either LZ4_compress_continue() or LZ4_compress_limitedOutput_continue().
|
||||
Their behavior are identical to LZ4_compress() or LZ4_compress_limitedOutput(),
|
||||
but require the LZ4 Data Structure as their first argument, and check that each block starts right after the previous one.
|
||||
If next block does not begin immediately after the previous one, the compression will fail (return 0).
|
||||
|
||||
When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to :
|
||||
char* LZ4_slideInputBuffer(void* LZ4_Data);
|
||||
must be performed. It will typically copy the latest 64KB of input at the beginning of input buffer.
|
||||
Note that, for this function to work properly, minimum size of an input buffer must be 192KB.
|
||||
==> The memory position where the next input data block must start is provided as the result of the function.
|
||||
|
||||
Compression can then resume, using LZ4_compress_continue() or LZ4_compress_limitedOutput_continue(), as usual.
|
||||
|
||||
When compression is completed, a call to LZ4_free() will release the memory used by the LZ4 Data Structure.
|
||||
*/
|
||||
|
||||
|
||||
int LZ4_sizeofStreamState(void);
|
||||
int LZ4_resetStreamState(void* state, const char* inputBuffer);
|
||||
* LZ4_compress_continue
|
||||
* Compress data block 'source', using blocks compressed before as dictionary to improve compression ratio
|
||||
* Previous data blocks are assumed to still be present at their previous location.
|
||||
*/
|
||||
int LZ4_compress_continue (void* LZ4_stream, const char* source, char* dest, int inputSize);
|
||||
|
||||
/*
|
||||
These functions achieve the same result as :
|
||||
void* LZ4_create (const char* inputBuffer);
|
||||
|
||||
They are provided here to allow the user program to allocate memory using its own routines.
|
||||
|
||||
To know how much space must be allocated, use LZ4_sizeofStreamState();
|
||||
Note also that space must be 4-bytes aligned.
|
||||
|
||||
Once space is allocated, you must initialize it using : LZ4_resetStreamState(void* state, const char* inputBuffer);
|
||||
void* state is a pointer to the space allocated.
|
||||
It must be aligned on 4-bytes boundaries, and be large enough.
|
||||
The parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
|
||||
The input buffer must be already allocated, and size at least 192KB.
|
||||
'inputBuffer' will also be the 'const char* source' of the first block.
|
||||
|
||||
The same space can be re-used multiple times, just by initializing it each time with LZ4_resetStreamState().
|
||||
return value of LZ4_resetStreamState() must be 0 is OK.
|
||||
Any other value means there was an error (typically, pointer is not aligned on 4-bytes boundaries).
|
||||
*/
|
||||
|
||||
|
||||
int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int inputSize, int maxOutputSize);
|
||||
int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int outputSize);
|
||||
* LZ4_compress_limitedOutput_continue
|
||||
* Same as before, but also specify a maximum target compressed size (maxOutputSize)
|
||||
* If objective cannot be met, compression exits, and returns a zero.
|
||||
*/
|
||||
int LZ4_compress_limitedOutput_continue (void* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize);
|
||||
|
||||
/*
|
||||
*_withPrefix64k() :
|
||||
These decoding functions work the same as their "normal name" versions,
|
||||
but can use up to 64KB of data in front of 'char* dest'.
|
||||
These functions are necessary to decode inter-dependant blocks.
|
||||
* LZ4_saveDict
|
||||
* If previously compressed data block is not guaranteed to remain at its previous memory location
|
||||
* save it into a safe place (char* safeBuffer)
|
||||
* Note : you don't need to call LZ4_loadDict() afterwards,
|
||||
* dictionary is immediately usable, you can therefore call again LZ4_compress_continue()
|
||||
* Return : 1 if OK, 0 if error
|
||||
* Note : any dictSize > 64 KB will be interpreted as 64KB.
|
||||
*/
|
||||
int LZ4_saveDict (void* LZ4_stream, char* safeBuffer, int dictSize);
|
||||
|
||||
|
||||
/************************************************
|
||||
Experimental Streaming Decompression Functions
|
||||
************************************************/
|
||||
|
||||
#define LZ4_STREAMDECODESIZE_U32 4
|
||||
#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U32 * sizeof(unsigned int))
|
||||
/*
|
||||
* LZ4_streamDecode_t
|
||||
* information structure to track an LZ4 stream.
|
||||
* important : set this structure content to zero before first use !
|
||||
*/
|
||||
typedef struct { unsigned int table[LZ4_STREAMDECODESIZE_U32]; } LZ4_streamDecode_t;
|
||||
|
||||
/*
|
||||
* If you prefer dynamic allocation methods,
|
||||
* LZ4_createStreamDecode()
|
||||
* provides a pointer (void*) towards an initialized LZ4_streamDecode_t structure.
|
||||
* LZ4_free just frees it.
|
||||
*/
|
||||
void* LZ4_createStreamDecode();
|
||||
int LZ4_free (void* LZ4_stream); /* yes, it's the same one as for compression */
|
||||
|
||||
/*
|
||||
*_continue() :
|
||||
These decoding functions allow decompression of multiple blocks in "streaming" mode.
|
||||
Previously decoded blocks must still be available at the memory position where they were decoded.
|
||||
If it's not possible, save the relevant part of decoded data into a safe buffer,
|
||||
and indicate where it stands using LZ4_setDictDecode()
|
||||
*/
|
||||
int LZ4_decompress_safe_continue (void* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize);
|
||||
int LZ4_decompress_fast_continue (void* LZ4_streamDecode, const char* source, char* dest, int originalSize);
|
||||
|
||||
/*
|
||||
* LZ4_setDictDecode
|
||||
* Use this function to instruct where to find the dictionary.
|
||||
* This function can be used to specify a static dictionary,
|
||||
* or to instruct where to find some previously decoded data saved into a different memory space.
|
||||
* Setting a size of 0 is allowed (same effect as no dictionary).
|
||||
* Return : 1 if OK, 0 if error
|
||||
*/
|
||||
int LZ4_setDictDecode (void* LZ4_streamDecode, const char* dictionary, int dictSize);
|
||||
|
||||
|
||||
/*
|
||||
Advanced decoding functions :
|
||||
*_usingDict() :
|
||||
These decoding functions work the same as
|
||||
a combination of LZ4_setDictDecode() followed by LZ4_decompress_x_continue()
|
||||
all together into a single function call.
|
||||
It doesn't use nor update an LZ4_streamDecode_t structure.
|
||||
*/
|
||||
int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize);
|
||||
int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize);
|
||||
|
||||
|
||||
|
||||
|
||||
/**************************************
|
||||
Obsolete Functions
|
||||
**************************************/
|
||||
/*
|
||||
These functions are deprecated and should no longer be used.
|
||||
They are provided here for compatibility with existing user programs.
|
||||
Obsolete decompression functions
|
||||
These function names are deprecated and should no longer be used.
|
||||
They are only provided here for compatibility with older user programs.
|
||||
- LZ4_uncompress is the same as LZ4_decompress_fast
|
||||
- LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe
|
||||
*/
|
||||
int LZ4_uncompress (const char* source, char* dest, int outputSize);
|
||||
int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
|
||||
|
||||
/* Obsolete functions for externally allocated state; use streaming interface instead */
|
||||
int LZ4_sizeofState(void);
|
||||
int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
|
||||
int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
|
||||
|
||||
/* Obsolete streaming functions; use new streaming interface whenever possible */
|
||||
void* LZ4_create (const char* inputBuffer);
|
||||
int LZ4_sizeofStreamState(void);
|
||||
int LZ4_resetStreamState(void* state, const char* inputBuffer);
|
||||
char* LZ4_slideInputBuffer (void* state);
|
||||
|
||||
/* Obsolete streaming decoding functions */
|
||||
int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int compressedSize, int maxOutputSize);
|
||||
int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int originalSize);
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
|
||||
6
lz4hc.c
6
lz4hc.c
@ -54,8 +54,9 @@
|
||||
**************************************/
|
||||
/* 32 or 64 bits ? */
|
||||
#if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \
|
||||
|| defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__) \
|
||||
|| defined(__64BIT__) || defined(_LP64) || defined(__LP64__) \
|
||||
|| defined(__powerpc64__) || defined(__powerpc64le__) \
|
||||
|| defined(__ppc64__) || defined(__ppc64le__) \
|
||||
|| defined(__PPC64__) || defined(__PPC64LE__) \
|
||||
|| defined(__ia64) || defined(__itanium__) || defined(_M_IA64) ) /* Detects 64 bits mode */
|
||||
# define LZ4_ARCH64 1
|
||||
#else
|
||||
@ -66,6 +67,7 @@
|
||||
* Little Endian or Big Endian ?
|
||||
* Overwrite the #define below if you know your architecture endianess
|
||||
*/
|
||||
#include <stdlib.h> /* Apparently required to detect endianess */
|
||||
#if defined (__GLIBC__)
|
||||
# include <endian.h>
|
||||
# if (__BYTE_ORDER == __BIG_ENDIAN)
|
||||
|
||||
1
lz4hc.h
1
lz4hc.h
@ -104,6 +104,7 @@ They just use the externally allocated memory area instead of allocating their o
|
||||
/**************************************
|
||||
Streaming Functions
|
||||
**************************************/
|
||||
/* Note : these streaming functions still follows the older model */
|
||||
void* LZ4_createHC (const char* inputBuffer);
|
||||
int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize);
|
||||
int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize);
|
||||
|
||||
@ -30,7 +30,7 @@
|
||||
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
|
||||
# ################################################################
|
||||
|
||||
RELEASE=r116
|
||||
RELEASE=r118
|
||||
DESTDIR=
|
||||
PREFIX=/usr
|
||||
CC:=$(CC)
|
||||
@ -42,18 +42,32 @@ MANDIR=$(PREFIX)/share/man/man1
|
||||
LZ4DIR=..
|
||||
|
||||
TEST_FILES = COPYING
|
||||
TEST_TARGETS=test-64 test-32
|
||||
BENCH_NB=-i5
|
||||
|
||||
# Minimize test target for Travis CI's Build Matrix
|
||||
ifeq ($(LZ4_TRAVIS_CI_ENV),-m32)
|
||||
TEST_TARGETS=test-32
|
||||
BENCH_NB=-i1
|
||||
else ifeq ($(LZ4_TRAVIS_CI_ENV),-m64)
|
||||
TEST_TARGETS=test-64
|
||||
BENCH_NB=-i1
|
||||
endif
|
||||
|
||||
|
||||
# Define *.exe as extension for Windows systems
|
||||
ifneq (,$(filter Windows%,$(OS)))
|
||||
EXT =.exe
|
||||
VOID = nul
|
||||
else
|
||||
EXT =
|
||||
VOID = /dev/null
|
||||
endif
|
||||
|
||||
|
||||
default: lz4 lz4c
|
||||
|
||||
all: lz4 lz4c lz4c32 fuzzer fuzzer32 fullbench fullbench32
|
||||
all: lz4 lz4c lz4c32 fullbench fullbench32 fuzzer fuzzer32 datagen
|
||||
|
||||
lz4: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4io.c lz4cli.c
|
||||
$(CC) $(FLAGS) -DDISABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT)
|
||||
@ -64,22 +78,27 @@ lz4c : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4io.c lz4cli.c
|
||||
lz4c32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4io.c lz4cli.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fuzzer : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c fuzzer.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fuzzer32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c fuzzer.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fullbench : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c xxhash.c fullbench.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fullbench32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c xxhash.c fullbench.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fuzzer : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c xxhash.c fuzzer.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fuzzer32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c xxhash.c fuzzer.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
datagen : datagen.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
|
||||
clean:
|
||||
@rm -f core *.o \
|
||||
lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) \
|
||||
fuzzer$(EXT) fuzzer32$(EXT) fullbench$(EXT) fullbench32$(EXT)
|
||||
fullbench$(EXT) fullbench32$(EXT) \
|
||||
fuzzer$(EXT) fuzzer32$(EXT) datagen$(EXT)
|
||||
@echo Cleaning completed
|
||||
|
||||
|
||||
@ -102,15 +121,35 @@ uninstall:
|
||||
[ -x $(DESTDIR)$(BINDIR)/lz4 ] && rm -f $(DESTDIR)$(BINDIR)/lz4
|
||||
[ -x $(DESTDIR)$(BINDIR)/lz4c ] && rm -f $(DESTDIR)$(BINDIR)/lz4c
|
||||
[ -f $(DESTDIR)$(MANDIR)/lz4.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4.1
|
||||
[ -f $(DESTDIR)$(MANDIR)/lz4c.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4c.1
|
||||
[ -f $(DESTDIR)$(MANDIR)/lz4cat.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4cat.1
|
||||
@echo lz4 successfully uninstalled
|
||||
|
||||
test: test-lz4 test-lz4c test-lz4c32 test-fuzzer test-fuzzer32 test-fullbench test-fullbench32
|
||||
test: $(TEST_TARGETS)
|
||||
|
||||
test-lz4:
|
||||
test-64: test-lz4 test-lz4c test-fullbench test-fuzzer test-mem
|
||||
|
||||
test-lz4c:
|
||||
test-32: test-lz4 test-lz4c32 test-fullbench32 test-fuzzer32 test-mem32
|
||||
|
||||
test-lz4c32:
|
||||
test-lz4: lz4 datagen
|
||||
./datagen | ./lz4 | ./lz4 -vdq > $(VOID)
|
||||
./datagen -g256MB | ./lz4 -vqB4D | ./lz4 -vdq > $(VOID)
|
||||
./datagen -g6GB | ./lz4 -vqB5D | ./lz4 -vdq > $(VOID)
|
||||
|
||||
test-lz4c: lz4c datagen
|
||||
|
||||
test-lz4c32: lz4c32 lz4 datagen
|
||||
./datagen | ./lz4c32 | ./lz4c32 -vdq > $(VOID)
|
||||
./datagen | ./lz4c32 | ./lz4 -vdq > $(VOID)
|
||||
./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4c32 -vdq > $(VOID)
|
||||
./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4 -vdq > $(VOID)
|
||||
./datagen -g6GB | ./lz4c32 -vqB5D | ./lz4c32 -vdq > $(VOID)
|
||||
|
||||
test-fullbench: fullbench
|
||||
./fullbench --no-prompt $(BENCH_NB) $(TEST_FILES)
|
||||
|
||||
test-fullbench32: fullbench32
|
||||
./fullbench32 --no-prompt $(BENCH_NB) $(TEST_FILES)
|
||||
|
||||
test-fuzzer: fuzzer
|
||||
./fuzzer --no-prompt
|
||||
@ -118,10 +157,14 @@ test-fuzzer: fuzzer
|
||||
test-fuzzer32: fuzzer32
|
||||
./fuzzer32 --no-prompt
|
||||
|
||||
test-fullbench: fullbench
|
||||
./fullbench --no-prompt $(TEST_FILES)
|
||||
test-mem: lz4 datagen
|
||||
./datagen -g256MB > tmp
|
||||
valgrind ./lz4 -B4D -f tmp /dev/null
|
||||
./datagen -g16MB > tmp
|
||||
valgrind ./lz4 -9 -B5D -f tmp /dev/null
|
||||
rm tmp
|
||||
|
||||
test-fullbench32: fullbench32
|
||||
./fullbench32 --no-prompt $(TEST_FILES)
|
||||
test-mem32: lz4c32 datagen
|
||||
# unfortunately, valgrind doesn't seem to work with non-native binary. If someone knows how to do a valgrind-test on a 32-bits exe with a 64-bits system...
|
||||
|
||||
endif
|
||||
|
||||
283
programs/datagen.c
Normal file
283
programs/datagen.c
Normal file
@ -0,0 +1,283 @@
|
||||
/*
|
||||
datagen.c - compressible data generator test tool
|
||||
Copyright (C) Yann Collet 2012-2014
|
||||
GPL v2 License
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
|
||||
- LZ4 source repository : http://code.google.com/p/lz4/
|
||||
*/
|
||||
|
||||
/**************************************
|
||||
Remove Visual warning messages
|
||||
**************************************/
|
||||
#define _CRT_SECURE_NO_WARNINGS // fgets
|
||||
|
||||
|
||||
/**************************************
|
||||
Includes
|
||||
**************************************/
|
||||
//#include <stdlib.h>
|
||||
#include <stdio.h> // fgets, sscanf
|
||||
#include <string.h> // strcmp
|
||||
|
||||
|
||||
/**************************************
|
||||
Basic Types
|
||||
**************************************/
|
||||
#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
|
||||
# include <stdint.h>
|
||||
typedef uint8_t BYTE;
|
||||
typedef uint16_t U16;
|
||||
typedef uint32_t U32;
|
||||
typedef int32_t S32;
|
||||
typedef uint64_t U64;
|
||||
#else
|
||||
typedef unsigned char BYTE;
|
||||
typedef unsigned short U16;
|
||||
typedef unsigned int U32;
|
||||
typedef signed int S32;
|
||||
typedef unsigned long long U64;
|
||||
#endif
|
||||
|
||||
|
||||
/**************************************
|
||||
Constants
|
||||
**************************************/
|
||||
#ifndef LZ4_VERSION
|
||||
# define LZ4_VERSION "rc118"
|
||||
#endif
|
||||
|
||||
#define KB *(1U<<10)
|
||||
#define MB *(1U<<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
#define CDG_SIZE_DEFAULT (64 KB)
|
||||
#define CDG_SEED_DEFAULT 0
|
||||
#define CDG_COMPRESSIBILITY_DEFAULT 50
|
||||
#define PRIME1 2654435761U
|
||||
#define PRIME2 2246822519U
|
||||
|
||||
|
||||
/**************************************
|
||||
Macros
|
||||
**************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
|
||||
|
||||
/**************************************
|
||||
Local Parameters
|
||||
**************************************/
|
||||
static int no_prompt = 0;
|
||||
static char* programName;
|
||||
static int displayLevel = 2;
|
||||
|
||||
|
||||
/*********************************************************
|
||||
Fuzzer functions
|
||||
*********************************************************/
|
||||
|
||||
#define CDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
|
||||
static unsigned int CDG_rand(U32* src)
|
||||
{
|
||||
U32 rand32 = *src;
|
||||
rand32 *= PRIME1;
|
||||
rand32 += PRIME2;
|
||||
rand32 = CDG_rotl32(rand32, 13);
|
||||
*src = rand32;
|
||||
return rand32;
|
||||
}
|
||||
|
||||
|
||||
#define CDG_RAND15BITS ((CDG_rand(seed) >> 3) & 32767)
|
||||
#define CDG_RANDLENGTH ( ((CDG_rand(seed) >> 7) & 3) ? (CDG_rand(seed) % 14) : (CDG_rand(seed) & 511) + 15)
|
||||
#define CDG_RANDCHAR (((CDG_rand(seed) >> 9) & 63) + '0')
|
||||
static void CDG_generate(U64 size, U32* seed, double proba)
|
||||
{
|
||||
BYTE fullbuff[32 KB + 128 KB + 1];
|
||||
BYTE* buff = fullbuff + 32 KB;
|
||||
U64 total=0;
|
||||
U32 P32 = (U32)(32768 * proba);
|
||||
U32 pos=0;
|
||||
U32 genBlockSize = 128 KB;
|
||||
|
||||
// Build initial prefix
|
||||
while (pos<32 KB)
|
||||
{
|
||||
// Select : Literal (char) or Match (within 32K)
|
||||
if (CDG_RAND15BITS < P32)
|
||||
{
|
||||
// Copy (within 64K)
|
||||
U32 d;
|
||||
int ref;
|
||||
int length = CDG_RANDLENGTH + 4;
|
||||
U32 offset = CDG_RAND15BITS + 1;
|
||||
if (offset > pos) offset = pos;
|
||||
ref = pos - offset;
|
||||
d = pos + length;
|
||||
while (pos < d) fullbuff[pos++] = fullbuff[ref++];
|
||||
}
|
||||
else
|
||||
{
|
||||
// Literal (noise)
|
||||
U32 d;
|
||||
int length = CDG_RANDLENGTH;
|
||||
d = pos + length;
|
||||
while (pos < d) fullbuff[pos++] = CDG_RANDCHAR;
|
||||
}
|
||||
}
|
||||
|
||||
// Generate compressible data
|
||||
pos = 0;
|
||||
while (total < size)
|
||||
{
|
||||
if (size-total < 128 KB) genBlockSize = (U32)(size-total);
|
||||
total += genBlockSize;
|
||||
buff[genBlockSize] = 0;
|
||||
pos = 0;
|
||||
while (pos<genBlockSize)
|
||||
{
|
||||
// Select : Literal (char) or Match (within 32K)
|
||||
if (CDG_RAND15BITS < P32)
|
||||
{
|
||||
// Copy (within 64K)
|
||||
int ref;
|
||||
U32 d;
|
||||
int length = CDG_RANDLENGTH + 4;
|
||||
U32 offset = CDG_RAND15BITS + 1;
|
||||
if (pos + length > genBlockSize ) length = genBlockSize - pos;
|
||||
ref = pos - offset;
|
||||
d = pos + length;
|
||||
while (pos < d) buff[pos++] = buff[ref++];
|
||||
}
|
||||
else
|
||||
{
|
||||
// Literal (noise)
|
||||
U32 d;
|
||||
int length = CDG_RANDLENGTH;
|
||||
if (pos + length > genBlockSize) length = genBlockSize - pos;
|
||||
d = pos + length;
|
||||
while (pos < d) buff[pos++] = CDG_RANDCHAR;
|
||||
}
|
||||
}
|
||||
pos=0;
|
||||
for (;pos+512<=genBlockSize;pos+=512) printf("%512.512s", buff+pos);
|
||||
for (;pos<genBlockSize;pos++) printf("%c", buff[pos]);
|
||||
// Regenerate prefix
|
||||
memcpy(fullbuff, buff + 96 KB, 32 KB);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
int CDG_usage()
|
||||
{
|
||||
DISPLAY( "Compressible data generator\n");
|
||||
DISPLAY( "Usage :\n");
|
||||
DISPLAY( " %s [size] [args]\n", programName);
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Arguments :\n");
|
||||
DISPLAY( " -g# : generate # data (default:%i)\n", CDG_SIZE_DEFAULT);
|
||||
DISPLAY( " -s# : Select seed (default:%i)\n", CDG_SEED_DEFAULT);
|
||||
DISPLAY( " -p# : Select compressibility in %% (default:%i%%)\n", CDG_COMPRESSIBILITY_DEFAULT);
|
||||
DISPLAY( " -h : display help and exit\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
int argNb;
|
||||
int proba = CDG_COMPRESSIBILITY_DEFAULT;
|
||||
U64 size = CDG_SIZE_DEFAULT;
|
||||
U32 seed = CDG_SEED_DEFAULT;
|
||||
|
||||
// Check command line
|
||||
programName = argv[0];
|
||||
for(argNb=1; argNb<argc; argNb++)
|
||||
{
|
||||
char* argument = argv[argNb];
|
||||
|
||||
if(!argument) continue; // Protection if argument empty
|
||||
|
||||
// Decode command (note : aggregated commands are allowed)
|
||||
if (argument[0]=='-')
|
||||
{
|
||||
if (!strcmp(argument, "--no-prompt")) { no_prompt=1; continue; }
|
||||
|
||||
while (argument[1]!=0)
|
||||
{
|
||||
argument++;
|
||||
switch(*argument)
|
||||
{
|
||||
case 'h':
|
||||
return CDG_usage();
|
||||
case 'g':
|
||||
argument++;
|
||||
size=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
size *= 10;
|
||||
size += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
if (*argument=='K') { size <<= 10; argument++; }
|
||||
if (*argument=='M') { size <<= 20; argument++; }
|
||||
if (*argument=='G') { size <<= 30; argument++; }
|
||||
if (*argument=='B') { argument++; }
|
||||
break;
|
||||
case 's':
|
||||
argument++;
|
||||
seed=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
seed *= 10;
|
||||
seed += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
case 'p':
|
||||
argument++;
|
||||
proba=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
proba *= 10;
|
||||
proba += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
if (proba<0) proba=0;
|
||||
if (proba>100) proba=100;
|
||||
break;
|
||||
case 'v':
|
||||
displayLevel = 4;
|
||||
break;
|
||||
default: ;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// Get Seed
|
||||
DISPLAYLEVEL(4, "Data Generator %s \n", LZ4_VERSION);
|
||||
DISPLAYLEVEL(3, "Seed = %u \n", seed);
|
||||
if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", proba);
|
||||
|
||||
CDG_generate(size, &seed, ((double)proba) / 100);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -118,8 +118,8 @@
|
||||
#define MAX_MEM (1984<<20)
|
||||
#define DEFAULT_CHUNKSIZE (4<<20)
|
||||
|
||||
#define ALL_COMPRESSORS -1
|
||||
#define ALL_DECOMPRESSORS -1
|
||||
#define ALL_COMPRESSORS 0
|
||||
#define ALL_DECOMPRESSORS 0
|
||||
|
||||
|
||||
//**************************************
|
||||
@ -250,78 +250,108 @@ static U64 BMK_GetFileSize(char* infilename)
|
||||
}
|
||||
|
||||
|
||||
//*********************************************************
|
||||
// Public function
|
||||
//*********************************************************
|
||||
/*********************************************************
|
||||
Benchmark function
|
||||
*********************************************************/
|
||||
|
||||
static inline int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize)
|
||||
static int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compress_limitedOutput(in, out, inSize, LZ4_compressBound(inSize));
|
||||
}
|
||||
|
||||
static void* stateLZ4;
|
||||
static inline int local_LZ4_compress_withState(const char* in, char* out, int inSize)
|
||||
static int local_LZ4_compress_withState(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compress_withState(stateLZ4, in, out, inSize);
|
||||
}
|
||||
|
||||
static inline int local_LZ4_compress_limitedOutput_withState(const char* in, char* out, int inSize)
|
||||
static int local_LZ4_compress_limitedOutput_withState(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compress_limitedOutput_withState(stateLZ4, in, out, inSize, LZ4_compressBound(inSize));
|
||||
}
|
||||
|
||||
static void* ctx;
|
||||
static inline int local_LZ4_compress_continue(const char* in, char* out, int inSize)
|
||||
static int local_LZ4_compress_continue(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compress_continue(ctx, in, out, inSize);
|
||||
}
|
||||
|
||||
static inline int local_LZ4_compress_limitedOutput_continue(const char* in, char* out, int inSize)
|
||||
static int local_LZ4_compress_limitedOutput_continue(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compress_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize));
|
||||
}
|
||||
|
||||
|
||||
LZ4_stream_t LZ4_dict;
|
||||
static void* local_LZ4_resetDictT(const char* fake)
|
||||
{
|
||||
(void)fake;
|
||||
memset(&LZ4_dict, 0, sizeof(LZ4_stream_t));
|
||||
return NULL;
|
||||
}
|
||||
|
||||
int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize);
|
||||
static int local_LZ4_compress_forceDict(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compress_forceExtDict(&LZ4_dict, in, out, inSize);
|
||||
}
|
||||
|
||||
|
||||
static void* stateLZ4HC;
|
||||
static inline int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize)
|
||||
static int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compressHC_withStateHC(stateLZ4HC, in, out, inSize);
|
||||
}
|
||||
|
||||
static inline int local_LZ4_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize)
|
||||
static int local_LZ4_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, in, out, inSize, LZ4_compressBound(inSize));
|
||||
}
|
||||
|
||||
static inline int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize)
|
||||
static int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compressHC_limitedOutput(in, out, inSize, LZ4_compressBound(inSize));
|
||||
}
|
||||
|
||||
static inline int local_LZ4_compressHC_continue(const char* in, char* out, int inSize)
|
||||
static int local_LZ4_compressHC_continue(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compressHC_continue(ctx, in, out, inSize);
|
||||
}
|
||||
|
||||
static inline int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize)
|
||||
static int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compressHC_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize));
|
||||
}
|
||||
|
||||
static inline int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize)
|
||||
static int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize)
|
||||
{
|
||||
(void)inSize;
|
||||
LZ4_decompress_fast(in, out, outSize);
|
||||
return outSize;
|
||||
}
|
||||
|
||||
static inline int local_LZ4_decompress_fast_withPrefix64k(const char* in, char* out, int inSize, int outSize)
|
||||
static int local_LZ4_decompress_fast_withPrefix64k(const char* in, char* out, int inSize, int outSize)
|
||||
{
|
||||
(void)inSize;
|
||||
LZ4_decompress_fast_withPrefix64k(in, out, outSize);
|
||||
return outSize;
|
||||
}
|
||||
|
||||
static inline int local_LZ4_decompress_safe_partial(const char* in, char* out, int inSize, int outSize)
|
||||
static int local_LZ4_decompress_fast_usingDict(const char* in, char* out, int inSize, int outSize)
|
||||
{
|
||||
(void)inSize;
|
||||
LZ4_decompress_fast_usingDict(in, out, outSize, in - 65536, 65536);
|
||||
return outSize;
|
||||
}
|
||||
|
||||
static int local_LZ4_decompress_safe_usingDict(const char* in, char* out, int inSize, int outSize)
|
||||
{
|
||||
(void)inSize;
|
||||
LZ4_decompress_safe_usingDict(in, out, inSize, outSize, in - 65536, 65536);
|
||||
return outSize;
|
||||
}
|
||||
|
||||
static int local_LZ4_decompress_safe_partial(const char* in, char* out, int inSize, int outSize)
|
||||
{
|
||||
return LZ4_decompress_safe_partial(in, out, inSize, outSize - 5, outSize);
|
||||
}
|
||||
@ -330,24 +360,16 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
|
||||
{
|
||||
int fileIdx=0;
|
||||
char* orig_buff;
|
||||
# define NB_COMPRESSION_ALGORITHMS 12
|
||||
# define NB_COMPRESSION_ALGORITHMS 13
|
||||
# define MINCOMPRESSIONCHAR '0'
|
||||
# define MAXCOMPRESSIONCHAR (MINCOMPRESSIONCHAR + NB_COMPRESSION_ALGORITHMS)
|
||||
static char* compressionNames[] = { "LZ4_compress", "LZ4_compress_limitedOutput",
|
||||
"LZ4_compress_withState", "LZ4_compress_limitedOutput_withState",
|
||||
"LZ4_compress_continue", "LZ4_compress_limitedOutput_continue",
|
||||
"LZ4_compressHC", "LZ4_compressHC_limitedOutput",
|
||||
"LZ4_compressHC_withStateHC", "LZ4_compressHC_limitedOutput_withStateHC",
|
||||
"LZ4_compressHC_continue", "LZ4_compressHC_limitedOutput_continue" };
|
||||
double totalCTime[NB_COMPRESSION_ALGORITHMS] = {0};
|
||||
double totalCSize[NB_COMPRESSION_ALGORITHMS] = {0};
|
||||
# define NB_DECOMPRESSION_ALGORITHMS 5
|
||||
double totalCTime[NB_COMPRESSION_ALGORITHMS+1] = {0};
|
||||
double totalCSize[NB_COMPRESSION_ALGORITHMS+1] = {0};
|
||||
# define NB_DECOMPRESSION_ALGORITHMS 7
|
||||
# define MINDECOMPRESSIONCHAR '0'
|
||||
# define MAXDECOMPRESSIONCHAR (MINDECOMPRESSIONCHAR + NB_DECOMPRESSION_ALGORITHMS)
|
||||
static char* decompressionNames[] = { "LZ4_decompress_fast", "LZ4_decompress_fast_withPrefix64k", "LZ4_decompress_safe", "LZ4_decompress_safe_withPrefix64k", "LZ4_decompress_safe_partial" };
|
||||
double totalDTime[NB_DECOMPRESSION_ALGORITHMS] = {0};
|
||||
|
||||
U64 totals = 0;
|
||||
static char* decompressionNames[] = { "LZ4_decompress_fast", "LZ4_decompress_fast_withPrefix64k", "LZ4_decompress_fast_usingDict",
|
||||
"LZ4_decompress_safe", "LZ4_decompress_safe_withPrefix64k", "LZ4_decompress_safe_usingDict", "LZ4_decompress_safe_partial" };
|
||||
double totalDTime[NB_DECOMPRESSION_ALGORITHMS+1] = {0};
|
||||
|
||||
|
||||
// Loop for each file
|
||||
@ -450,9 +472,9 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
|
||||
DISPLAY(" %s : \n", inFileName);
|
||||
|
||||
// Compression Algorithms
|
||||
for (cAlgNb=0; (cAlgNb < NB_COMPRESSION_ALGORITHMS) && (compressionTest); cAlgNb++)
|
||||
for (cAlgNb=1; (cAlgNb <= NB_COMPRESSION_ALGORITHMS) && (compressionTest); cAlgNb++)
|
||||
{
|
||||
char* cName = compressionNames[cAlgNb];
|
||||
char* compressorName;
|
||||
int (*compressionFunction)(const char*, char*, int);
|
||||
void* (*initFunction)(const char*) = NULL;
|
||||
double bestTime = 100000000.;
|
||||
@ -461,18 +483,19 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
|
||||
|
||||
switch(cAlgNb)
|
||||
{
|
||||
case 0 : compressionFunction = LZ4_compress; break;
|
||||
case 1 : compressionFunction = local_LZ4_compress_limitedOutput; break;
|
||||
case 2 : compressionFunction = local_LZ4_compress_withState; break;
|
||||
case 3 : compressionFunction = local_LZ4_compress_limitedOutput_withState; break;
|
||||
case 4 : compressionFunction = local_LZ4_compress_continue; initFunction = LZ4_create; break;
|
||||
case 5 : compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = LZ4_create; break;
|
||||
case 6 : compressionFunction = LZ4_compressHC; break;
|
||||
case 7 : compressionFunction = local_LZ4_compressHC_limitedOutput; break;
|
||||
case 8 : compressionFunction = local_LZ4_compressHC_withStateHC; break;
|
||||
case 9 : compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; break;
|
||||
case 10: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; break;
|
||||
case 11: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; break;
|
||||
case 1 : compressionFunction = LZ4_compress; compressorName = "LZ4_compress"; break;
|
||||
case 2 : compressionFunction = local_LZ4_compress_limitedOutput; compressorName = "LZ4_compress_limitedOutput"; break;
|
||||
case 3 : compressionFunction = local_LZ4_compress_withState; compressorName = "LZ4_compress_withState"; break;
|
||||
case 4 : compressionFunction = local_LZ4_compress_limitedOutput_withState; compressorName = "LZ4_compress_limitedOutput_withState"; break;
|
||||
case 5 : compressionFunction = local_LZ4_compress_continue; initFunction = LZ4_create; compressorName = "LZ4_compress_continue"; break;
|
||||
case 6 : compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = LZ4_create; compressorName = "LZ4_compress_limitedOutput_continue"; break;
|
||||
case 7 : compressionFunction = LZ4_compressHC; compressorName = "LZ4_compressHC"; break;
|
||||
case 8 : compressionFunction = local_LZ4_compressHC_limitedOutput; compressorName = "LZ4_compressHC_limitedOutput"; break;
|
||||
case 9 : compressionFunction = local_LZ4_compressHC_withStateHC; compressorName = "LZ4_compressHC_withStateHC"; break;
|
||||
case 10: compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; compressorName = "LZ4_compressHC_limitedOutput_withStateHC"; break;
|
||||
case 11: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_continue"; break;
|
||||
case 12: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_limitedOutput_continue"; break;
|
||||
case 13: compressionFunction = local_LZ4_compress_forceDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_forceDict"; break;
|
||||
default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1;
|
||||
}
|
||||
|
||||
@ -481,7 +504,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
|
||||
double averageTime;
|
||||
int milliTime;
|
||||
|
||||
PROGRESS("%1i-%-19.19s : %9i ->\r", loopNb, cName, (int)benchedSize);
|
||||
PROGRESS("%1i-%-26.26s : %9i ->\r", loopNb, compressorName, (int)benchedSize);
|
||||
{ size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; } // warmimg up memory
|
||||
|
||||
nb_loops = 0;
|
||||
@ -494,7 +517,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
|
||||
for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
|
||||
{
|
||||
chunkP[chunkNb].compressedSize = compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
|
||||
if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", cName), exit(1);
|
||||
if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", compressorName), exit(1);
|
||||
}
|
||||
if (initFunction!=NULL) free(ctx);
|
||||
nb_loops++;
|
||||
@ -505,13 +528,13 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
|
||||
if (averageTime < bestTime) bestTime = averageTime;
|
||||
cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize;
|
||||
ratio = (double)cSize/(double)benchedSize*100.;
|
||||
PROGRESS("%1i-%-19.19s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
|
||||
PROGRESS("%1i-%-26.26s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
|
||||
}
|
||||
|
||||
if (ratio<100.)
|
||||
DISPLAY("%-21.21s : %9i -> %9i (%5.2f%%),%7.1f MB/s\n", cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
|
||||
DISPLAY("%-28.28s : %9i -> %9i (%5.2f%%),%7.1f MB/s\n", compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
|
||||
else
|
||||
DISPLAY("%-21.21s : %9i -> %9i (%5.1f%%),%7.1f MB/s\n", cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
|
||||
DISPLAY("%-28.28s : %9i -> %9i (%5.1f%%),%7.1f MB/s\n", compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
|
||||
|
||||
totalCTime[cAlgNb] += bestTime;
|
||||
totalCSize[cAlgNb] += cSize;
|
||||
@ -521,7 +544,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
|
||||
for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
|
||||
{
|
||||
chunkP[chunkNb].compressedSize = LZ4_compress(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
|
||||
if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", compressionNames[0]), exit(1);
|
||||
if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", "LZ4_compress"), exit(1);
|
||||
}
|
||||
{ size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } // zeroing source area, for CRC checking
|
||||
|
||||
@ -532,16 +555,18 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
|
||||
int (*decompressionFunction)(const char*, char*, int, int);
|
||||
double bestTime = 100000000.;
|
||||
|
||||
if ((decompressionAlgo != ALL_DECOMPRESSORS) && (decompressionAlgo != dAlgNb)) continue;
|
||||
if ((decompressionAlgo != ALL_DECOMPRESSORS) && (decompressionAlgo != dAlgNb+1)) continue;
|
||||
|
||||
switch(dAlgNb)
|
||||
{
|
||||
case 0: decompressionFunction = local_LZ4_decompress_fast; break;
|
||||
case 1: decompressionFunction = local_LZ4_decompress_fast_withPrefix64k; break;
|
||||
case 2: decompressionFunction = LZ4_decompress_safe; break;
|
||||
case 3: decompressionFunction = LZ4_decompress_safe_withPrefix64k; break;
|
||||
case 4: decompressionFunction = local_LZ4_decompress_safe_partial; break;
|
||||
default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1;
|
||||
case 2: decompressionFunction = local_LZ4_decompress_fast_usingDict; break;
|
||||
case 3: decompressionFunction = LZ4_decompress_safe; break;
|
||||
case 4: decompressionFunction = LZ4_decompress_safe_withPrefix64k; break;
|
||||
case 5: decompressionFunction = local_LZ4_decompress_safe_usingDict; break;
|
||||
case 6: decompressionFunction = local_LZ4_decompress_safe_partial; break;
|
||||
default : DISPLAY("ERROR ! Bad decompression algorithm Id !! \n"); free(chunkP); return 1;
|
||||
}
|
||||
|
||||
for (loopNb = 1; loopNb <= nbIterations; loopNb++)
|
||||
@ -550,7 +575,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
|
||||
int milliTime;
|
||||
U32 crcDecoded;
|
||||
|
||||
PROGRESS("%1i-%-24.24s :%10i ->\r", loopNb, dName, (int)benchedSize);
|
||||
PROGRESS("%1i-%-29.29s :%10i ->\r", loopNb, dName, (int)benchedSize);
|
||||
|
||||
nb_loops = 0;
|
||||
milliTime = BMK_GetMilliStart();
|
||||
@ -570,19 +595,18 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
|
||||
averageTime = (double)milliTime / nb_loops;
|
||||
if (averageTime < bestTime) bestTime = averageTime;
|
||||
|
||||
PROGRESS("%1i-%-24.24s :%10i -> %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.);
|
||||
PROGRESS("%1i-%-29.29s :%10i -> %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.);
|
||||
|
||||
// CRC Checking
|
||||
crcDecoded = XXH32(orig_buff, (int)benchedSize, 0);
|
||||
if (crcOriginal!=crcDecoded) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOriginal, (unsigned)crcDecoded); exit(1); }
|
||||
}
|
||||
|
||||
DISPLAY("%-26.26s :%10i -> %7.1f MB/s\n", dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.);
|
||||
DISPLAY("%-31.31s :%10i -> %7.1f MB/s\n", dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.);
|
||||
|
||||
totalDTime[dAlgNb] += bestTime;
|
||||
}
|
||||
|
||||
totals += benchedSize;
|
||||
}
|
||||
|
||||
free(orig_buff);
|
||||
@ -590,25 +614,6 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
|
||||
free(chunkP);
|
||||
}
|
||||
|
||||
if (nbFiles > 1)
|
||||
{
|
||||
int AlgNb;
|
||||
|
||||
DISPLAY(" ** TOTAL ** : \n");
|
||||
for (AlgNb = 0; (AlgNb < NB_COMPRESSION_ALGORITHMS) && (compressionTest); AlgNb ++)
|
||||
{
|
||||
char* cName = compressionNames[AlgNb];
|
||||
if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != AlgNb)) continue;
|
||||
DISPLAY("%-21.21s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s\n", cName, (long long unsigned int)totals, (long long unsigned int)totalCSize[AlgNb], (double)totalCSize[AlgNb]/(double)totals*100., (double)totals/totalCTime[AlgNb]/1000.);
|
||||
}
|
||||
for (AlgNb = 0; (AlgNb < NB_DECOMPRESSION_ALGORITHMS) && (decompressionTest); AlgNb ++)
|
||||
{
|
||||
char* dName = decompressionNames[AlgNb];
|
||||
if ((decompressionAlgo != ALL_DECOMPRESSORS) && (decompressionAlgo != AlgNb)) continue;
|
||||
DISPLAY("%-21.21s :%10llu -> %6.1f MB/s\n", dName, (long long unsigned int)totals, (double)totals/totalDTime[AlgNb]/1000.);
|
||||
}
|
||||
}
|
||||
|
||||
if (BMK_pause) { printf("press enter...\n"); getchar(); }
|
||||
|
||||
return 0;
|
||||
@ -629,8 +634,8 @@ int usage(char* exename)
|
||||
int usage_advanced()
|
||||
{
|
||||
DISPLAY( "\nAdvanced options :\n");
|
||||
DISPLAY( " -c# : test only compression function # [%c-%c]\n", MINCOMPRESSIONCHAR, MAXCOMPRESSIONCHAR);
|
||||
DISPLAY( " -d# : test only decompression function # [%c-%c]\n", MINDECOMPRESSIONCHAR, MAXDECOMPRESSIONCHAR);
|
||||
DISPLAY( " -c# : test only compression function # [1-%i]\n", NB_COMPRESSION_ALGORITHMS);
|
||||
DISPLAY( " -d# : test only decompression function # [1-%i]\n", NB_DECOMPRESSION_ALGORITHMS);
|
||||
DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS);
|
||||
DISPLAY( " -B# : Block size [4-7](default : 7)\n");
|
||||
//DISPLAY( " -BD : Block dependency (improve compression ratio)\n");
|
||||
@ -652,7 +657,7 @@ int main(int argc, char** argv)
|
||||
char* input_filename=0;
|
||||
|
||||
// Welcome message
|
||||
DISPLAY( WELCOME_MESSAGE);
|
||||
DISPLAY(WELCOME_MESSAGE);
|
||||
|
||||
if (argc<2) { badusage(exename); return 1; }
|
||||
|
||||
@ -679,15 +684,23 @@ int main(int argc, char** argv)
|
||||
// Select compression algorithm only
|
||||
case 'c':
|
||||
decompressionTest = 0;
|
||||
if ((argument[1]>= MINCOMPRESSIONCHAR) && (argument[1]<= MAXCOMPRESSIONCHAR))
|
||||
compressionAlgo = argument[1] - '0', argument++;
|
||||
while ((argument[1]>= '0') && (argument[1]<= '9'))
|
||||
{
|
||||
compressionAlgo *= 10;
|
||||
compressionAlgo += argument[1] - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
|
||||
// Select decompression algorithm only
|
||||
case 'd':
|
||||
compressionTest = 0;
|
||||
if ((argument[1]>= MINDECOMPRESSIONCHAR) && (argument[1]<= MAXDECOMPRESSIONCHAR))
|
||||
decompressionAlgo = argument[1] - '0', argument++;
|
||||
while ((argument[1]>= '0') && (argument[1]<= '9'))
|
||||
{
|
||||
decompressionAlgo *= 10;
|
||||
decompressionAlgo += argument[1] - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
|
||||
// Display help on usage
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
/*
|
||||
fuzzer.c - Fuzzer test tool for LZ4
|
||||
Copyright (C) Yann Collet - Andrew Mahone 2012-2014
|
||||
Copyright (C) Yann Collet 2012-2014
|
||||
GPL v2 License
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
@ -22,47 +22,78 @@
|
||||
- LZ4 source repository : http://code.google.com/p/lz4/
|
||||
*/
|
||||
|
||||
//**************************************
|
||||
// Remove Visual warning messages
|
||||
//**************************************
|
||||
/**************************************
|
||||
Remove Visual warning messages
|
||||
**************************************/
|
||||
#define _CRT_SECURE_NO_WARNINGS // fgets
|
||||
|
||||
|
||||
//**************************************
|
||||
// Includes
|
||||
//**************************************
|
||||
/**************************************
|
||||
Includes
|
||||
**************************************/
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h> // fgets, sscanf
|
||||
#include <sys/timeb.h> // timeb
|
||||
#include <string.h> // strcmp
|
||||
#include "lz4.h"
|
||||
#include "lz4hc.h"
|
||||
#include "xxhash.h"
|
||||
|
||||
|
||||
//**************************************
|
||||
// Constants
|
||||
//**************************************
|
||||
#ifndef LZ4_VERSION
|
||||
# define LZ4_VERSION ""
|
||||
/**************************************
|
||||
Basic Types
|
||||
**************************************/
|
||||
#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
|
||||
# include <stdint.h>
|
||||
typedef uint8_t BYTE;
|
||||
typedef uint16_t U16;
|
||||
typedef uint32_t U32;
|
||||
typedef int32_t S32;
|
||||
typedef uint64_t U64;
|
||||
#else
|
||||
typedef unsigned char BYTE;
|
||||
typedef unsigned short U16;
|
||||
typedef unsigned int U32;
|
||||
typedef signed int S32;
|
||||
typedef unsigned long long U64;
|
||||
#endif
|
||||
|
||||
#define NB_ATTEMPTS (1<<17)
|
||||
#define LEN ((1<<15))
|
||||
#define SEQ_POW 2
|
||||
#define NUM_SEQ (1 << SEQ_POW)
|
||||
#define SEQ_MSK ((NUM_SEQ) - 1)
|
||||
#define MOD_SEQ(x) ((((x) >> 8) & 255) == 0)
|
||||
#define NEW_SEQ(x) ((((x) >> 10) %10) == 0)
|
||||
#define PAGE_SIZE 4096
|
||||
#define ROUND_PAGE(x) (((x) + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1))
|
||||
|
||||
/**************************************
|
||||
Constants
|
||||
**************************************/
|
||||
#ifndef LZ4_VERSION
|
||||
# define LZ4_VERSION "rc118"
|
||||
#endif
|
||||
|
||||
#define NB_ATTEMPTS (1<<16)
|
||||
#define COMPRESSIBLE_NOISE_LENGTH (1 << 21)
|
||||
#define FUZ_MAX_BLOCK_SIZE (1 << 17)
|
||||
#define FUZ_MAX_DICT_SIZE (1 << 15)
|
||||
#define FUZ_COMPRESSIBILITY_DEFAULT 50
|
||||
#define PRIME1 2654435761U
|
||||
#define PRIME2 2246822519U
|
||||
#define PRIME3 3266489917U
|
||||
|
||||
|
||||
//*********************************************************
|
||||
// Functions
|
||||
//*********************************************************
|
||||
//**************************************
|
||||
// Macros
|
||||
//**************************************
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
|
||||
|
||||
/*****************************************
|
||||
Local Parameters
|
||||
*****************************************/
|
||||
static int no_prompt = 0;
|
||||
static char* programName;
|
||||
static int displayLevel = 2;
|
||||
|
||||
|
||||
/*********************************************************
|
||||
Fuzzer functions
|
||||
*********************************************************/
|
||||
static int FUZ_GetMilliStart()
|
||||
{
|
||||
struct timeb tb;
|
||||
@ -82,20 +113,54 @@ static int FUZ_GetMilliSpan( int nTimeStart )
|
||||
}
|
||||
|
||||
|
||||
# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
|
||||
unsigned int FUZ_rand(unsigned int* src)
|
||||
{
|
||||
*src = ((*src) * PRIME1) + PRIME2;
|
||||
return *src;
|
||||
U32 rand32 = *src;
|
||||
rand32 *= PRIME1;
|
||||
rand32 += PRIME2;
|
||||
rand32 = FUZ_rotl32(rand32, 13);
|
||||
*src = rand32;
|
||||
return rand32 >> 3;
|
||||
}
|
||||
|
||||
|
||||
int test_canary(unsigned char *buf)
|
||||
#define FUZ_RAND15BITS ((FUZ_rand(seed) >> 3) & 32767)
|
||||
#define FUZ_RANDLENGTH ( ((FUZ_rand(seed) >> 7) & 3) ? (FUZ_rand(seed) % 14) : (FUZ_rand(seed) & 511) + 15)
|
||||
void FUZ_fillCompressibleNoiseBuffer(void* buffer, int bufferSize, double proba, U32* seed)
|
||||
{
|
||||
int i;
|
||||
for (i = 0; i < 2048; i++)
|
||||
if (buf[i] != buf[i + 2048])
|
||||
return 0;
|
||||
return 1;
|
||||
BYTE* BBuffer = (BYTE*)buffer;
|
||||
int pos = 0;
|
||||
U32 P32 = (U32)(32768 * proba);
|
||||
|
||||
// First Byte
|
||||
BBuffer[pos++] = (BYTE)(FUZ_rand(seed));
|
||||
|
||||
while (pos < bufferSize)
|
||||
{
|
||||
// Select : Literal (noise) or copy (within 64K)
|
||||
if (FUZ_RAND15BITS < P32)
|
||||
{
|
||||
// Copy (within 64K)
|
||||
int ref, d;
|
||||
int length = FUZ_RANDLENGTH + 4;
|
||||
int offset = FUZ_RAND15BITS + 1;
|
||||
if (offset > pos) offset = pos;
|
||||
if (pos + length > bufferSize) length = bufferSize - pos;
|
||||
ref = pos - offset;
|
||||
d = pos + length;
|
||||
while (pos < d) BBuffer[pos++] = BBuffer[ref++];
|
||||
}
|
||||
else
|
||||
{
|
||||
// Literal (noise)
|
||||
int d;
|
||||
int length = FUZ_RANDLENGTH;
|
||||
if (pos + length > bufferSize) length = bufferSize - pos;
|
||||
d = pos + length;
|
||||
while (pos < d) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -105,7 +170,8 @@ int FUZ_SecurityTest()
|
||||
char* input;
|
||||
int i, r;
|
||||
|
||||
printf("Overflow test (issue 52)...\n");
|
||||
// Overflow test, by Ludwig Strigeus
|
||||
printf("Overflow test (issue 52)...");
|
||||
input = (char*) malloc (20<<20);
|
||||
output = (char*) malloc (20<<20);
|
||||
input[0] = 0x0F;
|
||||
@ -121,27 +187,471 @@ int FUZ_SecurityTest()
|
||||
return 0;
|
||||
}
|
||||
|
||||
#define FUZ_MAX(a,b) (a>b?a:b)
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
const int no_prompt = (argc > 1) && (!strcmp(argv[1], "--no-prompt"));
|
||||
int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility) {
|
||||
unsigned long long bytes = 0;
|
||||
unsigned long long cbytes = 0;
|
||||
unsigned long long hcbytes = 0;
|
||||
unsigned char buf[LEN];
|
||||
unsigned char testOut[LEN+1];
|
||||
unsigned long long ccbytes = 0;
|
||||
void* CNBuffer;
|
||||
char* compressedBuffer;
|
||||
char* decodedBuffer;
|
||||
# define FUZ_max LZ4_COMPRESSBOUND(LEN)
|
||||
# define FUZ_avail ROUND_PAGE(FUZ_max)
|
||||
const int off_full = FUZ_avail - FUZ_max;
|
||||
unsigned char cbuf[FUZ_avail + PAGE_SIZE];
|
||||
unsigned int seed, randState, cur_seq=PRIME3, seeds[NUM_SEQ], timestamp=FUZ_GetMilliStart();
|
||||
int i, j, k, ret, len, lenHC, attemptNb;
|
||||
char userInput[30] = {0};
|
||||
# define FUZ_CHECKTEST(cond, message) if (cond) { printf("Test %i : %s : seed %u, cycle %i \n", testNb, message, seed, attemptNb); goto _output_error; }
|
||||
# define FUZ_DISPLAYTEST testNb++; no_prompt ? 0 : printf("%2i\b\b", testNb);
|
||||
unsigned int randState=seed;
|
||||
int ret, cycleNb;
|
||||
# define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %i : ", testNb); printf(__VA_ARGS__); \
|
||||
printf(" (seed %u, cycle %i) \n", seed, cycleNb); goto _output_error; }
|
||||
# define FUZ_DISPLAYTEST { testNb++; ((displayLevel<3) || no_prompt) ? 0 : printf("%2i\b\b", testNb); if (displayLevel==4) fflush(stdout); }
|
||||
void* stateLZ4 = malloc(LZ4_sizeofState());
|
||||
void* stateLZ4HC = malloc(LZ4_sizeofStateHC());
|
||||
void* LZ4continue;
|
||||
LZ4_stream_t LZ4dict;
|
||||
U32 crcOrig, crcCheck;
|
||||
int displayRefresh;
|
||||
|
||||
printf("starting LZ4 fuzzer (%s)\n", LZ4_VERSION);
|
||||
|
||||
// init
|
||||
memset(&LZ4dict, 0, sizeof(LZ4dict));
|
||||
|
||||
// Create compressible test buffer
|
||||
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
|
||||
FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
|
||||
compressedBuffer = malloc(LZ4_compressBound(FUZ_MAX_BLOCK_SIZE));
|
||||
decodedBuffer = malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE);
|
||||
|
||||
// display refresh rate
|
||||
switch(displayLevel)
|
||||
{
|
||||
case 0: displayRefresh = nbCycles+1; break;
|
||||
case 1: displayRefresh=FUZ_MAX(1, nbCycles / 100); break;
|
||||
case 2: displayRefresh=89; break;
|
||||
default : displayRefresh=1;
|
||||
}
|
||||
|
||||
// move to startCycle
|
||||
for (cycleNb = 0; cycleNb < startCycle; cycleNb++)
|
||||
{
|
||||
// synd rand & dict
|
||||
int dictSize, blockSize, blockStart;
|
||||
char* dict;
|
||||
char* block;
|
||||
|
||||
blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE;
|
||||
blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize);
|
||||
dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
|
||||
if (dictSize > blockStart) dictSize = blockStart;
|
||||
block = ((char*)CNBuffer) + blockStart;
|
||||
dict = block - dictSize;
|
||||
LZ4_loadDict(&LZ4dict, dict, dictSize);
|
||||
LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
|
||||
LZ4_loadDict(&LZ4dict, dict, dictSize);
|
||||
LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
|
||||
LZ4_loadDict(&LZ4dict, dict, dictSize);
|
||||
LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
|
||||
}
|
||||
|
||||
// Test loop
|
||||
for (cycleNb = startCycle; cycleNb < nbCycles; cycleNb++)
|
||||
{
|
||||
int testNb = 0;
|
||||
char* dict;
|
||||
char* block;
|
||||
int dictSize, blockSize, blockStart, compressedSize, HCcompressedSize;
|
||||
int blockContinueCompressedSize;
|
||||
|
||||
if ((cycleNb%displayRefresh) == 0)
|
||||
{
|
||||
printf("\r%7i /%7i - ", cycleNb, nbCycles);
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
// Select block to test
|
||||
blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE;
|
||||
blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize);
|
||||
dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
|
||||
if (dictSize > blockStart) dictSize = blockStart;
|
||||
block = ((char*)CNBuffer) + blockStart;
|
||||
dict = block - dictSize;
|
||||
|
||||
/* Compression tests */
|
||||
|
||||
// Test compression HC
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_compressHC(block, compressedBuffer, blockSize);
|
||||
FUZ_CHECKTEST(ret==0, "LZ4_compressHC() failed");
|
||||
HCcompressedSize = ret;
|
||||
|
||||
// Test compression HC using external state
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_compressHC_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize);
|
||||
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed");
|
||||
|
||||
// Test compression using external state
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_compress_withState(stateLZ4, block, compressedBuffer, blockSize);
|
||||
FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed");
|
||||
|
||||
// Test compression
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_compress(block, compressedBuffer, blockSize);
|
||||
FUZ_CHECKTEST(ret==0, "LZ4_compress() failed");
|
||||
compressedSize = ret;
|
||||
|
||||
/* Decompression tests */
|
||||
|
||||
crcOrig = XXH32(block, blockSize, 0);
|
||||
|
||||
// Test decoding with output size being exactly what's necessary => must work
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize);
|
||||
FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space");
|
||||
FUZ_CHECKTEST(ret!=compressedSize, "LZ4_decompress_fast failed : did not fully read compressed data");
|
||||
crcCheck = XXH32(decodedBuffer, blockSize, 0);
|
||||
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data");
|
||||
|
||||
// Test decoding with one byte missing => must fail
|
||||
FUZ_DISPLAYTEST;
|
||||
decodedBuffer[blockSize-1] = 0;
|
||||
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize-1);
|
||||
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small");
|
||||
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast overrun specified output buffer");
|
||||
|
||||
// Test decoding with one byte too much => must fail
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1);
|
||||
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large");
|
||||
|
||||
// Test decoding with output size exactly what's necessary => must work
|
||||
FUZ_DISPLAYTEST;
|
||||
decodedBuffer[blockSize] = 0;
|
||||
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize);
|
||||
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space");
|
||||
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data");
|
||||
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size");
|
||||
crcCheck = XXH32(decodedBuffer, blockSize, 0);
|
||||
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
|
||||
|
||||
// Test decoding with more than enough output size => must work
|
||||
FUZ_DISPLAYTEST;
|
||||
decodedBuffer[blockSize] = 0;
|
||||
decodedBuffer[blockSize+1] = 0;
|
||||
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize+1);
|
||||
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite amply sufficient space");
|
||||
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data");
|
||||
//FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe wrote more than (unknown) target size"); // well, is that an issue ?
|
||||
FUZ_CHECKTEST(decodedBuffer[blockSize+1], "LZ4_decompress_safe overrun specified output buffer size");
|
||||
crcCheck = XXH32(decodedBuffer, blockSize, 0);
|
||||
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
|
||||
|
||||
// Test decoding with output size being one byte too short => must fail
|
||||
FUZ_DISPLAYTEST;
|
||||
decodedBuffer[blockSize-1] = 0;
|
||||
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-1);
|
||||
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short");
|
||||
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe overrun specified output buffer size");
|
||||
|
||||
// Test decoding with output size being 10 bytes too short => must fail
|
||||
FUZ_DISPLAYTEST;
|
||||
if (blockSize>10)
|
||||
{
|
||||
decodedBuffer[blockSize-10] = 0;
|
||||
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-10);
|
||||
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being 10 bytes too short");
|
||||
FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe overrun specified output buffer size");
|
||||
}
|
||||
|
||||
// Test decoding with input size being one byte too short => must fail
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize-1, blockSize);
|
||||
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, ret=%i, compressedSize=%i)", blockSize, ret, compressedSize);
|
||||
|
||||
// Test decoding with input size being one byte too large => must fail
|
||||
FUZ_DISPLAYTEST;
|
||||
decodedBuffer[blockSize] = 0;
|
||||
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize+1, blockSize);
|
||||
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being too large");
|
||||
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size");
|
||||
|
||||
// Test partial decoding with target output size being max/2 => must work
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize/2, blockSize);
|
||||
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");
|
||||
|
||||
// Test partial decoding with target output size being just below max => must work
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize-3, blockSize);
|
||||
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");
|
||||
|
||||
/* Test Compression with limited output size */
|
||||
|
||||
// Test compression with output size being exactly what's necessary (should work)
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_compress_limitedOutput(block, compressedBuffer, blockSize, compressedSize);
|
||||
FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput() failed despite sufficient space");
|
||||
|
||||
// Test compression with output size being exactly what's necessary and external state (should work)
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_compress_limitedOutput_withState(stateLZ4, block, compressedBuffer, blockSize, compressedSize);
|
||||
FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput_withState() failed despite sufficient space");
|
||||
|
||||
// Test HC compression with output size being exactly what's necessary (should work)
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_compressHC_limitedOutput(block, compressedBuffer, blockSize, HCcompressedSize);
|
||||
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space");
|
||||
|
||||
// Test HC compression with output size being exactly what's necessary (should work)
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize);
|
||||
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput_withStateHC() failed despite sufficient space");
|
||||
|
||||
// Test compression with just one missing byte into output buffer => must fail
|
||||
FUZ_DISPLAYTEST;
|
||||
compressedBuffer[compressedSize-1] = 0;
|
||||
ret = LZ4_compress_limitedOutput(block, compressedBuffer, blockSize, compressedSize-1);
|
||||
FUZ_CHECKTEST(ret, "LZ4_compress_limitedOutput should have failed (output buffer too small by 1 byte)");
|
||||
FUZ_CHECKTEST(compressedBuffer[compressedSize-1], "LZ4_compress_limitedOutput overran output buffer")
|
||||
|
||||
// Test HC compression with just one missing byte into output buffer => must fail
|
||||
FUZ_DISPLAYTEST;
|
||||
compressedBuffer[compressedSize-1] = 0;
|
||||
ret = LZ4_compressHC_limitedOutput(block, compressedBuffer, blockSize, HCcompressedSize-1);
|
||||
FUZ_CHECKTEST(ret, "LZ4_compressHC_limitedOutput should have failed (output buffer too small by 1 byte)");
|
||||
FUZ_CHECKTEST(compressedBuffer[compressedSize-1], "LZ4_compressHC_limitedOutput overran output buffer")
|
||||
|
||||
/* Dictionary tests */
|
||||
|
||||
// Compress using dictionary
|
||||
FUZ_DISPLAYTEST;
|
||||
LZ4continue = LZ4_create (dict);
|
||||
LZ4_compress_continue (LZ4continue, dict, compressedBuffer, dictSize); // Just to fill hash tables
|
||||
blockContinueCompressedSize = LZ4_compress_continue (LZ4continue, block, compressedBuffer, blockSize);
|
||||
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed");
|
||||
LZ4_free (LZ4continue);
|
||||
|
||||
// Decompress with dictionary as prefix
|
||||
FUZ_DISPLAYTEST;
|
||||
memcpy(decodedBuffer, dict, dictSize);
|
||||
ret = LZ4_decompress_fast_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockSize);
|
||||
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_withPrefix64k did not read all compressed block input");
|
||||
crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
|
||||
if (crcCheck!=crcOrig)
|
||||
{
|
||||
int i=0;
|
||||
while (block[i]==decodedBuffer[i]) i++;
|
||||
printf("Wrong Byte at position %i/%i\n", i, blockSize);
|
||||
|
||||
}
|
||||
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_withPrefix64k corrupted decoded data (dict %i)", dictSize);
|
||||
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_decompress_safe_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize);
|
||||
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_withPrefix64k did not regenerate original data");
|
||||
crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
|
||||
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_withPrefix64k corrupted decoded data");
|
||||
|
||||
// Compress using External dictionary
|
||||
FUZ_DISPLAYTEST;
|
||||
dict -= 9; // Separation, so it is an ExtDict
|
||||
if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
|
||||
LZ4_loadDict(&LZ4dict, dict, dictSize);
|
||||
blockContinueCompressedSize = LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
|
||||
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed");
|
||||
|
||||
FUZ_DISPLAYTEST;
|
||||
LZ4_loadDict(&LZ4dict, dict, dictSize);
|
||||
ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
|
||||
FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer");
|
||||
|
||||
FUZ_DISPLAYTEST;
|
||||
LZ4_loadDict(&LZ4dict, dict, dictSize);
|
||||
ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize);
|
||||
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
|
||||
FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer");
|
||||
|
||||
// Decompress with dictionary as external
|
||||
FUZ_DISPLAYTEST;
|
||||
decodedBuffer[blockSize] = 0;
|
||||
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
|
||||
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
|
||||
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size")
|
||||
crcCheck = XXH32(decodedBuffer, blockSize, 0);
|
||||
if (crcCheck!=crcOrig)
|
||||
{
|
||||
int i=0;
|
||||
while (block[i]==decodedBuffer[i]) i++;
|
||||
printf("Wrong Byte at position %i/%i\n", i, blockSize);
|
||||
|
||||
}
|
||||
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
|
||||
|
||||
FUZ_DISPLAYTEST;
|
||||
decodedBuffer[blockSize] = 0;
|
||||
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
|
||||
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
|
||||
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size")
|
||||
crcCheck = XXH32(decodedBuffer, blockSize, 0);
|
||||
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
|
||||
|
||||
FUZ_DISPLAYTEST;
|
||||
decodedBuffer[blockSize-1] = 0;
|
||||
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
|
||||
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_withDict should have failed : wrong original size (-1 byte)");
|
||||
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
|
||||
|
||||
FUZ_DISPLAYTEST;
|
||||
decodedBuffer[blockSize-1] = 0;
|
||||
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize);
|
||||
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
|
||||
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
|
||||
|
||||
FUZ_DISPLAYTEST;
|
||||
if (blockSize > 10)
|
||||
{
|
||||
decodedBuffer[blockSize-10] = 0;
|
||||
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-10, dict, dictSize);
|
||||
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-10 byte)");
|
||||
FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-10 byte) (blockSize=%i)", blockSize);
|
||||
}
|
||||
|
||||
|
||||
// Fill stats
|
||||
bytes += blockSize;
|
||||
cbytes += compressedSize;
|
||||
hcbytes += HCcompressedSize;
|
||||
ccbytes += blockContinueCompressedSize;
|
||||
}
|
||||
|
||||
printf("\r%7i /%7i - ", cycleNb, nbCycles);
|
||||
printf("all tests completed successfully \n");
|
||||
printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100);
|
||||
printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100);
|
||||
printf("ratio with dict: %0.3f%%\n", (double)ccbytes/bytes*100);
|
||||
|
||||
// unalloc
|
||||
if(!no_prompt) getchar();
|
||||
free(CNBuffer);
|
||||
free(compressedBuffer);
|
||||
free(decodedBuffer);
|
||||
free(stateLZ4);
|
||||
free(stateLZ4HC);
|
||||
return 0;
|
||||
|
||||
_output_error:
|
||||
if(!no_prompt) getchar();
|
||||
free(CNBuffer);
|
||||
free(compressedBuffer);
|
||||
free(decodedBuffer);
|
||||
free(stateLZ4);
|
||||
free(stateLZ4HC);
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
int FUZ_usage()
|
||||
{
|
||||
DISPLAY( "Usage :\n");
|
||||
DISPLAY( " %s [args]\n", programName);
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Arguments :\n");
|
||||
DISPLAY( " -i# : Nb of tests (default:%i) \n", NB_ATTEMPTS);
|
||||
DISPLAY( " -s# : Select seed (default:prompt user)\n");
|
||||
DISPLAY( " -t# : Select starting test number (default:0)\n");
|
||||
DISPLAY( " -p# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
|
||||
DISPLAY( " -v : verbose\n");
|
||||
DISPLAY( " -h : display help and exit\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
U32 timestamp = FUZ_GetMilliStart();
|
||||
U32 seed=0;
|
||||
int seedset=0;
|
||||
int argNb;
|
||||
int nbTests = NB_ATTEMPTS;
|
||||
int testNb = 0;
|
||||
int proba = FUZ_COMPRESSIBILITY_DEFAULT;
|
||||
|
||||
// Check command line
|
||||
programName = argv[0];
|
||||
for(argNb=1; argNb<argc; argNb++)
|
||||
{
|
||||
char* argument = argv[argNb];
|
||||
|
||||
if(!argument) continue; // Protection if argument empty
|
||||
|
||||
// Decode command (note : aggregated commands are allowed)
|
||||
if (argument[0]=='-')
|
||||
{
|
||||
if (!strcmp(argument, "--no-prompt")) { no_prompt=1; seedset=1; displayLevel=1; continue; }
|
||||
|
||||
while (argument[1]!=0)
|
||||
{
|
||||
argument++;
|
||||
switch(*argument)
|
||||
{
|
||||
case 'h':
|
||||
return FUZ_usage();
|
||||
case 'v':
|
||||
argument++;
|
||||
displayLevel=4;
|
||||
break;
|
||||
case 'i':
|
||||
argument++;
|
||||
nbTests=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
nbTests *= 10;
|
||||
nbTests += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
case 's':
|
||||
argument++;
|
||||
seed=0; seedset=1;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
seed *= 10;
|
||||
seed += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
case 't':
|
||||
argument++;
|
||||
testNb=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
testNb *= 10;
|
||||
testNb += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
case 'p':
|
||||
argument++;
|
||||
proba=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
proba *= 10;
|
||||
proba += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
if (proba<0) proba=0;
|
||||
if (proba>100) proba=100;
|
||||
break;
|
||||
default: ;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// Get Seed
|
||||
printf("Starting LZ4 fuzzer (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION);
|
||||
|
||||
if (!seedset)
|
||||
{
|
||||
char userInput[50] = {0};
|
||||
printf("Select an Initialisation number (default : random) : ");
|
||||
fflush(stdout);
|
||||
if ( no_prompt || fgets(userInput, sizeof userInput, stdin) )
|
||||
@ -149,165 +659,13 @@ int main(int argc, char** argv) {
|
||||
if ( sscanf(userInput, "%u", &seed) == 1 ) {}
|
||||
else seed = FUZ_GetMilliSpan(timestamp);
|
||||
}
|
||||
printf("Seed = %u\n", seed);
|
||||
randState = seed;
|
||||
}
|
||||
printf("Seed = %u\n", seed);
|
||||
if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) printf("Compressibility : %i%%\n", proba);
|
||||
|
||||
//FUZ_SecurityTest();
|
||||
FUZ_SecurityTest();
|
||||
|
||||
for (i = 0; i < 2048; i++)
|
||||
cbuf[FUZ_avail + i] = cbuf[FUZ_avail + 2048 + i] = FUZ_rand(&randState) >> 16;
|
||||
if (nbTests<=0) nbTests=1;
|
||||
|
||||
for (attemptNb = 0; attemptNb < NB_ATTEMPTS; attemptNb++)
|
||||
{
|
||||
int testNb = 0;
|
||||
|
||||
// note : promptThrottle is throtting stdout to prevent
|
||||
// Travis-CI's output limit (10MB) and false hangup detection.
|
||||
const int promptThrottle = (attemptNb % (NB_ATTEMPTS / 100) == 0);
|
||||
if (!no_prompt || attemptNb == 0 || promptThrottle)
|
||||
{
|
||||
printf("\r%7i /%7i - ", attemptNb, NB_ATTEMPTS);
|
||||
if (no_prompt)
|
||||
{
|
||||
fflush(stdout);
|
||||
}
|
||||
}
|
||||
|
||||
for (j = 0; j < NUM_SEQ; j++) {
|
||||
seeds[j] = FUZ_rand(&randState) << 8;
|
||||
seeds[j] ^= (FUZ_rand(&randState) >> 8) & 65535;
|
||||
}
|
||||
for (j = 0; j < LEN; j++) {
|
||||
k = FUZ_rand(&randState);
|
||||
if (j == 0 || NEW_SEQ(k))
|
||||
cur_seq = seeds[(FUZ_rand(&randState) >> 16) & SEQ_MSK];
|
||||
if (MOD_SEQ(k)) {
|
||||
k = (FUZ_rand(&randState) >> 16) & SEQ_MSK;
|
||||
seeds[k] = FUZ_rand(&randState) << 8;
|
||||
seeds[k] ^= (FUZ_rand(&randState) >> 8) & 65535;
|
||||
}
|
||||
buf[j] = FUZ_rand(&cur_seq) >> 16;
|
||||
}
|
||||
|
||||
// Test compression HC
|
||||
FUZ_DISPLAYTEST; // 1
|
||||
ret = LZ4_compressHC_limitedOutput((const char*)buf, (char*)&cbuf[off_full], LEN, FUZ_max);
|
||||
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space");
|
||||
lenHC = ret;
|
||||
|
||||
// Test compression HC using external state
|
||||
FUZ_DISPLAYTEST; // 1
|
||||
ret = LZ4_compressHC_withStateHC(stateLZ4HC, (const char*)buf, (char*)&cbuf[off_full], LEN);
|
||||
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed");
|
||||
|
||||
// Test compression using external state
|
||||
FUZ_DISPLAYTEST; // 2
|
||||
ret = LZ4_compress_withState(stateLZ4, (const char*)buf, (char*)&cbuf[off_full], LEN);
|
||||
FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed");
|
||||
|
||||
// Test compression
|
||||
FUZ_DISPLAYTEST; // 2
|
||||
ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[off_full], LEN, FUZ_max);
|
||||
FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput() failed despite sufficient space");
|
||||
len = ret;
|
||||
|
||||
// Test decoding with output size being exactly what's necessary => must work
|
||||
FUZ_DISPLAYTEST; // 3
|
||||
ret = LZ4_decompress_fast((char*)&cbuf[off_full], (char*)testOut, LEN);
|
||||
FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space");
|
||||
|
||||
// Test decoding with one byte missing => must fail
|
||||
FUZ_DISPLAYTEST; // 4
|
||||
ret = LZ4_decompress_fast((char*)&cbuf[off_full], (char*)testOut, LEN-1);
|
||||
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small");
|
||||
|
||||
// Test decoding with one byte too much => must fail
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_decompress_fast((char*)&cbuf[off_full], (char*)testOut, LEN+1);
|
||||
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large");
|
||||
|
||||
// Test decoding with enough output size => must work
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len, LEN+1);
|
||||
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space");
|
||||
|
||||
// Test decoding with output size being exactly what's necessary => must work
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len, LEN);
|
||||
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space");
|
||||
|
||||
// Test decoding with output size being one byte too short => must fail
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len, LEN-1);
|
||||
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short");
|
||||
|
||||
// Test decoding with input size being one byte too short => must fail
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len-1, LEN);
|
||||
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short");
|
||||
|
||||
// Test decoding with input size being one byte too large => must fail
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len+1, LEN);
|
||||
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being too large");
|
||||
//if (ret>=0) { printf("Test 10 : decompression should have failed, due to input size being too large : seed %u, len %d\n", seed, LEN); goto _output_error; }
|
||||
|
||||
// Test partial decoding with target output size being max/2 => must work
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_decompress_safe_partial((char*)&cbuf[off_full], (char*)testOut, len, LEN/2, LEN);
|
||||
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");
|
||||
|
||||
// Test partial decoding with target output size being just below max => must work
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_decompress_safe_partial((char*)&cbuf[off_full], (char*)testOut, len, LEN-3, LEN);
|
||||
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");
|
||||
|
||||
// Test compression with output size being exactly what's necessary (should work)
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-len], LEN, len);
|
||||
FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput() failed despite sufficient space");
|
||||
FUZ_CHECKTEST(!test_canary(&cbuf[FUZ_avail]), "compression overran output buffer");
|
||||
|
||||
// Test compression with output size being exactly what's necessary and external state (should work)
|
||||
FUZ_DISPLAYTEST; // 2
|
||||
ret = LZ4_compress_limitedOutput_withState(stateLZ4, (const char*)buf, (char*)&cbuf[off_full], LEN, len);
|
||||
FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput_withState() failed despite sufficient space");
|
||||
FUZ_CHECKTEST(!test_canary(&cbuf[FUZ_avail]), "compression overran output buffer");
|
||||
|
||||
// Test HC compression with output size being exactly what's necessary (should work)
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_compressHC_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-len], LEN, lenHC);
|
||||
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space");
|
||||
|
||||
// Test HC compression with output size being exactly what's necessary (should work)
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, (const char*)buf, (char*)&cbuf[FUZ_avail-len], LEN, lenHC);
|
||||
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput_withStateHC() failed despite sufficient space");
|
||||
|
||||
// Test compression with just one missing byte into output buffer => must fail
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-(len-1)], LEN, len-1);
|
||||
FUZ_CHECKTEST(ret, "compression overran output buffer");
|
||||
FUZ_CHECKTEST(!test_canary(&cbuf[FUZ_avail]), "compression overran output buffer");
|
||||
|
||||
// Test HC compression with just one missing byte into output buffer => must fail
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_compressHC_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-(len-1)], LEN, lenHC-1);
|
||||
FUZ_CHECKTEST(ret, "HC compression overran output buffer");
|
||||
|
||||
bytes += LEN;
|
||||
cbytes += len;
|
||||
hcbytes += lenHC;
|
||||
FUZ_rand(&randState);
|
||||
}
|
||||
|
||||
printf("all tests completed successfully \n");
|
||||
printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100);
|
||||
printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100);
|
||||
if(!no_prompt) getchar();
|
||||
return 0;
|
||||
|
||||
_output_error:
|
||||
if(!no_prompt) getchar();
|
||||
return 1;
|
||||
return FUZ_test(seed, nbTests, testNb, ((double)proba) / 100);
|
||||
}
|
||||
|
||||
@ -64,6 +64,7 @@ following options
|
||||
.TP
|
||||
.B \-B#
|
||||
block size [4-7](default : 7)
|
||||
B4= 64KB ; B5= 256KB ; B6= 1MB ; B7= 4MB
|
||||
.TP
|
||||
.B \-BD
|
||||
block dependency (improve compression ratio)
|
||||
@ -84,4 +85,4 @@ following options
|
||||
Report bugs at:- https://code.google.com/p/lz4/
|
||||
|
||||
.SH AUTHOR
|
||||
Yann Collet
|
||||
Yann Collet
|
||||
|
||||
@ -7,28 +7,26 @@
|
||||
.hy 0
|
||||
.nr HY 0
|
||||
|
||||
.TH lz4cat "1" "2014-04-15" "lz4cat" "User Commands"
|
||||
.TH lz4cat "1" "2014-06-20" "lz4cat" "User Commands"
|
||||
.SH NAME
|
||||
\fBlz4cat\fR - Extremely fast compression algorithm
|
||||
\fBlz4cat\fR - Utility based on LZ4
|
||||
|
||||
.SH SYNOPSIS
|
||||
.TP 5
|
||||
\fBlz4cat\fR [\fBOPTIONS\fR] [-|INPUT-FILE] <OUTPUT-FILE>
|
||||
\fBlz4cat\fR [\fBOPTIONS\fR] [-|INPUT-FILE]
|
||||
|
||||
.SH DESCRIPTION
|
||||
.PP
|
||||
\fBlz4\fR is an extremely fast lossless compression algorithm.
|
||||
\fBlz4cat\fR is an utility based on \fBlz4\fR, an extremely fast lossless compression algorithm.
|
||||
|
||||
\fBlz4cat\fR is an utility based on \fBlz4\fR.
|
||||
\fBlz4cat\fR decompress input file or stream, redirecting its output to the console.
|
||||
It is equivalent to \fBlz4 -cd\fR,
|
||||
|
||||
\fBlz4cat\fR is equivalent to \fBlz4 -cd\fR,
|
||||
which forces decompression and redirect its output to the console.
|
||||
|
||||
All other options are the same as \fBlz4\fR ones (man lz4).
|
||||
Available options are the same as \fBlz4\fR ones (man lz4).
|
||||
|
||||
|
||||
.SH BUGS
|
||||
Report bugs at:- https://code.google.com/p/lz4/
|
||||
|
||||
.SH AUTHOR
|
||||
Yann Collet
|
||||
Yann Collet
|
||||
|
||||
@ -109,7 +109,7 @@
|
||||
//****************************
|
||||
#define COMPRESSOR_NAME "LZ4 Compression CLI"
|
||||
#ifndef LZ4_VERSION
|
||||
# define LZ4_VERSION "v1.1.5"
|
||||
# define LZ4_VERSION "v1.2.0"
|
||||
#endif
|
||||
#define AUTHOR "Yann Collet"
|
||||
#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_VERSION, AUTHOR, __DATE__
|
||||
@ -344,6 +344,19 @@ int main(int argc, char** argv)
|
||||
if (*argument=='s') { displayLevel=1; continue; } // -s (silent mode)
|
||||
#endif // DISABLE_LZ4C_LEGACY_OPTIONS
|
||||
|
||||
if ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
cLevel = 0;
|
||||
while ((*argument >= '0') && (*argument <= '9'))
|
||||
{
|
||||
cLevel *= 10;
|
||||
cLevel += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
argument--;
|
||||
continue;
|
||||
}
|
||||
|
||||
switch(argument[0])
|
||||
{
|
||||
// Display help
|
||||
@ -354,20 +367,6 @@ int main(int argc, char** argv)
|
||||
// Compression (default)
|
||||
case 'z': forceCompress = 1; break;
|
||||
|
||||
// Compression level
|
||||
case '0':
|
||||
case '1':
|
||||
case '2':
|
||||
case '3':
|
||||
case '4':
|
||||
case '5':
|
||||
case '6':
|
||||
case '7':
|
||||
case '8':
|
||||
case '9':
|
||||
case 'A': /* non documented (hidden) */
|
||||
cLevel=*argument -'0'; break;
|
||||
|
||||
// Use Legacy format (for Linux kernel compression)
|
||||
case 'l': legacy_format=1; break;
|
||||
|
||||
@ -405,9 +404,8 @@ int main(int argc, char** argv)
|
||||
case '7':
|
||||
{
|
||||
int B = argument[1] - '0';
|
||||
int S = 1 << (8 + 2*B);
|
||||
BMK_SetBlocksize(S);
|
||||
blockSize = LZ4IO_setBlockSizeID(B);
|
||||
BMK_SetBlocksize(blockSize);
|
||||
argument++;
|
||||
break;
|
||||
}
|
||||
|
||||
134
programs/lz4io.c
134
programs/lz4io.c
@ -121,7 +121,7 @@
|
||||
|
||||
#define CACHELINE 64
|
||||
#define LEGACY_BLOCKSIZE (8 MB)
|
||||
#define MIN_STREAM_BUFSIZE (1 MB + 64 KB)
|
||||
#define MIN_STREAM_BUFSIZE (192 KB)
|
||||
#define LZ4S_BLOCKSIZEID_DEFAULT 7
|
||||
#define LZ4S_CHECKSUM_SEED 0
|
||||
#define LZ4S_EOS 0
|
||||
@ -365,17 +365,37 @@ int LZ4IO_compressFilename_Legacy(char* input_filename, char* output_filename, i
|
||||
}
|
||||
|
||||
|
||||
static void* LZ4IO_LZ4_createStream (const char* inputBuffer)
|
||||
{
|
||||
(void)inputBuffer;
|
||||
return LZ4_createStream();
|
||||
}
|
||||
|
||||
static int LZ4IO_LZ4_compress_limitedOutput_continue (void* ctx, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel)
|
||||
{
|
||||
(void)compressionLevel;
|
||||
return LZ4_compress_limitedOutput_continue(ctx, source, dest, inputSize, maxOutputSize);
|
||||
}
|
||||
|
||||
static int LZ4IO_LZ4_slideInputBufferHC (void* ctx, char* buffer, int size)
|
||||
{
|
||||
(void)size; (void)buffer;
|
||||
LZ4_slideInputBufferHC (ctx);
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
static int compress_file_blockDependency(char* input_filename, char* output_filename, int compressionlevel)
|
||||
{
|
||||
void* (*initFunction) (const char*);
|
||||
int (*compressionFunction)(void*, const char*, char*, int, int);
|
||||
char* (*translateFunction) (void*);
|
||||
int (*compressionFunction)(void*, const char*, char*, int, int, int);
|
||||
int (*nextBlockFunction) (void*, char*, int);
|
||||
int (*freeFunction) (void*);
|
||||
void* ctx;
|
||||
unsigned long long filesize = 0;
|
||||
unsigned long long compressedfilesize = 0;
|
||||
unsigned int checkbits;
|
||||
char* in_buff, *in_start, *in_end;
|
||||
char* in_buff, *in_blockStart;
|
||||
char* out_buff;
|
||||
FILE* finput;
|
||||
FILE* foutput;
|
||||
@ -384,34 +404,35 @@ static int compress_file_blockDependency(char* input_filename, char* output_file
|
||||
size_t sizeCheck, header_size;
|
||||
void* streamChecksumState=NULL;
|
||||
|
||||
|
||||
// Init
|
||||
start = clock();
|
||||
if ((displayLevel==2) && (compressionlevel>=3)) displayLevel=3;
|
||||
if (compressionlevel>=3)
|
||||
|
||||
if (compressionlevel<3)
|
||||
{
|
||||
initFunction = LZ4_createHC;
|
||||
compressionFunction = LZ4_compressHC_limitedOutput_continue;
|
||||
translateFunction = LZ4_slideInputBufferHC;
|
||||
freeFunction = LZ4_freeHC;
|
||||
initFunction = LZ4IO_LZ4_createStream;
|
||||
compressionFunction = LZ4IO_LZ4_compress_limitedOutput_continue;
|
||||
nextBlockFunction = LZ4_saveDict;
|
||||
freeFunction = LZ4_free;
|
||||
}
|
||||
else
|
||||
{
|
||||
initFunction = LZ4_create;
|
||||
compressionFunction = LZ4_compress_limitedOutput_continue;
|
||||
translateFunction = LZ4_slideInputBuffer;
|
||||
initFunction = LZ4_createHC;
|
||||
compressionFunction = LZ4_compressHC2_limitedOutput_continue;
|
||||
nextBlockFunction = LZ4IO_LZ4_slideInputBufferHC;
|
||||
freeFunction = LZ4_free;
|
||||
}
|
||||
|
||||
get_fileHandle(input_filename, output_filename, &finput, &foutput);
|
||||
blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId);
|
||||
|
||||
// Allocate Memory
|
||||
inputBufferSize = blockSize + 64 KB;
|
||||
if (inputBufferSize < MIN_STREAM_BUFSIZE) inputBufferSize = MIN_STREAM_BUFSIZE;
|
||||
inputBufferSize = 64 KB + blockSize;
|
||||
in_buff = (char*)malloc(inputBufferSize);
|
||||
out_buff = (char*)malloc(blockSize+CACHELINE);
|
||||
if (!in_buff || !out_buff) EXM_THROW(31, "Allocation error : not enough memory");
|
||||
in_start = in_buff; in_end = in_buff + inputBufferSize;
|
||||
in_blockStart = in_buff + 64 KB;
|
||||
if (compressionlevel>=3) in_blockStart = in_buff;
|
||||
if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);
|
||||
ctx = initFunction(in_buff);
|
||||
|
||||
@ -435,19 +456,19 @@ static int compress_file_blockDependency(char* input_filename, char* output_file
|
||||
{
|
||||
unsigned int outSize;
|
||||
unsigned int inSize;
|
||||
|
||||
// Read Block
|
||||
if ((in_start+blockSize) > in_end) in_start = translateFunction(ctx);
|
||||
inSize = (unsigned int) fread(in_start, (size_t)1, (size_t)blockSize, finput);
|
||||
inSize = (unsigned int) fread(in_blockStart, (size_t)1, (size_t)blockSize, finput);
|
||||
if( inSize==0 ) break; // No more input : end of compression
|
||||
filesize += inSize;
|
||||
DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20));
|
||||
if (streamChecksum) XXH32_update(streamChecksumState, in_start, inSize);
|
||||
if (streamChecksum) XXH32_update(streamChecksumState, in_blockStart, inSize);
|
||||
|
||||
// Compress Block
|
||||
outSize = compressionFunction(ctx, in_start, out_buff+4, inSize, inSize-1);
|
||||
outSize = compressionFunction(ctx, in_blockStart, out_buff+4, inSize, inSize-1, compressionlevel);
|
||||
if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += inSize+4;
|
||||
if (blockChecksum) compressedfilesize+=4;
|
||||
DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);
|
||||
DISPLAYLEVEL(3, "==> %.2f%% ", (double)compressedfilesize/filesize*100);
|
||||
|
||||
// Write Block
|
||||
if (outSize > 0)
|
||||
@ -462,24 +483,28 @@ static int compress_file_blockDependency(char* input_filename, char* output_file
|
||||
sizeToWrite = 4 + outSize + (4*blockChecksum);
|
||||
sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput);
|
||||
if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block");
|
||||
|
||||
}
|
||||
else // Copy Original
|
||||
{
|
||||
* (unsigned int*) out_buff = LITTLE_ENDIAN_32(inSize|0x80000000); // Add Uncompressed flag
|
||||
sizeCheck = fwrite(out_buff, 1, 4, foutput);
|
||||
if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header");
|
||||
sizeCheck = fwrite(in_start, 1, inSize, foutput);
|
||||
sizeCheck = fwrite(in_blockStart, 1, inSize, foutput);
|
||||
if (sizeCheck!=(size_t)(inSize)) EXM_THROW(35, "Write error : cannot write block");
|
||||
if (blockChecksum)
|
||||
{
|
||||
unsigned int checksum = XXH32(in_start, inSize, LZ4S_CHECKSUM_SEED);
|
||||
unsigned int checksum = XXH32(in_blockStart, inSize, LZ4S_CHECKSUM_SEED);
|
||||
* (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);
|
||||
sizeCheck = fwrite(out_buff, 1, 4, foutput);
|
||||
if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum");
|
||||
}
|
||||
}
|
||||
in_start += inSize;
|
||||
{
|
||||
size_t sizeToMove = 64 KB;
|
||||
if (inSize < 64 KB) sizeToMove = inSize;
|
||||
nextBlockFunction(ctx, in_blockStart - sizeToMove, (int)sizeToMove);
|
||||
if (compressionlevel>=3) in_blockStart = in_buff + 64 KB;
|
||||
}
|
||||
}
|
||||
|
||||
// End of Stream mark
|
||||
@ -542,7 +567,7 @@ int LZ4IO_compressFilename(char* input_filename, char* output_filename, int comp
|
||||
// Init
|
||||
start = clock();
|
||||
if ((displayLevel==2) && (compressionLevel>=3)) displayLevel=3;
|
||||
if (compressionLevel <= 3) compressionFunction = LZ4_compress_limitedOutput_local;
|
||||
if (compressionLevel <= 3) compressionFunction = LZ4_compress_limitedOutput_local;
|
||||
else { compressionFunction = LZ4_compressHC2_limitedOutput; }
|
||||
get_fileHandle(input_filename, output_filename, &finput, &foutput);
|
||||
blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId);
|
||||
@ -587,7 +612,7 @@ int LZ4IO_compressFilename(char* input_filename, char* output_filename, int comp
|
||||
outSize = compressionFunction(in_buff, out_buff+4, (int)readSize, (int)readSize-1, compressionLevel);
|
||||
if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += readSize+4;
|
||||
if (blockChecksum) compressedfilesize+=4;
|
||||
DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);
|
||||
DISPLAYLEVEL(3, "==> %.2f%% ", (double)compressedfilesize/filesize*100);
|
||||
|
||||
// Write Block
|
||||
if (outSize > 0)
|
||||
@ -724,8 +749,11 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
|
||||
size_t sizeCheck;
|
||||
int blockChecksumFlag, streamChecksumFlag, blockIndependenceFlag;
|
||||
void* streamChecksumState=NULL;
|
||||
int (*decompressionFunction)(const char*, char*, int, int) = LZ4_decompress_safe;
|
||||
unsigned int prefix64k = 0;
|
||||
int (*decompressionFunction)(void* ctx, const char* src, char* dst, int cSize, int maxOSize) = LZ4_decompress_safe_continue;
|
||||
LZ4_streamDecode_t ctx;
|
||||
|
||||
// init
|
||||
memset(&ctx, 0, sizeof(ctx));
|
||||
|
||||
// Decode stream descriptor
|
||||
nbReadBytes = fread(descriptor, 1, 3, finput);
|
||||
@ -761,23 +789,17 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
|
||||
if (checkBits != checkBits_xxh32) EXM_THROW(69, "Stream descriptor error detected");
|
||||
}
|
||||
|
||||
if (!blockIndependenceFlag)
|
||||
{
|
||||
decompressionFunction = LZ4_decompress_safe_withPrefix64k;
|
||||
prefix64k = 64 KB;
|
||||
}
|
||||
|
||||
// Allocate Memory
|
||||
{
|
||||
unsigned int outbuffSize = prefix64k+maxBlockSize;
|
||||
size_t outBuffSize = maxBlockSize + 64 KB;
|
||||
if (outBuffSize < MIN_STREAM_BUFSIZE) outBuffSize = MIN_STREAM_BUFSIZE;
|
||||
in_buff = (char*)malloc(maxBlockSize);
|
||||
if (outbuffSize < MIN_STREAM_BUFSIZE) outbuffSize = MIN_STREAM_BUFSIZE;
|
||||
out_buff = (char*)malloc(outbuffSize);
|
||||
out_end = out_buff + outbuffSize;
|
||||
out_start = out_buff + prefix64k;
|
||||
out_buff = (char*)malloc(outBuffSize);
|
||||
out_start = out_buff;
|
||||
out_end = out_start + outBuffSize;
|
||||
if (!in_buff || !out_buff) EXM_THROW(70, "Allocation error : not enough memory");
|
||||
if (streamChecksumFlag) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);
|
||||
}
|
||||
if (streamChecksumFlag) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);
|
||||
|
||||
// Main Loop
|
||||
while (1)
|
||||
@ -817,23 +839,17 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
|
||||
if (streamChecksumFlag) XXH32_update(streamChecksumState, in_buff, blockSize);
|
||||
if (!blockIndependenceFlag)
|
||||
{
|
||||
if (blockSize >= prefix64k)
|
||||
{
|
||||
memcpy(out_buff, in_buff + (blockSize - prefix64k), prefix64k); // Required for reference for next blocks
|
||||
out_start = out_buff + prefix64k;
|
||||
continue;
|
||||
}
|
||||
else
|
||||
{
|
||||
memcpy(out_start, in_buff, blockSize);
|
||||
decodedBytes = blockSize;
|
||||
}
|
||||
// handle dictionary for streaming
|
||||
memcpy(in_buff + blockSize - 64 KB, out_buff, 64 KB);
|
||||
LZ4_setDictDecode(&ctx, out_buff, 64 KB);
|
||||
out_start = out_buff + 64 KB;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Decode Block
|
||||
decodedBytes = decompressionFunction(in_buff, out_start, blockSize, maxBlockSize);
|
||||
if (out_start + maxBlockSize > out_end) out_start = out_buff;
|
||||
decodedBytes = decompressionFunction(&ctx, in_buff, out_start, blockSize, maxBlockSize);
|
||||
if (decodedBytes < 0) EXM_THROW(77, "Decoding Failed ! Corrupted input detected !");
|
||||
filesize += decodedBytes;
|
||||
if (streamChecksumFlag) XXH32_update(streamChecksumState, out_start, decodedBytes);
|
||||
@ -841,17 +857,9 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
|
||||
// Write Block
|
||||
sizeCheck = fwrite(out_start, 1, decodedBytes, foutput);
|
||||
if (sizeCheck != (size_t)decodedBytes) EXM_THROW(78, "Write error : cannot write decoded block\n");
|
||||
out_start += decodedBytes;
|
||||
}
|
||||
|
||||
if (!blockIndependenceFlag)
|
||||
{
|
||||
out_start += decodedBytes;
|
||||
if ((size_t)(out_end - out_start) < (size_t)maxBlockSize)
|
||||
{
|
||||
memcpy(out_buff, out_start - prefix64k, prefix64k);
|
||||
out_start = out_buff + prefix64k;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Stream Checksum
|
||||
@ -862,7 +870,7 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
|
||||
sizeCheck = fread(&readChecksum, 1, 4, finput);
|
||||
if (sizeCheck != 4) EXM_THROW(74, "Read error : cannot read stream checksum");
|
||||
readChecksum = LITTLE_ENDIAN_32(readChecksum); // Convert to little endian
|
||||
if (checksum != readChecksum) EXM_THROW(75, "Error : invalid stream checksum detected");
|
||||
if (checksum != readChecksum) EXM_THROW(79, "Error : invalid stream checksum detected");
|
||||
}
|
||||
|
||||
// Free
|
||||
|
||||
Loading…
Reference in New Issue
Block a user