2011-04-24 22:16:00 +08:00
|
|
|
/*
|
|
|
|
LZ4 - Fast LZ compression algorithm
|
2011-04-27 07:49:24 +08:00
|
|
|
Copyright (C) 2011, Yann Collet.
|
2011-05-21 15:18:49 +08:00
|
|
|
BSD License
|
2011-04-27 07:49:24 +08:00
|
|
|
|
|
|
|
Redistribution and use in source and binary forms, with or without
|
|
|
|
modification, are permitted provided that the following conditions are
|
|
|
|
met:
|
|
|
|
|
|
|
|
* Redistributions of source code must retain the above copyright
|
|
|
|
notice, this list of conditions and the following disclaimer.
|
|
|
|
* Redistributions in binary form must reproduce the above
|
|
|
|
copyright notice, this list of conditions and the following disclaimer
|
|
|
|
in the documentation and/or other materials provided with the
|
|
|
|
distribution.
|
|
|
|
|
|
|
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
|
|
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
|
|
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
|
|
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
|
|
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
|
|
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
|
|
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
|
|
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
|
|
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
|
|
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
|
|
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
2011-04-24 22:16:00 +08:00
|
|
|
*/
|
|
|
|
|
|
|
|
//**************************************
|
|
|
|
// Includes
|
|
|
|
//**************************************
|
|
|
|
#include <stdlib.h> // for malloc
|
|
|
|
#include <string.h> // for memset
|
|
|
|
#include "lz4.h"
|
|
|
|
|
|
|
|
|
2011-06-05 01:15:43 +08:00
|
|
|
//**************************************
|
2011-06-06 05:23:42 +08:00
|
|
|
// Performance parameter
|
2011-06-05 01:15:43 +08:00
|
|
|
//**************************************
|
|
|
|
// Lowering this value reduce memory usage
|
|
|
|
// It may also improve speed, especially if you reach L1 cache size (32KB for Intel, 64KB for AMD)
|
|
|
|
// Expanding memory usage typically improves compression ratio
|
2011-08-31 06:35:09 +08:00
|
|
|
// Memory usage formula for 32 bits systems : N->2^(N+2) Bytes (examples : 17 -> 512KB ; 12 -> 16KB)
|
2011-06-06 05:23:42 +08:00
|
|
|
#define HASH_LOG 12
|
2011-06-05 01:15:43 +08:00
|
|
|
|
|
|
|
|
2011-04-24 22:16:00 +08:00
|
|
|
//**************************************
|
|
|
|
// Basic Types
|
|
|
|
//**************************************
|
2011-09-03 20:43:13 +08:00
|
|
|
#if defined(_MSC_VER)
|
2011-05-26 06:25:57 +08:00
|
|
|
#define BYTE unsigned __int8
|
|
|
|
#define U16 unsigned __int16
|
|
|
|
#define U32 unsigned __int32
|
|
|
|
#define S32 __int32
|
|
|
|
#else
|
|
|
|
#include <stdint.h>
|
|
|
|
#define BYTE uint8_t
|
|
|
|
#define U16 uint16_t
|
|
|
|
#define U32 uint32_t
|
|
|
|
#define S32 int32_t
|
|
|
|
#endif
|
2011-04-24 22:16:00 +08:00
|
|
|
|
|
|
|
|
|
|
|
//**************************************
|
|
|
|
// Constants
|
|
|
|
//**************************************
|
|
|
|
#define MINMATCH 4
|
|
|
|
#define INCOMPRESSIBLE 128
|
|
|
|
|
|
|
|
#define MAXD_LOG 16
|
|
|
|
#define MAX_DISTANCE ((1 << MAXD_LOG) - 1)
|
|
|
|
|
|
|
|
#define HASHTABLESIZE (1 << HASH_LOG)
|
|
|
|
#define HASH_MASK (HASHTABLESIZE - 1)
|
|
|
|
|
|
|
|
#define ML_BITS 4
|
|
|
|
#define ML_MASK ((1U<<ML_BITS)-1)
|
|
|
|
#define RUN_BITS (8-ML_BITS)
|
|
|
|
#define RUN_MASK ((1U<<RUN_BITS)-1)
|
|
|
|
|
|
|
|
|
|
|
|
//**************************************
|
|
|
|
// Local structures
|
|
|
|
//**************************************
|
|
|
|
struct refTables
|
|
|
|
{
|
|
|
|
BYTE* hashTable[HASHTABLESIZE];
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
//**************************************
|
|
|
|
// Macros
|
|
|
|
//**************************************
|
2011-09-14 02:19:42 +08:00
|
|
|
#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-HASH_LOG))
|
|
|
|
#define HASH_VALUE(p) HASH_FUNCTION(*(U32*)(p))
|
2011-04-24 22:16:00 +08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
//****************************
|
|
|
|
// Compression CODE
|
|
|
|
//****************************
|
|
|
|
|
|
|
|
int LZ4_compressCtx(void** ctx,
|
|
|
|
char* source,
|
|
|
|
char* dest,
|
|
|
|
int isize)
|
|
|
|
{
|
|
|
|
struct refTables *srt = (struct refTables *) (*ctx);
|
|
|
|
BYTE** HashTable;
|
|
|
|
|
2011-09-13 20:01:11 +08:00
|
|
|
BYTE *ip = (BYTE*) source,
|
2011-06-08 01:57:25 +08:00
|
|
|
*anchor = ip,
|
2011-06-06 05:23:42 +08:00
|
|
|
*incompressible = anchor + INCOMPRESSIBLE,
|
2011-06-08 01:57:25 +08:00
|
|
|
*iend = ip + isize,
|
|
|
|
*ilimit = iend - MINMATCH;
|
2011-04-24 22:16:00 +08:00
|
|
|
|
2011-09-13 20:01:11 +08:00
|
|
|
BYTE *op = (BYTE*) dest,
|
2011-04-24 22:16:00 +08:00
|
|
|
*ref,
|
|
|
|
*orun, *l_end;
|
|
|
|
|
2011-06-08 01:57:25 +08:00
|
|
|
int len, length;
|
2011-04-24 22:16:00 +08:00
|
|
|
U32 step=1;
|
|
|
|
|
|
|
|
|
|
|
|
// Init
|
|
|
|
if (*ctx == NULL)
|
|
|
|
{
|
|
|
|
srt = (struct refTables *) malloc ( sizeof(struct refTables) );
|
|
|
|
*ctx = (void*) srt;
|
|
|
|
}
|
|
|
|
HashTable = srt->hashTable;
|
|
|
|
memset(HashTable, 0, sizeof(srt->hashTable));
|
|
|
|
|
|
|
|
// Main Loop
|
|
|
|
while (ip < ilimit)
|
|
|
|
{
|
2011-06-08 01:57:25 +08:00
|
|
|
ref = HashTable[HASH_VALUE(ip)];
|
|
|
|
HashTable[HASH_VALUE(ip)] = ip;
|
2011-04-24 22:16:00 +08:00
|
|
|
|
2011-06-06 05:23:42 +08:00
|
|
|
// Min Match
|
2011-06-08 01:57:25 +08:00
|
|
|
if ((ref < ip - MAX_DISTANCE) || (*(U32*)ref != *(U32*)ip))
|
2011-04-24 22:16:00 +08:00
|
|
|
{
|
2011-06-06 05:23:42 +08:00
|
|
|
if (ip>incompressible) { incompressible += INCOMPRESSIBLE << (step >> 1); step++; }
|
|
|
|
ip+=step;
|
2011-04-24 22:16:00 +08:00
|
|
|
continue;
|
2011-06-06 05:23:42 +08:00
|
|
|
}
|
|
|
|
step=1;
|
2011-04-24 22:16:00 +08:00
|
|
|
|
2011-06-06 05:23:42 +08:00
|
|
|
// Catch up
|
2011-09-03 20:43:13 +08:00
|
|
|
while ((ip>anchor) && (ref>(BYTE*)source) && (ip[-1]==ref[-1])) { ip--; ref--; }
|
2011-04-24 22:16:00 +08:00
|
|
|
|
|
|
|
// Encode Literal length
|
2011-06-06 05:23:42 +08:00
|
|
|
length = ip - anchor;
|
2011-06-05 01:15:43 +08:00
|
|
|
orun = op++;
|
2011-06-08 01:57:25 +08:00
|
|
|
if (length>=(int)RUN_MASK) { *orun=(RUN_MASK<<ML_BITS); len = length-RUN_MASK; for(; len > 254 ; len-=255) *op++ = 255; *op++ = (BYTE)len; }
|
2011-06-06 05:23:42 +08:00
|
|
|
else *orun = (length<<ML_BITS);
|
2011-04-24 22:16:00 +08:00
|
|
|
|
|
|
|
// Copy Literals
|
|
|
|
l_end = op + length;
|
2011-09-13 20:01:11 +08:00
|
|
|
do { *(U32*)op = *(U32*)anchor; op+=4; anchor+=4; } while (op<l_end) ;
|
2011-04-24 22:16:00 +08:00
|
|
|
op = l_end;
|
|
|
|
|
|
|
|
// Encode Offset
|
|
|
|
*(U16*)op = (ip-ref); op+=2;
|
|
|
|
|
|
|
|
// Start Counting
|
2011-06-08 01:57:25 +08:00
|
|
|
ip+=MINMATCH; ref+=MINMATCH; // MinMatch verified
|
2011-04-24 22:16:00 +08:00
|
|
|
anchor = ip;
|
2011-06-06 05:23:42 +08:00
|
|
|
while (ip<(iend-3))
|
|
|
|
{
|
2011-06-08 01:57:25 +08:00
|
|
|
if (*(U32*)ref == *(U32*)ip) { ip+=4; ref+=4; continue; }
|
2011-06-06 05:23:42 +08:00
|
|
|
if (*(U16*)ref == *(U16*)ip) { ip+=2; ref+=2; }
|
|
|
|
if (*ref == *ip) ip++;
|
|
|
|
goto _endCount;
|
|
|
|
}
|
|
|
|
if ((ip<(iend-1)) && (*(U16*)ref == *(U16*)ip)) { ip+=2; ref+=2; }
|
|
|
|
if ((ip<iend) && (*ref == *ip)) ip++;
|
|
|
|
_endCount:
|
2011-04-24 22:16:00 +08:00
|
|
|
len = (ip - anchor);
|
|
|
|
|
|
|
|
// Encode MatchLength
|
2011-06-08 01:57:25 +08:00
|
|
|
if (len>=(int)ML_MASK) { *orun+=ML_MASK; len-=ML_MASK; for(; len > 509 ; len-=510) { *op++ = 255; *op++ = 255; } if (len > 254) { len-=255; *op++ = 255; } *op++ = (BYTE)len; }
|
2011-04-24 22:16:00 +08:00
|
|
|
else *orun += len;
|
|
|
|
|
|
|
|
// Prepare next loop
|
|
|
|
anchor = ip;
|
2011-06-06 05:23:42 +08:00
|
|
|
incompressible = anchor + INCOMPRESSIBLE;
|
2011-04-24 22:16:00 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
// Encode Last Literals
|
|
|
|
len = length = iend - anchor;
|
2011-06-05 01:15:43 +08:00
|
|
|
if (length)
|
|
|
|
{
|
|
|
|
orun=op++;
|
2011-06-08 01:57:25 +08:00
|
|
|
if (len>=(int)RUN_MASK) { *orun=(RUN_MASK<<ML_BITS); len-=RUN_MASK; for(; len > 254 ; len-=255) *op++ = 255; *op++ = (BYTE) len; }
|
2011-06-05 01:15:43 +08:00
|
|
|
else *orun = (len<<ML_BITS);
|
|
|
|
for(;length>0;length-=4) { *(U32*)op = *(U32*)anchor; op+=4; anchor+=4; }
|
|
|
|
op += length; // correction
|
|
|
|
}
|
2011-04-24 22:16:00 +08:00
|
|
|
|
|
|
|
// End
|
2011-05-21 03:54:28 +08:00
|
|
|
return (int) (((char*)op)-dest);
|
2011-04-24 22:16:00 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2011-09-14 02:19:42 +08:00
|
|
|
int LZ4_compress(char* source,
|
|
|
|
char* dest,
|
|
|
|
int isize)
|
|
|
|
{
|
|
|
|
void* ctx = malloc(sizeof(struct refTables));
|
|
|
|
int result = LZ4_compressCtx(&ctx, source, dest, isize);
|
|
|
|
free(ctx);
|
|
|
|
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2011-04-24 22:16:00 +08:00
|
|
|
|
|
|
|
//****************************
|
|
|
|
// Decompression CODE
|
|
|
|
//****************************
|
2011-09-03 20:43:13 +08:00
|
|
|
|
|
|
|
// Note : The decoding functions LZ4_uncompress() and LZ4_uncompress_unknownOutputSize()
|
|
|
|
// are safe against "buffer overflow" attack type
|
|
|
|
// since they will *never* write outside of the provided output buffer :
|
|
|
|
// they both check this condition *before* writing anything.
|
|
|
|
// A corrupted packet however can make them *read* within the first 64K before the output buffer.
|
|
|
|
|
2011-06-05 01:15:43 +08:00
|
|
|
int LZ4_uncompress(char* source,
|
2011-04-24 22:16:00 +08:00
|
|
|
char* dest,
|
2011-06-05 01:15:43 +08:00
|
|
|
int osize)
|
2011-04-24 22:16:00 +08:00
|
|
|
{
|
|
|
|
// Local Variables
|
2011-06-05 01:15:43 +08:00
|
|
|
BYTE *ip = (BYTE*) source;
|
2011-04-24 22:16:00 +08:00
|
|
|
|
2011-05-21 03:54:28 +08:00
|
|
|
BYTE *op = (BYTE*) dest,
|
2011-06-08 01:57:25 +08:00
|
|
|
*olimit = op + osize - 4,
|
2011-04-24 22:16:00 +08:00
|
|
|
*ref, *cpy,
|
|
|
|
runcode;
|
|
|
|
|
|
|
|
U32 dec[4]={0, 3, 2, 3};
|
2011-06-08 01:57:25 +08:00
|
|
|
int length;
|
2011-04-24 22:16:00 +08:00
|
|
|
|
|
|
|
|
|
|
|
// Main Loop
|
2011-06-05 01:15:43 +08:00
|
|
|
while (1)
|
2011-04-24 22:16:00 +08:00
|
|
|
{
|
|
|
|
// get runlength
|
|
|
|
runcode = *ip++;
|
2011-06-08 01:57:25 +08:00
|
|
|
if ((length=(runcode>>ML_BITS)) == RUN_MASK) { for (;*ip==255;length+=255) {ip++;} length += *ip++; }
|
2011-04-24 22:16:00 +08:00
|
|
|
|
|
|
|
// copy literals
|
|
|
|
ref = op+length;
|
2011-06-08 01:57:25 +08:00
|
|
|
if (ref > olimit)
|
2011-06-05 01:15:43 +08:00
|
|
|
{
|
2011-06-08 01:57:25 +08:00
|
|
|
if (ref > olimit+4) goto _output_error;
|
|
|
|
while(op <= olimit) { *(U32*)op=*(U32*)ip; op+=4; ip+=4; }
|
|
|
|
while(op < ref) *op++=*ip++;
|
2011-06-05 01:15:43 +08:00
|
|
|
break; // Necessarily EOF
|
|
|
|
}
|
2011-06-08 01:57:25 +08:00
|
|
|
do { *(U32*)op = *(U32*)ip; op+=4; ip+=4; } while (op<ref) ;
|
2011-04-24 22:16:00 +08:00
|
|
|
ip-=(op-ref); op=ref; // correction
|
|
|
|
|
|
|
|
// get offset
|
|
|
|
ref -= *(U16*)ip; ip+=2;
|
|
|
|
|
|
|
|
// get matchlength
|
2011-06-08 01:57:25 +08:00
|
|
|
if ((length=(runcode&ML_MASK)) == ML_MASK) { for (;*ip==255;length+=255) {ip++;} length += *ip++; }
|
2011-04-24 22:16:00 +08:00
|
|
|
length += MINMATCH;
|
|
|
|
|
|
|
|
// copy repeated sequence
|
|
|
|
cpy = op + length;
|
|
|
|
if (op-ref<4)
|
|
|
|
{
|
|
|
|
*op++ = *ref++;
|
|
|
|
*op++ = *ref++;
|
|
|
|
*op++ = *ref++;
|
|
|
|
*op++ = *ref++;
|
|
|
|
ref -= dec[op-ref];
|
2011-06-08 01:57:25 +08:00
|
|
|
} else { *(U32*)op=*(U32*)ref; op+=4; ref+=4; }
|
|
|
|
if (cpy > olimit)
|
2011-06-05 01:15:43 +08:00
|
|
|
{
|
2011-06-08 01:57:25 +08:00
|
|
|
if (cpy > olimit+4) goto _output_error;
|
|
|
|
while(op < cpy-3) { *(U32*)op=*(U32*)ref; op+=4; ref+=4; }
|
|
|
|
while(op < cpy) *op++=*ref++;
|
|
|
|
if (op >= olimit+4) break; // Check EOF
|
2011-06-05 01:15:43 +08:00
|
|
|
continue;
|
|
|
|
}
|
2011-06-08 01:57:25 +08:00
|
|
|
do { *(U32*)op = *(U32*)ref; op+=4; ref+=4; } while (op<cpy) ;
|
2011-06-05 01:15:43 +08:00
|
|
|
op=cpy; // correction
|
|
|
|
}
|
|
|
|
|
|
|
|
// end of decoding
|
|
|
|
return (int) (((char*)ip)-source);
|
|
|
|
|
|
|
|
// write overflow error detected
|
|
|
|
_output_error:
|
|
|
|
return (int) (-(((char*)ip)-source));
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
int LZ4_uncompress_unknownOutputSize(
|
|
|
|
char* source,
|
|
|
|
char* dest,
|
|
|
|
int isize,
|
|
|
|
int maxOutputSize)
|
|
|
|
{
|
|
|
|
// Local Variables
|
|
|
|
BYTE *ip = (BYTE*) source,
|
|
|
|
*iend = ip + isize;
|
|
|
|
|
|
|
|
BYTE *op = (BYTE*) dest,
|
|
|
|
*oend = op + maxOutputSize,
|
|
|
|
*ref, *cpy,
|
|
|
|
runcode;
|
|
|
|
|
|
|
|
U32 dec[4]={0, 3, 2, 3};
|
|
|
|
int len, length;
|
|
|
|
|
|
|
|
|
|
|
|
// Main Loop
|
|
|
|
while (ip<iend)
|
|
|
|
{
|
|
|
|
// get runlength
|
|
|
|
runcode = *ip++;
|
|
|
|
if ((length=(runcode>>ML_BITS)) == RUN_MASK) { for (;(len=*ip++)==255;length+=255){} length += len; }
|
|
|
|
|
|
|
|
// copy literals
|
|
|
|
ref = op+length;
|
|
|
|
if (ref>oend-4)
|
|
|
|
{
|
|
|
|
if (ref > oend) goto _output_error;
|
|
|
|
while(op<oend-3) { *(U32*)op=*(U32*)ip; op+=4; ip+=4; }
|
|
|
|
while(op<ref) *op++=*ip++;
|
|
|
|
break; // Necessarily EOF
|
|
|
|
}
|
2011-06-08 01:57:25 +08:00
|
|
|
do { *(U32*)op = *(U32*)ip; op+=4; ip+=4; } while (op<ref) ;
|
2011-06-05 01:15:43 +08:00
|
|
|
ip-=(op-ref); op=ref; // correction
|
|
|
|
if (ip>=iend) break; // check EOF
|
|
|
|
|
|
|
|
// get offset
|
|
|
|
ref -= *(U16*)ip; ip+=2;
|
|
|
|
|
|
|
|
// get matchlength
|
|
|
|
if ((length=(runcode&ML_MASK)) == ML_MASK) { for (;(len=*ip++)==255;length+=255){} length += len; }
|
|
|
|
length += MINMATCH;
|
|
|
|
|
|
|
|
// copy repeated sequence
|
|
|
|
cpy = op + length;
|
|
|
|
if (op-ref<4)
|
|
|
|
{
|
|
|
|
*op++ = *ref++;
|
|
|
|
*op++ = *ref++;
|
|
|
|
*op++ = *ref++;
|
|
|
|
*op++ = *ref++;
|
|
|
|
ref -= dec[op-ref];
|
2011-06-08 01:57:25 +08:00
|
|
|
} else { *(U32*)op=*(U32*)ref; op+=4; ref+=4; }
|
2011-06-05 01:15:43 +08:00
|
|
|
if (cpy>oend-4)
|
|
|
|
{
|
|
|
|
if (cpy > oend) goto _output_error;
|
|
|
|
while(op<cpy-3) { *(U32*)op=*(U32*)ref; op+=4; ref+=4; }
|
|
|
|
while(op<cpy) *op++=*ref++;
|
|
|
|
if (op>=oend) break; // Check EOF
|
|
|
|
continue;
|
|
|
|
}
|
2011-06-08 01:57:25 +08:00
|
|
|
do { *(U32*)op = *(U32*)ref; op+=4; ref+=4; } while (op<cpy) ;
|
2011-04-24 22:16:00 +08:00
|
|
|
op=cpy; // correction
|
|
|
|
}
|
|
|
|
|
|
|
|
// end of decoding
|
2011-05-21 03:54:28 +08:00
|
|
|
return (int) (((char*)op)-dest);
|
2011-06-05 01:15:43 +08:00
|
|
|
|
|
|
|
// write overflow error detected
|
|
|
|
_output_error:
|
|
|
|
return (int) (-(((char*)ip)-source));
|
2011-04-24 22:16:00 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|