mirrors/lz4
0
0
Fork 0
mirror of https://github.com/lz4/lz4.git synced 2024-11-24 18:33:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

Continuous streaming mode (automatic)

Browse Source
This commit is contained in:
Yann Collet 2014-05-21 19:46:36 +01:00
parent 914f8a9a2c
commit 4b43c2bd98
3 changed files with 63 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -92,7 +92,7 @@ lz4programs: lz4.c lz4hc.c
liblz4: lz4.c lz4hc.c
@echo compiling static library
@$(CC) $(CFLAGS) -c $^
@ar rcs liblz4.a lz4.o lz4hc.o
@$(AR) rcs liblz4.a lz4.o lz4hc.o
@echo compiling dynamic library
@$(CC) $(CFLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER)
@echo creating versioned links

74
lz4.c
View File

@ -418,7 +418,7 @@ static unsigned LZ4_count(const BYTE* pIn, const BYTE* pRef, const BYTE* pInLimi
}
int LZ4_compress_generic(
static int LZ4_compress_generic(
void* ctx,
const char* source,
char* dest,
@ -638,10 +638,9 @@ int LZ4_compress_limitedOutput_withState (void* state, const char* source, char*
Experimental : Streaming functions
*****************************************/
void LZ4_renormDictT(LZ4_dict_t_internal* LZ4_dict, const char* source)
void LZ4_renormDictT(LZ4_dict_t_internal* LZ4_dict)
{
if ((source - LZ4_dict->currentOffset > source)
|| (LZ4_dict->currentOffset > 0x80000000))
if (LZ4_dict->currentOffset > 0x80000000)
{
/* rescale hash table */
U32 delta = LZ4_dict->currentOffset - 64 KB;
@ -656,24 +655,20 @@ void LZ4_renormDictT(LZ4_dict_t_internal* LZ4_dict, const char* source)
}
static int LZ4_compress_usingDict_generic (LZ4_dict_t* LZ4_dict,
static inline int LZ4_compress_2_generic (LZ4_dict_t* LZ4_dict,
const char* source, char* dest, int inputSize, int maxOutputSize,
limitedOutput_directive outputLimited)
const limitedOutput_directive outputLimited,
const dict_directive dict)
{
LZ4_dict_t_internal* const streamPtr = (LZ4_dict_t_internal*)LZ4_dict;
const tableType_t tableType = byU32;
U32 currentOffset;
const U32 currentOffset = streamPtr->currentOffset;
const U32 dictSize = streamPtr->dictSize;
const BYTE* const dictionary = streamPtr->dictionary;
if (streamPtr->initCheck) return 0; /* structure not initialized */
LZ4_renormDictT(streamPtr, source);
currentOffset = streamPtr->currentOffset;
streamPtr->dictionary = (const BYTE*)source;
streamPtr->dictSize = (U32)inputSize;
streamPtr->currentOffset += inputSize;
streamPtr->currentOffset += (U32)inputSize;
{
U32 ipIndex = currentOffset;
@ -721,7 +716,7 @@ static int LZ4_compress_usingDict_generic (LZ4_dict_t* LZ4_dict,
if (unlikely(ipIndex > indexLimit)) goto _last_literals;
ip = base + ipIndex;
if (refIndex < currentOffset)
if ((dict==usingDict) && (refIndex < currentOffset))
{
ref = dictBase + refIndex;
lowLimit = dictionary;
@ -729,13 +724,13 @@ static int LZ4_compress_usingDict_generic (LZ4_dict_t* LZ4_dict,
else
{
ref = base + refIndex;
lowLimit = (const BYTE*)source;
if (dict==usingDict) lowLimit = (const BYTE*)source;
}
} while ((refIndex + MAX_DISTANCE < ipIndex) || (A32(ref) != A32(ip)));
/* Catch up */
while ((ip>anchor) && (ref>lowLimit) && (unlikely(ip[-1]==ref[-1]))) { ip--; ref--; }
while ((ip>anchor) && ((dict==usingDict)?(ref>lowLimit):1) && (unlikely(ip[-1]==ref[-1]))) { ip--; ref--; }
{
/* Encode Literal length */
@ -762,9 +757,7 @@ static int LZ4_compress_usingDict_generic (LZ4_dict_t* LZ4_dict,
/* Encode MatchLength */
{
unsigned matchLength;
if (refIndex >= currentOffset)
matchLength = LZ4_count(ip+MINMATCH, ref+MINMATCH, matchlimit);
else
if ((dict==usingDict) && (refIndex < currentOffset))
{
const BYTE* dicLimit = ip + (dictEnd - ref);
if (dicLimit > matchlimit) dicLimit = matchlimit;
@ -772,6 +765,10 @@ static int LZ4_compress_usingDict_generic (LZ4_dict_t* LZ4_dict,
if (ref + MINMATCH + matchLength == dictEnd)
matchLength += LZ4_count(ip+MINMATCH+matchLength, (const BYTE*)source, matchlimit);
}
else
{
matchLength = LZ4_count(ip+MINMATCH, ref+MINMATCH, matchlimit);
}
ip += matchLength + MINMATCH;
if (matchLength>=ML_MASK)
{
@ -811,6 +808,22 @@ static int LZ4_compress_usingDict_generic (LZ4_dict_t* LZ4_dict,
}
static inline int LZ4_compress_usingDict_generic (LZ4_dict_t* LZ4_dict,
const char* source, char* dest, int inputSize, int maxOutputSize,
const limitedOutput_directive outputLimited)
{
LZ4_dict_t_internal* const streamPtr = (LZ4_dict_t_internal*)LZ4_dict;
if (streamPtr->initCheck) return 0; /* structure not initialized */
LZ4_renormDictT(streamPtr);
if ((streamPtr->dictionary + streamPtr->dictSize == (const BYTE*)source) && (streamPtr->dictSize >= 64 KB))
return LZ4_compress_2_generic(LZ4_dict, source, dest, inputSize, maxOutputSize, outputLimited, withPrefix64k);
else
return LZ4_compress_2_generic(LZ4_dict, source, dest, inputSize, maxOutputSize, outputLimited, usingDict);
}
int LZ4_compress_usingDict (LZ4_dict_t* LZ4_dict, const char* source, char* dest, int inputSize)
{
return LZ4_compress_usingDict_generic(LZ4_dict, source, dest, inputSize, 0, notLimited);
@ -821,6 +834,12 @@ int LZ4_compress_limitedOutput_usingDict (LZ4_dict_t* LZ4_dict, const char* sour
return LZ4_compress_usingDict_generic(LZ4_dict, source, dest, inputSize, maxOutputSize, limitedOutput);
}
// Debug function only, to measure performance differences
int LZ4_compress_forceDict (LZ4_dict_t* LZ4_dict, const char* source, char* dest, int inputSize)
{
return LZ4_compress_2_generic(LZ4_dict, source, dest, inputSize, 0, notLimited, usingDict);
}
int LZ4_setDictPos (LZ4_dict_t* LZ4_dict, const char* dictionary, int dictSize)
{
@ -833,7 +852,22 @@ int LZ4_setDictPos (LZ4_dict_t* LZ4_dict, const char* dictionary, int dictSize)
}
int LZ4_loadDict (LZ4_dict_t* LZ4_dict, const char* dictionary, int dictSize)
int LZ4_moveDict (LZ4_dict_t* LZ4_dict, char* safeBuffer, int dictSize)
{
LZ4_dict_t_internal* dict = (LZ4_dict_t_internal*) LZ4_dict;
const BYTE* previousDictEnd = dict->dictionary + dict->dictSize;
if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */
if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize;
memcpy(safeBuffer, previousDictEnd - dictSize, dictSize);
dict->dictSize = (U32)dictSize;
return 1;
}
int LZ4_loadDict (LZ4_dict_t* LZ4_dict, const char* dictionary, int dictSize)
{
LZ4_dict_t_internal* dict = (LZ4_dict_t_internal*) LZ4_dict;
const BYTE* p = (const BYTE*)dictionary;

9
programs/fullbench.c
View File

@ -300,6 +300,12 @@ static int local_LZ4_compress_limitedOutput_usingDict(const char* in, char* out,
return LZ4_compress_limitedOutput_usingDict(&LZ4_dict, in, out, inSize, LZ4_compressBound(inSize));
}
int LZ4_compress_forceDict (LZ4_dict_t* LZ4_dict, const char* source, char* dest, int inputSize);
static int local_LZ4_compress_forceDict(const char* in, char* out, int inSize)
{
return LZ4_compress_forceDict(&LZ4_dict, in, out, inSize);
}
static void* stateLZ4HC;
static int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize)
@ -364,7 +370,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
{
int fileIdx=0;
char* orig_buff;
# define NB_COMPRESSION_ALGORITHMS 14
# define NB_COMPRESSION_ALGORITHMS 15
# define MINCOMPRESSIONCHAR '0'
# define MAXCOMPRESSIONCHAR (MINCOMPRESSIONCHAR + NB_COMPRESSION_ALGORITHMS)
double totalCTime[NB_COMPRESSION_ALGORITHMS+1] = {0};
@ -504,6 +510,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
case 12: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_limitedOutput_continue"; break;
case 13: compressionFunction = local_LZ4_compress_usingDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_usingDict"; break;
case 14: compressionFunction = local_LZ4_compress_limitedOutput_usingDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_limitedOutput_usingDict"; break;
case 15: compressionFunction = local_LZ4_compress_forceDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_forceDict"; break;
default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1;
}