mirrors/lz4
0
0
Fork 0
mirror of https://github.com/lz4/lz4.git synced 2024-11-25 02:43:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed multiples files on lz4cat (#184, by @libor-m)

Browse Source
This commit is contained in:
Yann Collet 2016-11-09 16:37:16 -08:00
parent 1b16805941
commit fe48cef50e
3 changed files with 93 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
programs/lz4cli.c
View File

@ -311,34 +311,37 @@ int main(int argc, const char** argv)
if(!argument) continue; /* Protection if argument empty */ if(!argument) continue; /* Protection if argument empty */
/* long commands (--long-word) */
if (!strcmp(argument, "--compress")) { mode = om_compress; continue; }
if ((!strcmp(argument, "--decompress"))
|| (!strcmp(argument, "--uncompress"))) { mode = om_decompress; continue; }
if (!strcmp(argument, "--multiple")) { multiple_inputs = 1; if (inFileNames==NULL) inFileNames = (const char**)malloc(argc * sizeof(char*)); continue; }
if (!strcmp(argument, "--test")) { mode = om_test; continue; }
if (!strcmp(argument, "--force")) { LZ4IO_setOverwrite(1); continue; }
if (!strcmp(argument, "--no-force")) { LZ4IO_setOverwrite(0); continue; }
if ((!strcmp(argument, "--stdout"))
|| (!strcmp(argument, "--to-stdout"))) { forceStdout=1; output_filename=stdoutmark; displayLevel=1; continue; }
if (!strcmp(argument, "--frame-crc")) { LZ4IO_setStreamChecksumMode(1); continue; }
if (!strcmp(argument, "--no-frame-crc")) { LZ4IO_setStreamChecksumMode(0); continue; }
if (!strcmp(argument, "--content-size")) { LZ4IO_setContentSize(1); continue; }
if (!strcmp(argument, "--no-content-size")) { LZ4IO_setContentSize(0); continue; }
if (!strcmp(argument, "--sparse")) { LZ4IO_setSparseFile(2); continue; }
if (!strcmp(argument, "--no-sparse")) { LZ4IO_setSparseFile(0); continue; }
if (!strcmp(argument, "--verbose")) { displayLevel++; continue; }
if (!strcmp(argument, "--quiet")) { if (displayLevel) displayLevel--; continue; }
if (!strcmp(argument, "--version")) { DISPLAY(WELCOME_MESSAGE); return 0; }
if (!strcmp(argument, "--keep")) { LZ4IO_setRemoveSrcFile(0); continue; } /* keep source file (default anyway; just for xz/lzma compatibility) */
if (!strcmp(argument, "--rm")) { LZ4IO_setRemoveSrcFile(1); continue; }
/* Short commands (note : aggregated short commands are allowed) */ /* Short commands (note : aggregated short commands are allowed) */
if (argument[0]=='-') { if (argument[0]=='-') {
/* '-' means stdin/stdout */ /* '-' means stdin/stdout */
if (argument[1]==0) { if (argument[1]==0) {
if (!input_filename) input_filename=stdinmark; if (!input_filename) input_filename=stdinmark;
else output_filename=stdoutmark; else output_filename=stdoutmark;
continue;
}
/* long commands (--long-word) */
if (argument[1]=='-') {
if (!strcmp(argument, "--compress")) { mode = om_compress; continue; }
if ((!strcmp(argument, "--decompress"))
|| (!strcmp(argument, "--uncompress"))) { mode = om_decompress; continue; }
if (!strcmp(argument, "--multiple")) { multiple_inputs = 1; if (inFileNames==NULL) inFileNames = (const char**)malloc(argc * sizeof(char*)); continue; }
if (!strcmp(argument, "--test")) { mode = om_test; continue; }
if (!strcmp(argument, "--force")) { LZ4IO_setOverwrite(1); continue; }
if (!strcmp(argument, "--no-force")) { LZ4IO_setOverwrite(0); continue; }
if ((!strcmp(argument, "--stdout"))
|| (!strcmp(argument, "--to-stdout"))) { forceStdout=1; output_filename=stdoutmark; continue; }
if (!strcmp(argument, "--frame-crc")) { LZ4IO_setStreamChecksumMode(1); continue; }
if (!strcmp(argument, "--no-frame-crc")) { LZ4IO_setStreamChecksumMode(0); continue; }
if (!strcmp(argument, "--content-size")) { LZ4IO_setContentSize(1); continue; }
if (!strcmp(argument, "--no-content-size")) { LZ4IO_setContentSize(0); continue; }
if (!strcmp(argument, "--sparse")) { LZ4IO_setSparseFile(2); continue; }
if (!strcmp(argument, "--no-sparse")) { LZ4IO_setSparseFile(0); continue; }
if (!strcmp(argument, "--verbose")) { displayLevel++; continue; }
if (!strcmp(argument, "--quiet")) { if (displayLevel) displayLevel--; continue; }
if (!strcmp(argument, "--version")) { DISPLAY(WELCOME_MESSAGE); return 0; }
if (!strcmp(argument, "--keep")) { LZ4IO_setRemoveSrcFile(0); continue; } /* keep source file (default) */
if (!strcmp(argument, "--rm")) { LZ4IO_setRemoveSrcFile(1); continue; }
} }
while (argument[1]!=0) { while (argument[1]!=0) {
@ -382,7 +385,7 @@ int main(int argc, const char** argv)
case 'd': mode = om_decompress; break; case 'd': mode = om_decompress; break;
/* Force stdout, even if stdout==console */ /* Force stdout, even if stdout==console */
case 'c': forceStdout=1; output_filename=stdoutmark; displayLevel=1; break; case 'c': forceStdout=1; output_filename=stdoutmark; break;
/* Test integrity */ /* Test integrity */
case 't': mode = om_test; break; case 't': mode = om_test; break;
@ -409,7 +412,7 @@ int main(int argc, const char** argv)
case '5': case '5':
case '6': case '6':
case '7': case '7':
{ int B = argument[1] - '0'; { int const B = argument[1] - '0';
blockSize = LZ4IO_setBlockSizeID(B); blockSize = LZ4IO_setBlockSizeID(B);
BMK_setNotificationLevel(displayLevel); BMK_setNotificationLevel(displayLevel);
BMK_SetBlockSize(blockSize); BMK_SetBlockSize(blockSize);
@ -417,7 +420,7 @@ int main(int argc, const char** argv)
break; break;
} }
case 'D': LZ4IO_setBlockMode(LZ4IO_blockLinked); argument++; break; case 'D': LZ4IO_setBlockMode(LZ4IO_blockLinked); argument++; break;
case 'X': LZ4IO_setBlockChecksumMode(1); argument ++; break; /* currently disabled */ case 'X': LZ4IO_setBlockChecksumMode(1); argument ++; break; /* disabled by default */
default : exitBlockProperties=1; default : exitBlockProperties=1;
} }
if (exitBlockProperties) break; if (exitBlockProperties) break;
@ -487,7 +490,7 @@ int main(int argc, const char** argv)
/* No input filename ==> use stdin */ /* No input filename ==> use stdin */
if (multiple_inputs) { if (multiple_inputs) {
input_filename = inFileNames[0]; input_filename = inFileNames[0];
output_filename = (const char*)(inFileNames[0]); //output_filename = (const char*)(inFileNames[0]);
#ifdef UTIL_HAS_CREATEFILELIST #ifdef UTIL_HAS_CREATEFILELIST
if (recursive) { /* at this stage, filenameTable is a list of paths, which can contain both files and directories */ if (recursive) { /* at this stage, filenameTable is a list of paths, which can contain both files and directories */
extendedFileList = UTIL_createFileList(inFileNames, ifnIdx, &fileNamesBuf, &fileNamesNb); extendedFileList = UTIL_createFileList(inFileNames, ifnIdx, &fileNamesBuf, &fileNamesNb);
@ -518,8 +521,7 @@ int main(int argc, const char** argv)
/* compress or decompress */ /* compress or decompress */
if (!input_filename) { input_filename=stdinmark; } if (!input_filename) { input_filename=stdinmark; }
/* Check if input is defined as console; trigger an error in this case */ /* Check if input is defined as console; trigger an error in this case */
if (!strcmp(input_filename, stdinmark) if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) {
&& IS_CONSOLE(stdin) ) {
DISPLAYLEVEL(1, "refusing to read from a console\n"); DISPLAYLEVEL(1, "refusing to read from a console\n");
exit(1); exit(1);
} }
@ -565,15 +567,15 @@ int main(int argc, const char** argv)
exit(1); exit(1);
} }
/* Downgrade notification level in pure pipe mode (stdin + stdout) and multiple file mode */ /* Downgrade notification level in stdout and multiple file mode */
if (!strcmp(input_filename, stdinmark) && !strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1; if (!strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1;
if ((multiple_inputs) && (displayLevel==2)) displayLevel=1; if ((multiple_inputs) && (displayLevel==2)) displayLevel=1;
/* IO Stream/File */ /* IO Stream/File */
LZ4IO_setNotificationLevel(displayLevel); LZ4IO_setNotificationLevel(displayLevel);
if (mode == om_decompress) { if (mode == om_decompress) {
if (multiple_inputs) if (multiple_inputs)
operationResult = LZ4IO_decompressMultipleFilenames(inFileNames, ifnIdx, LZ4_EXTENSION); operationResult = LZ4IO_decompressMultipleFilenames(inFileNames, ifnIdx, !strcmp(output_filename,stdoutmark) ? stdoutmark : LZ4_EXTENSION);
else else
operationResult = DEFAULT_DECOMPRESSOR(input_filename, output_filename); operationResult = DEFAULT_DECOMPRESSOR(input_filename, output_filename);
} else { /* compression is default action */ } else { /* compression is default action */

85
programs/lz4io.c
View File

@ -247,18 +247,22 @@ static int LZ4IO_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ
static int LZ4IO_getFiles(const char* input_filename, const char* output_filename, FILE** pfinput, FILE** pfoutput) static int LZ4IO_getFiles(const char* input_filename, const char* output_filename, FILE** pfinput, FILE** pfoutput)
{ {
if (!strcmp (input_filename, stdinmark)) { if (pfinput!=NULL) { /* no need to open finput */
DISPLAYLEVEL(4,"Using stdin for input \n"); if (!strcmp (input_filename, stdinmark)) {
*pfinput = stdin; DISPLAYLEVEL(4,"Using stdin for input \n");
SET_BINARY_MODE(stdin); *pfinput = stdin;
} else { SET_BINARY_MODE(stdin);
*pfinput = fopen(input_filename, "rb"); } else {
*pfinput = fopen(input_filename, "rb");
}
if ( *pfinput==0 ) {
DISPLAYLEVEL(1, "Unable to access file for processing: %s \n", input_filename);
return 1;
}
} }
if ( *pfinput==0 ) { if (pfoutput==NULL) return 0; /* no need to open foutput */
DISPLAYLEVEL(1, "Unable to access file for processing: %s \n", input_filename);
return 1;
}
if (!strcmp (output_filename, stdoutmark)) { if (!strcmp (output_filename, stdoutmark)) {
DISPLAYLEVEL(4,"Using stdout for output \n"); DISPLAYLEVEL(4,"Using stdout for output \n");
@ -739,6 +743,7 @@ typedef struct {
size_t srcBufferSize; size_t srcBufferSize;
void* dstBuffer; void* dstBuffer;
size_t dstBufferSize; size_t dstBufferSize;
FILE* dstFile;
LZ4F_decompressionContext_t dCtx; LZ4F_decompressionContext_t dCtx;
} dRess_t; } dRess_t;
@ -880,7 +885,7 @@ static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutpu
g_magicRead = 0; g_magicRead = 0;
} else { } else {
size_t const nbReadBytes = fread(MNstore, 1, MAGICNUMBER_SIZE, finput); size_t const nbReadBytes = fread(MNstore, 1, MAGICNUMBER_SIZE, finput);
if (nbReadBytes==0) return ENDOFSTREAM; /* EOF */ if (nbReadBytes==0) { nbCalls = 0; return ENDOFSTREAM; } /* EOF */
if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(40, "Unrecognized header : Magic Number unreadable"); if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(40, "Unrecognized header : Magic Number unreadable");
magicNumber = LZ4IO_readLE32(MNstore); /* Little Endian format */ magicNumber = LZ4IO_readLE32(MNstore); /* Little Endian format */
} }
@ -904,8 +909,10 @@ static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutpu
EXTENDED_FORMAT; /* macro extension for custom formats */ EXTENDED_FORMAT; /* macro extension for custom formats */
default: default:
if (nbCalls == 1) { /* just started */ if (nbCalls == 1) { /* just started */
if (!g_testMode && g_overwrite) if (!g_testMode && g_overwrite) {
nbCalls = 0;
return LZ4IO_passThrough(finput, foutput, MNstore); return LZ4IO_passThrough(finput, foutput, MNstore);
}
EXM_THROW(44,"Unrecognized header : file cannot be decoded"); /* Wrong magic number at the beginning of 1st stream */ EXM_THROW(44,"Unrecognized header : file cannot be decoded"); /* Wrong magic number at the beginning of 1st stream */
} }
DISPLAYLEVEL(2, "Stream followed by undecodable data\n"); DISPLAYLEVEL(2, "Stream followed by undecodable data\n");
@ -914,19 +921,16 @@ static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutpu
} }
static int LZ4IO_decompressFile_extRess(dRess_t ress, const char* input_filename, const char* output_filename) static int LZ4IO_decompressSrcFile(dRess_t ress, const char* input_filename, const char* output_filename)
{ {
FILE* const foutput = ress.dstFile;
unsigned long long filesize = 0, decodedSize=0; unsigned long long filesize = 0, decodedSize=0;
FILE* finput; FILE* finput;
FILE* foutput;
/* Init */ /* Init */
if (LZ4IO_getFiles(input_filename, output_filename, &finput, &foutput)) if (LZ4IO_getFiles(input_filename, output_filename, &finput, NULL))
return 1; return 1;
/* sparse file */
if (g_sparseFileSupport) { SET_SPARSE_FILE_MODE(foutput); }
/* Loop over multiple streams */ /* Loop over multiple streams */
do { do {
decodedSize = selectDecoder(ress, finput, foutput); decodedSize = selectDecoder(ress, finput, foutput);
@ -936,6 +940,31 @@ static int LZ4IO_decompressFile_extRess(dRess_t ress, const char* input_filename
/* Close */ /* Close */
fclose(finput); fclose(finput);
if (g_removeSrcFile) { if (remove(input_filename)) EXM_THROW(45, "Remove error : %s: %s", input_filename, strerror(errno)); } /* remove source file : --rm */
/* Final Status */
DISPLAYLEVEL(2, "\r%79s\r", "");
DISPLAYLEVEL(2, "%-20.20s : decoded %llu bytes \n", input_filename, filesize);
return 0;
}
static int LZ4IO_decompressDstFile(dRess_t ress, const char* input_filename, const char* output_filename)
{
FILE* foutput;
/* Init */
if (LZ4IO_getFiles(input_filename, output_filename, NULL, &foutput))
return 1;
/* sparse file */
if (g_sparseFileSupport) { SET_SPARSE_FILE_MODE(foutput); }
ress.dstFile = foutput;
LZ4IO_decompressSrcFile(ress, input_filename, output_filename);
fclose(foutput); fclose(foutput);
/* Copy owner, file permissions and modification time */ /* Copy owner, file permissions and modification time */
@ -944,12 +973,6 @@ static int LZ4IO_decompressFile_extRess(dRess_t ress, const char* input_filename
UTIL_setFileStat(output_filename, &statbuf); UTIL_setFileStat(output_filename, &statbuf);
} }
if (g_removeSrcFile) { if (remove(input_filename)) EXM_THROW(45, "Remove error : %s: %s", input_filename, strerror(errno)); } /* remove source file : --rm */
/* Final Status */
DISPLAYLEVEL(2, "\r%79s\r", "");
DISPLAYLEVEL(2, "Successfully decoded %llu bytes \n", filesize);
return 0; return 0;
} }
@ -959,7 +982,7 @@ int LZ4IO_decompressFilename(const char* input_filename, const char* output_file
dRess_t const ress = LZ4IO_createDResources(); dRess_t const ress = LZ4IO_createDResources();
clock_t const start = clock(); clock_t const start = clock();
int const missingFiles = LZ4IO_decompressFile_extRess(ress, input_filename, output_filename); int const missingFiles = LZ4IO_decompressDstFile(ress, input_filename, output_filename);
{ clock_t const end = clock(); { clock_t const end = clock();
double const seconds = (double)(end - start) / CLOCKS_PER_SEC; double const seconds = (double)(end - start) / CLOCKS_PER_SEC;
@ -971,7 +994,6 @@ int LZ4IO_decompressFilename(const char* input_filename, const char* output_file
} }
#define MAXSUFFIXSIZE 8
int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix) int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix)
{ {
int i; int i;
@ -980,13 +1002,19 @@ int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSiz
char* outFileName = (char*)malloc(FNSPACE); char* outFileName = (char*)malloc(FNSPACE);
size_t ofnSize = FNSPACE; size_t ofnSize = FNSPACE;
size_t const suffixSize = strlen(suffix); size_t const suffixSize = strlen(suffix);
dRess_t const ress = LZ4IO_createDResources(); dRess_t ress = LZ4IO_createDResources();
if (outFileName==NULL) exit(1); /* not enough memory */ if (outFileName==NULL) exit(1); /* not enough memory */
ress.dstFile = stdout;
SET_BINARY_MODE(stdout);
for (i=0; i<ifntSize; i++) { for (i=0; i<ifntSize; i++) {
size_t const ifnSize = strlen(inFileNamesTable[i]); size_t const ifnSize = strlen(inFileNamesTable[i]);
const char* const suffixPtr = inFileNamesTable[i] + ifnSize - suffixSize; const char* const suffixPtr = inFileNamesTable[i] + ifnSize - suffixSize;
if (!strcmp(suffix, stdoutmark)) {
missingFiles += LZ4IO_decompressSrcFile(ress, inFileNamesTable[i], stdoutmark);
continue;
}
if (ofnSize <= ifnSize-suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); if (outFileName==NULL) exit(1); } if (ofnSize <= ifnSize-suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); if (outFileName==NULL) exit(1); }
if (ifnSize <= suffixSize || strcmp(suffixPtr, suffix) != 0) { if (ifnSize <= suffixSize || strcmp(suffixPtr, suffix) != 0) {
DISPLAYLEVEL(1, "File extension doesn't match expected LZ4_EXTENSION (%4s); will not process file: %s\n", suffix, inFileNamesTable[i]); DISPLAYLEVEL(1, "File extension doesn't match expected LZ4_EXTENSION (%4s); will not process file: %s\n", suffix, inFileNamesTable[i]);
@ -995,8 +1023,7 @@ int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSiz
} }
memcpy(outFileName, inFileNamesTable[i], ifnSize - suffixSize); memcpy(outFileName, inFileNamesTable[i], ifnSize - suffixSize);
outFileName[ifnSize-suffixSize] = '\0'; outFileName[ifnSize-suffixSize] = '\0';
missingFiles += LZ4IO_decompressDstFile(ress, inFileNamesTable[i], outFileName);
missingFiles += LZ4IO_decompressFile_extRess(ress, inFileNamesTable[i], outFileName);
} }
LZ4IO_freeDResources(ress); LZ4IO_freeDResources(ress);

8
tests/Makefile
View File

@ -225,8 +225,8 @@ test-lz4-basic: lz4 datagen
test-lz4-hugefile: lz4 datagen test-lz4-hugefile: lz4 datagen
@echo "\n ---- test huge files compression/decompression ----" @echo "\n ---- test huge files compression/decompression ----"
./datagen -g6GB | $(PRGDIR)/lz4 -vqB5D | $(PRGDIR)/lz4 -qt ./datagen -g6GB | $(PRGDIR)/lz4 -vB5D | $(PRGDIR)/lz4 -qt
./datagen -g6GB | $(PRGDIR)/lz4 -vq5BD | $(PRGDIR)/lz4 -qt ./datagen -g6GB | $(PRGDIR)/lz4 -v5BD | $(PRGDIR)/lz4 -qt
@$(RM) tmp* @$(RM) tmp*
test-lz4-testmode: lz4 datagen test-lz4-testmode: lz4 datagen
@ -240,8 +240,8 @@ test-lz4-testmode: lz4 datagen
./datagen | $(PRGDIR)/lz4 -df > $(VOID) ./datagen | $(PRGDIR)/lz4 -df > $(VOID)
@echo "Hello World !" > tmp1 @echo "Hello World !" > tmp1
$(PRGDIR)/lz4 -dcf tmp1 $(PRGDIR)/lz4 -dcf tmp1
# @echo "from underground..." > tmp2 @echo "from underground..." > tmp2
# $(PRGDIR)/lz4 -dcfm tmp1 tmp2 $(PRGDIR)/lz4 -dcfm tmp1 tmp2
@echo "\n ---- test cli ----" @echo "\n ---- test cli ----"
$(PRGDIR)/lz4 file-does-not-exist && false || true $(PRGDIR)/lz4 file-does-not-exist && false || true
$(PRGDIR)/lz4 -f file-does-not-exist && false || true $(PRGDIR)/lz4 -f file-does-not-exist && false || true