Now constructs final destination path without allocating new table

This commit is contained in:
Sen Huang 2019-10-03 13:53:04 -04:00
parent 4dc604cab8
commit 64bc441d7d
5 changed files with 157 additions and 176 deletions

View File

@ -628,6 +628,104 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
return (size_t)fileSize;
}
/* FIO_checkFilenameCollisions() :
* Checks for and warns if there are any files that would have the same output path
*/
int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
const char** filenameTableSorted;
const char* c, *prevElem;
char* filename;
unsigned u;
#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
c = "\\";
#else
c = "/";
#endif
filenameTableSorted = (const char**) malloc(sizeof(char*) * nbFiles);
if (!filenameTableSorted) {
DISPLAY("Unable to malloc new str array, not checking for name collisions\n");
return 1;
}
for (u = 0; u < nbFiles; ++u) {
filename = strrchr(filenameTable[u], c[0]);
if (filename == NULL) {
filenameTableSorted[u] = filenameTable[u];
} else {
filenameTableSorted[u] = filename+1;
}
}
qsort(filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr);
prevElem = filenameTableSorted[0];
for (u = 1; u < nbFiles; ++u) {
if (strcmp(prevElem, filenameTableSorted[u]) == 0) {
DISPLAY("WARNING: Two files have same filename: %s\n", prevElem);
}
prevElem = filenameTableSorted[u];
}
free(filenameTableSorted);
return 0;
}
/* FIO_determineDstFilenameOutdir() :
* Takes a source file name and specified output directory, and
* allocates memory for and returns a pointer to final path.
* This function never returns an error (it may abort() in case of pb)
*/
static char*
FIO_determineDstFilenameOutdir(const char* srcFilename, const char* outDirName, const size_t suffixLen)
{
const char* c, *filenameBegin;
char* filename, *result;
size_t finalPathLen;
#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
c = "\\";
#else
c = "/";
#endif
finalPathLen = strlen(outDirName);
filenameBegin = strrchr(srcFilename, c[0]);
if (filenameBegin == NULL) {
filename = (char*) malloc((strlen(srcFilename)+1) * sizeof(char));
if (!filename) {
EXM_THROW(30, "zstd: %s", strerror(errno));
}
strcpy(filename, srcFilename);
} else {
filename = (char*) malloc((strlen(filenameBegin+1)+1) * sizeof(char));
if (!filename) {
EXM_THROW(30, "zstd: %s", strerror(errno));
}
strcpy(filename, filenameBegin+1);
}
finalPathLen += strlen(filename);
result = (char*) malloc((finalPathLen+suffixLen+30) * sizeof(char));
if (!result) {
free(filename);
EXM_THROW(30, "zstd: %s", strerror(errno));
}
strcpy(result, outDirName);
if (outDirName[strlen(outDirName)-1] == c[0]) {
strcat(result, filename);
} else {
strcat(result, c);
strcat(result, filename);
}
free(filename);
return result;
}
#ifndef ZSTD_NOCOMPRESS
/* **********************************************************************
@ -1379,19 +1477,25 @@ int FIO_compressFilename(FIO_prefs_t* const prefs, const char* dstFileName,
return result;
}
/* FIO_determineCompressedName() :
* create a destination filename for compressed srcFileName.
* @return a pointer to it.
* This function never returns an error (it may abort() in case of pb)
*/
static const char*
FIO_determineCompressedName(const char* srcFileName, const char* suffix)
FIO_determineCompressedName(const char* srcFileName, const char* outDirName, const char* suffix)
{
static size_t dfnbCapacity = 0;
static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */
size_t const sfnSize = strlen(srcFileName);
char* outDirFilename = NULL;
size_t sfnSize = strlen(srcFileName);
size_t const suffixSize = strlen(suffix);
if (outDirName) {
outDirFilename = FIO_determineDstFilenameOutdir(srcFileName, outDirName, suffixSize);
sfnSize = strlen(outDirFilename);
assert(outDirFilename != NULL);
}
if (dfnbCapacity <= sfnSize+suffixSize+1) {
/* resize buffer for dstName */
free(dstFileNameBuffer);
@ -1399,9 +1503,16 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix)
dstFileNameBuffer = (char*)malloc(dfnbCapacity);
if (!dstFileNameBuffer) {
EXM_THROW(30, "zstd: %s", strerror(errno));
} }
}
}
assert(dstFileNameBuffer != NULL);
memcpy(dstFileNameBuffer, srcFileName, sfnSize);
if (outDirFilename) {
memcpy(dstFileNameBuffer, outDirFilename, sfnSize);
free(outDirFilename);
} else {
memcpy(dstFileNameBuffer, srcFileName, sfnSize);
}
memcpy(dstFileNameBuffer+sfnSize, suffix, suffixSize+1 /* Include terminating null */);
return dstFileNameBuffer;
}
@ -1414,24 +1525,17 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix)
* or into a destination folder (specified with -O)
*/
int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileNamesTable,
const char* outDirName, char** dstFileNamesTable,
unsigned nbFiles, const char* outFileName,
const char* suffix, const char* dictFileName,
int compressionLevel, ZSTD_compressionParameters comprParams)
const char* outDirName, unsigned nbFiles,
const char* outFileName, const char* suffix,
const char* dictFileName, int compressionLevel,
ZSTD_compressionParameters comprParams)
{
int error = 0;
cRess_t ress = FIO_createCResources(prefs, dictFileName, compressionLevel, comprParams);
/* init */
assert(outFileName != NULL || suffix != NULL);
if (outDirName != NULL) { /* output into a particular folder */
unsigned u;
for (u = 0; u < nbFiles; ++u) {
const char* const srcFileName = inFileNamesTable[u];
const char* const dstFileName = FIO_determineCompressedName(dstFileNamesTable[u], suffix);
error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
}
} else if (outFileName != NULL) { /* output into a single destination (stdout typically) */
if (outFileName != NULL) { /* output into a single destination (stdout typically) */
ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName);
if (ress.dstFile == NULL) { /* could not open outFileName */
error = 1;
@ -1448,9 +1552,11 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileN
unsigned u;
for (u=0; u<nbFiles; u++) {
const char* const srcFileName = inFileNamesTable[u];
const char* const dstFileName = FIO_determineCompressedName(srcFileName, suffix); /* cannot fail */
const char* const dstFileName = FIO_determineCompressedName(srcFileName, outDirName, suffix); /* cannot fail */
error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
} }
}
FIO_checkFilenameCollisions(inFileNamesTable ,nbFiles);
}
FIO_freeCResources(ress);
return error;
@ -2169,13 +2275,14 @@ int FIO_decompressFilename(FIO_prefs_t* const prefs,
* @return a pointer to it.
* @return == NULL if there is an error */
static const char*
FIO_determineDstName(const char* srcFileName)
FIO_determineDstName(const char* srcFileName, const char* outDirName)
{
static size_t dfnbCapacity = 0;
static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */
size_t const sfnSize = strlen(srcFileName);
char* dstFilenameOutDir = NULL;
size_t sfnSize = strlen(srcFileName);
size_t suffixSize;
const char* const suffixPtr = strrchr(srcFileName, '.');
if (suffixPtr == NULL) {
DISPLAYLEVEL(1, "zstd: %s: unknown suffix -- ignored \n",
@ -2213,19 +2320,29 @@ FIO_determineDstName(const char* srcFileName)
srcFileName, suffixlist);
return NULL;
}
if (outDirName) {
dstFilenameOutDir = FIO_determineDstFilenameOutdir(srcFileName, outDirName, 0);
sfnSize = strlen(dstFilenameOutDir);
assert(dstFilenameOutDir != NULL);
}
/* allocate enough space to write dstFilename into it */
if (dfnbCapacity+suffixSize <= sfnSize+1) {
/* allocate enough space to write dstFilename into it */
free(dstFileNameBuffer);
dfnbCapacity = sfnSize + 20;
dstFileNameBuffer = (char*)malloc(dfnbCapacity);
if (dstFileNameBuffer==NULL)
EXM_THROW(74, "%s : not enough memory for dstFileName", strerror(errno));
EXM_THROW(74, "%s : not enough memory for dstFileName", strerror(errno));
}
/* return dst name == src name truncated from suffix */
assert(dstFileNameBuffer != NULL);
memcpy(dstFileNameBuffer, srcFileName, sfnSize - suffixSize);
if (dstFilenameOutDir) {
memcpy(dstFileNameBuffer, dstFilenameOutDir, sfnSize - suffixSize);
free(dstFilenameOutDir);
} else {
memcpy(dstFileNameBuffer, srcFileName, sfnSize - suffixSize);
}
dstFileNameBuffer[sfnSize-suffixSize] = '\0';
return dstFileNameBuffer;
@ -2236,23 +2353,13 @@ FIO_determineDstName(const char* srcFileName)
int
FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
const char** srcNamesTable, unsigned nbFiles,
const char* outDirName, char** dstFileNamesTable,
const char* outFileName,
const char* outDirName, const char* outFileName,
const char* dictFileName)
{
int error = 0;
dRess_t ress = FIO_createDResources(prefs, dictFileName);
if (outDirName != NULL) { /* output into a particular folder */
unsigned u;
for (u = 0; u < nbFiles; ++u) {
const char* const srcFileName = srcNamesTable[u];
const char* const dstFileName = FIO_determineDstName(dstFileNamesTable[u]);
if (dstFileName == NULL) { error=1; continue; }
error |= FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName);
}
} else if (outFileName) {
if (outFileName) {
unsigned u;
ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName);
if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", outFileName);
@ -2265,11 +2372,12 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
unsigned u;
for (u=0; u<nbFiles; u++) { /* create dstFileName */
const char* const srcFileName = srcNamesTable[u];
const char* const dstFileName = FIO_determineDstName(srcFileName);
const char* const dstFileName = FIO_determineDstName(srcFileName, outDirName);
if (dstFileName == NULL) { error=1; continue; }
error |= FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName);
}
FIO_checkFilenameCollisions(srcNamesTable ,nbFiles);
}
FIO_freeDResources(ress);

View File

@ -105,19 +105,23 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
/** FIO_compressMultipleFilenames() :
@return : nb of missing files */
int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileNamesTable,
const char* outDirName, char** dstFileNamesTable,
unsigned nbFiles, const char* outFileName,
const char* suffix, const char* dictFileName,
int compressionLevel, ZSTD_compressionParameters comprParams);
const char* outDirName, unsigned nbFiles,
const char* outFileName, const char* suffix,
const char* dictFileName, int compressionLevel,
ZSTD_compressionParameters comprParams);
/** FIO_decompressMultipleFilenames() :
@return : nb of missing or skipped files */
int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
const char** srcNamesTable, unsigned nbFiles,
const char* outDirName, char** dstFilenameTable,
const char* outDirName,
const char* outFileName,
const char* dictFileName);
/* FIO_checkFilenameCollisions() :
* Checks for and warns if thereå are any files that would have the same output path
*/
int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles);
/*-*************************************
* Advanced stuff (should actually be hosted elsewhere)

View File

@ -94,99 +94,6 @@ int UTIL_compareStr(const void *p1, const void *p2) {
return strcmp(* (char * const *) p1, * (char * const *) p2);
}
int UTIL_checkFilenameCollisions(char** dstFilenameTable, unsigned nbFiles) {
char** dstFilenameTableSorted;
char* prevElem;
unsigned u;
dstFilenameTableSorted = (char**) malloc(sizeof(char*) * nbFiles);
if (!dstFilenameTableSorted) {
UTIL_DISPLAYLEVEL(1, "Unable to malloc new str array, not checking for name collisions\n");
return 1;
}
for (u = 0; u < nbFiles; ++u) {
dstFilenameTableSorted[u] = dstFilenameTable[u];
}
qsort(dstFilenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr);
prevElem = dstFilenameTableSorted[0];
for (u = 1; u < nbFiles; ++u) {
if (strcmp(prevElem, dstFilenameTableSorted[u]) == 0) {
UTIL_DISPLAYLEVEL(1, "WARNING: Two files have same filename as source : %s\n", prevElem);
}
prevElem = dstFilenameTableSorted[u];
}
free(dstFilenameTableSorted);
return 0;
}
void UTIL_createDestinationDirTable(char** dstFilenameTable, const char** filenameTable,
const unsigned nbFiles, const char* outDirName)
{
unsigned u;
const char* c;
#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
c = "\\";
#else
c = "/";
#endif
for (u = 0; u < nbFiles; ++u) {
char* filename;
const char* filenameBegin;
size_t finalPathLen;
finalPathLen = strlen(outDirName);
filenameBegin = strrchr(filenameTable[u], c[0]);
if (filenameBegin == NULL) {
filename = (char*) malloc((strlen(filenameTable[u])+1) * sizeof(char));
if (!filename) {
UTIL_DISPLAYLEVEL(1, "Unable to allocate space for filename str\n");
continue;
}
strcpy(filename, filenameTable[u]);
} else {
filename = (char*) malloc((strlen(filenameBegin+1)+1) * sizeof(char));
if (!filename) {
UTIL_DISPLAYLEVEL(1, "Unable to allocate space for filename str\n");
continue;
}
strcpy(filename, filenameBegin+1);
}
finalPathLen += strlen(filename);
dstFilenameTable[u] = (char*) malloc((finalPathLen+3) * sizeof(char));
if (!dstFilenameTable[u]) {
UTIL_DISPLAYLEVEL(1, "Unable to allocate space for file destination str\n");
free(filename);
continue;
}
strcpy(dstFilenameTable[u], outDirName);
if (outDirName[strlen(outDirName)-1] == c[0]) {
strcat(dstFilenameTable[u], filename);
} else {
strcat(dstFilenameTable[u], c);
strcat(dstFilenameTable[u], filename);
}
free(filename);
}
if (UTIL_checkFilenameCollisions(dstFilenameTable, nbFiles)) {
UTIL_DISPLAYLEVEL(1, "Checking for filename collisions failed");
}
}
void UTIL_freeDestinationFilenameTable(char** dstDirTable, unsigned nbFiles) {
unsigned u;
for (u = 0; u < nbFiles; ++u) {
if (dstDirTable[u] != NULL)
free(dstDirTable[u]);
}
if (dstDirTable != NULL) free((void*)dstDirTable);
}
int UTIL_isSameFile(const char* file1, const char* file2)
{
#if defined(_MSC_VER)

View File

@ -130,11 +130,6 @@ U32 UTIL_isDirectory(const char* infilename);
int UTIL_getFileStat(const char* infilename, stat_t* statbuf);
int UTIL_isSameFile(const char* file1, const char* file2);
int UTIL_compareStr(const void *p1, const void *p2);
int UTIL_checkFilenameCollisions(char** dstFilenameTable, unsigned nbFiles);
/* Populates dstFilenameTable using outDirName concatenated with entries from filenameTable */
void UTIL_createDestinationDirTable(char** dstFilenameTable, const char** filenameTable, const unsigned nbFiles,
const char* outDirName);
void UTIL_freeDestinationFilenameTable(char** dstDirTable, unsigned nbFiles);
U32 UTIL_isLink(const char* infilename);
#define UTIL_FILESIZE_UNKNOWN ((U64)(-1))

View File

@ -585,7 +585,6 @@ int main(int argCount, const char* argv[])
unsigned recursive = 0;
unsigned memLimit = 0;
const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */
char** dstFilenameTable;
unsigned filenameIdx = 0;
const char* programName = argv[0];
const char* outFileName = NULL;
@ -1175,26 +1174,10 @@ int main(int argCount, const char* argv[])
if (adaptMin > cLevel) cLevel = adaptMin;
if (adaptMax < cLevel) cLevel = adaptMax;
if (outDirName) {
if (UTIL_isDirectory(outDirName)) {
DISPLAY("Output of files will be in directory: %s\n", outDirName);
dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*));
UTIL_createDestinationDirTable(dstFilenameTable, filenameTable, filenameIdx, outDirName);
} else {
DISPLAY("%s is not a directory!\n", outDirName);
CLEAN_RETURN(1);
}
} else {
dstFilenameTable = NULL;
}
if ((filenameIdx==1) && outFileName)
operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams);
else
operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, outDirName, dstFilenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams);
if (dstFilenameTable)
UTIL_freeDestinationFilenameTable(dstFilenameTable, filenameIdx);
operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, outDirName, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams);
#else
(void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; /* not used when ZSTD_NOCOMPRESS set */
DISPLAY("Compression not supported \n");
@ -1209,26 +1192,10 @@ int main(int argCount, const char* argv[])
}
}
FIO_setMemLimit(prefs, memLimit);
if (outDirName) {
if (UTIL_isDirectory(outDirName)) {
DISPLAY("Output of files will be in directory: %s\n", outDirName);
dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*));
UTIL_createDestinationDirTable(dstFilenameTable, filenameTable, filenameIdx, outDirName);
} else {
DISPLAY("%s is not a directory!\n", outDirName);
CLEAN_RETURN(1);
}
} else {
dstFilenameTable = NULL;
}
if (filenameIdx==1 && outFileName)
operationResult = FIO_decompressFilename(prefs, outFileName, filenameTable[0], dictFileName);
else
operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, dstFilenameTable, outFileName, dictFileName);
if (dstFilenameTable)
UTIL_freeDestinationFilenameTable(dstFilenameTable, filenameIdx);
operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, outFileName, dictFileName);
#else
DISPLAY("Decompression not supported \n");
#endif