diff --git a/programs/fileio.c b/programs/fileio.c index 6fbe0c8ed..dc5d95b39 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -687,16 +687,17 @@ static void FIO_getDictFileStat(const char* fileName, stat_t* dictFileStat) { } } -/* FIO_createDictBuffer() : - * creates a buffer, pointed by `*bufferPtr`, +/* FIO_setDictBufferMalloc() : + * allocates a buffer, pointed by `dict->dictBuffer`, * loads `filename` content into it, up to DICTSIZE_MAX bytes. * @return : loaded size * if fileName==NULL, returns 0 and a NULL pointer */ -static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat) +static size_t FIO_setDictBufferMalloc(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat) { FILE* fileHandle; U64 fileSize; + void** bufferPtr = &dict->dictBuffer; assert(bufferPtr != NULL); assert(dictFileStat != NULL); @@ -733,20 +734,17 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName, FIO_p #if (PLATFORM_POSIX_VERSION > 0) #include -static void* FIO_mmap(size_t fileSize, int fileHandle) +static void FIO_munmap(FIO_Dict_t* dict) { - return mmap - (NULL, (size_t)fileSize, PROT_READ, MAP_PRIVATE, fileHandle, 0); + munmap(dict->dictBuffer, dict->dictBufferSize); + dict->dictBuffer = NULL; + dict->dictBufferSize = 0; } -static int FIO_munmap(void* buffer, size_t bufferSize) -{ - return munmap(buffer, bufferSize); -} -/* We might want to also do mapping for windows */ -static size_t FIO_createDictBufferMMap(void** bufferPtr, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat) +static size_t FIO_setDictBufferMMap(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat) { int fileHandle; U64 fileSize; + void** bufferPtr = &dict->dictBuffer; assert(bufferPtr != NULL); assert(dictFileStat != NULL); @@ -770,27 +768,90 @@ static size_t FIO_createDictBufferMMap(void** bufferPtr, const char* fileName, F } } - *bufferPtr = FIO_mmap((size_t)fileSize, fileHandle); + *bufferPtr = mmap(NULL, (size_t)fileSize, PROT_READ, MAP_PRIVATE, fileHandle, 0); + if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno)); close(fileHandle); return (size_t)fileSize; } -#else -static size_t FIO_createDictBufferMMap(void** bufferPtr, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat) +#elif defined(_MSC_VER) || defined(_WIN32) +#include +static void FIO_munmap(FIO_Dict_t* dict) { - return FIO_createDictBuffer(bufferPtr, fileName, prefs, dictFileStat); + UnmapViewOfFile(dict->dictBuffer); + CloseHandle(dict->dictHandle); + dict->dictBuffer = NULL; + dict->dictBufferSize = 0; } -static void FIO_munmap(void* buffer, size_t bufferSize) { - (void)bufferSize; - free(buffer); +static size_t FIO_setDictBufferMMap(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat) +{ + HANDLE fileHandle, mapping; + U64 fileSize; + void** bufferPtr = &dict->dictBuffer; + + assert(bufferPtr != NULL); + assert(dictFileStat != NULL); + *bufferPtr = NULL; + if (fileName == NULL) return 0; + + DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName); + + fileHandle = CreateFileA(fileName, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_READONLY, NULL); + + if (fileHandle == INVALID_HANDLE_VALUE) { + EXM_THROW(33, "Couldn't open dictionary %s: %s", fileName, strerror(errno)); + } + + fileSize = UTIL_getFileSizeStat(dictFileStat); + { + size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX; + if (fileSize > dictSizeMax) { + EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)", + fileName, (unsigned)dictSizeMax); /* avoid extreme cases */ + } + } + + mapping = CreateFileMapping(fileHandle, NULL, PAGE_READONLY, 0, 0, NULL); + if (mapping == NULL) { + EXM_THROW(35, "Couldn't map dictionary %s: %s", fileName, strerror(errno)); + } + + *bufferPtr = MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, (DWORD)fileSize); /* we can only cast to DWORD here because dictSize <= 2GB */ + if (*bufferPtr==NULL) EXM_THROW(36, "%s", strerror(errno)); + + dict->dictHandle = fileHandle; + return (size_t)fileSize; +} +#else +static size_t FIO_setDictBufferMMap(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat) +{ + return FIO_setDictBufferMalloc(dict, fileName, prefs, dictFileStat); +} +static void FIO_munmap(FIO_Dict_t* dict) { + free(dict->dictBuffer); + dict->dictBuffer = NULL; + dict->dictBufferSize = 0; } #endif -static void FIO_freeDict(const FIO_Dict_t* dict) { +static void FIO_freeDict(FIO_Dict_t* dict) { if (dict->dictBufferType == FIO_mallocDict) { free(dict->dictBuffer); + dict->dictBuffer = NULL; + dict->dictBufferSize = 0; } else if (dict->dictBufferType == FIO_mmapDict) { - FIO_munmap(dict->dictBuffer, dict->dictBufferSize); + FIO_munmap(dict); + } else { + assert(0); /* Should not reach this case */ + } +} + +static void FIO_initDict(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat, FIO_dictBufferType_t dictBufferType) { + dict->dictBufferType = dictBufferType; + if (dict->dictBufferType == FIO_mallocDict) { + dict->dictBufferSize = FIO_setDictBufferMalloc(dict, fileName, prefs, dictFileStat); + } else if (dict->dictBufferType == FIO_mmapDict) { + dict->dictBufferSize = FIO_setDictBufferMMap(dict, fileName, prefs, dictFileStat); } else { assert(0); /* Should not reach this case */ } @@ -1045,6 +1106,7 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs, int cLevel, ZSTD_compressionParameters comprParams) { int useMMap = prefs->mmapDict == ZSTD_ps_enable; int forceNoUseMMap = prefs->mmapDict == ZSTD_ps_disable; + FIO_dictBufferType_t dictBufferType; cRess_t ress; memset(&ress, 0, sizeof(ress)); @@ -1065,13 +1127,8 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs, FIO_adjustParamsForPatchFromMode(prefs, &comprParams, dictSize, ssSize > 0 ? ssSize : maxSrcFileSize, cLevel); } - ress.dict.dictBufferType = (useMMap && !forceNoUseMMap) ? FIO_mmapDict : FIO_mallocDict; - - if (ress.dict.dictBufferType == FIO_mallocDict) { - ress.dict.dictBufferSize = FIO_createDictBuffer(&ress.dict.dictBuffer, dictFileName, prefs, &ress.dictFileStat); /* works with dictFileName==NULL */ - } else { - ress.dict.dictBufferSize = FIO_createDictBufferMMap(&ress.dict.dictBuffer, dictFileName, prefs, &ress.dictFileStat); - } + dictBufferType = (useMMap && !forceNoUseMMap) ? FIO_mmapDict : FIO_mallocDict; + FIO_initDict(&ress.dict, dictFileName, prefs, &ress.dictFileStat, dictBufferType); /* works with dictFileName==NULL */ ress.writeCtx = AIO_WritePool_create(prefs, ZSTD_CStreamOutSize()); ress.readCtx = AIO_ReadPool_create(prefs, ZSTD_CStreamInSize()); @@ -1135,7 +1192,7 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs, return ress; } -static void FIO_freeCResources(const cRess_t* const ress) +static void FIO_freeCResources(cRess_t* const ress) { FIO_freeDict(&(ress->dict)); AIO_WritePool_free(ress->writeCtx); @@ -1976,7 +2033,7 @@ int FIO_compressFilename(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, const const char* srcFileName, const char* dictFileName, int compressionLevel, ZSTD_compressionParameters comprParams) { - cRess_t const ress = FIO_createCResources(prefs, dictFileName, UTIL_getFileSize(srcFileName), compressionLevel, comprParams); + cRess_t ress = FIO_createCResources(prefs, dictFileName, UTIL_getFileSize(srcFileName), compressionLevel, comprParams); int const result = FIO_compressFilename_srcFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel); #define DISPLAY_LEVEL_DEFAULT 2 @@ -2169,7 +2226,6 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi } /* Allocation */ - ress.dict.dictBufferType = (useMMap && !forceNoUseMMap) ? FIO_mmapDict : FIO_mallocDict; ress.dctx = ZSTD_createDStream(); if (ress.dctx==NULL) EXM_THROW(60, "Error: %s : can't create ZSTD_DStream", strerror(errno)); @@ -2177,11 +2233,9 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi CHECK( ZSTD_DCtx_setParameter(ress.dctx, ZSTD_d_forceIgnoreChecksum, !prefs->checksumFlag)); /* dictionary */ - { if (ress.dict.dictBufferType == FIO_mallocDict) { - ress.dict.dictBufferSize = FIO_createDictBuffer(&ress.dict.dictBuffer, dictFileName, prefs, &statbuf); - } else { - ress.dict.dictBufferSize = FIO_createDictBufferMMap(&ress.dict.dictBuffer, dictFileName, prefs, &statbuf); - } + { + FIO_dictBufferType_t dictBufferType = (useMMap && !forceNoUseMMap) ? FIO_mmapDict : FIO_mallocDict; + FIO_initDict(&ress.dict, dictFileName, prefs, &statbuf, dictBufferType); CHECK(ZSTD_DCtx_reset(ress.dctx, ZSTD_reset_session_only) ); diff --git a/programs/fileio_types.h b/programs/fileio_types.h index e85abd477..2994a6092 100644 --- a/programs/fileio_types.h +++ b/programs/fileio_types.h @@ -78,6 +78,9 @@ typedef struct { void* dictBuffer; size_t dictBufferSize; FIO_dictBufferType_t dictBufferType; +#if defined(_MSC_VER) || defined(_WIN32) + HANDLE dictHandle; +#endif } FIO_Dict_t; #endif /* FILEIO_TYPES_HEADER */