mirrors/zstd
0
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2024-11-24 08:26:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into compressionFlow

Browse Source
This commit is contained in:
Yann Collet 2017-04-20 11:16:40 -07:00
commit 16f9c572fc
11 changed files with 94 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -111,7 +111,7 @@ clean:
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD))
HOST_OS = POSIX
CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON
.PHONY: list
list:

2
build/cmake/CMakeLists.txt
View File

@ -8,7 +8,7 @@
# ################################################################
PROJECT(zstd)
CMAKE_MINIMUM_REQUIRED(VERSION 2.8.7)
CMAKE_MINIMUM_REQUIRED(VERSION 2.8.9)
SET(ZSTD_SOURCE_DIR "${CMAKE_SOURCE_DIR}/../..")
#-----------------------------------------------------------------------------

4
build/cmake/contrib/pzstd/CMakeLists.txt
View File

@ -21,8 +21,8 @@ SET(PZSTD_DIR ${ZSTD_SOURCE_DIR}/contrib/pzstd)
INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${PZSTD_DIR})
ADD_EXECUTABLE(pzstd ${PZSTD_DIR}/main.cpp ${PZSTD_DIR}/Options.cpp ${PZSTD_DIR}/Pzstd.cpp ${PZSTD_DIR}/SkippableFrame.cpp)
SET_TARGET_PROPERTIES(pzstd PROPERTIES COMPILE_DEFINITIONS "NDEBUG")
SET_TARGET_PROPERTIES(pzstd PROPERTIES COMPILE_OPTIONS "-Wno-shadow")
SET_PROPERTY(TARGET pzstd APPEND PROPERTY COMPILE_DEFINITIONS "NDEBUG")
SET_PROPERTY(TARGET pzstd APPEND PROPERTY COMPILE_OPTIONS "-Wno-shadow")
SET(THREADS_PREFER_PTHREAD_FLAG ON)
FIND_PACKAGE(Threads REQUIRED)

16
build/cmake/lib/CMakeLists.txt
View File

@ -97,9 +97,9 @@ ENDIF (ZSTD_BUILD_STATIC)
# Add specific compile definitions for MSVC project
IF (MSVC)
SET_TARGET_PROPERTIES(libzstd_shared PROPERTIES COMPILE_DEFINITIONS "ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS")
SET_PROPERTY(TARGET libzstd_shared APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS")
IF (ZSTD_BUILD_STATIC)
SET_TARGET_PROPERTIES(libzstd_static PROPERTIES COMPILE_DEFINITIONS "ZSTD_HEAPMODE=0;_CRT_SECURE_NO_WARNINGS")
SET_PROPERTY(TARGET libzstd_static APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_HEAPMODE=0;_CRT_SECURE_NO_WARNINGS")
ENDIF (ZSTD_BUILD_STATIC)
ENDIF (MSVC)
@ -130,8 +130,20 @@ IF (ZSTD_BUILD_STATIC)
ENDIF (ZSTD_BUILD_STATIC)
IF (UNIX)
# pkg-config
SET(PREFIX "${CMAKE_INSTALL_PREFIX}")
SET(LIBDIR "${CMAKE_INSTALL_PREFIX}/lib")
SET(INCLUDEDIR "${CMAKE_INSTALL_PREFIX}/include")
SET(VERSION "${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}")
ADD_CUSTOM_TARGET(libzstd.pc ALL
${CMAKE_COMMAND} -DIN="${LIBRARY_DIR}/libzstd.pc.in" -DOUT="libzstd.pc"
-DPREFIX="${PREFIX}" -DLIBDIR="${LIBDIR}" -DINCLUDEDIR="${INCLUDEDIR}" -DVERSION="${VERSION}"
-P "${CMAKE_SOURCE_DIR}/lib/pkgconfig.cmake"
COMMENT "Creating pkg-config file")
# install target
INSTALL(FILES ${LIBRARY_DIR}/zstd.h ${LIBRARY_DIR}/deprecated/zbuff.h ${LIBRARY_DIR}/dictBuilder/zdict.h DESTINATION "include")
INSTALL(FILES "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" DESTINATION "share/pkgconfig")
INSTALL(TARGETS libzstd_shared LIBRARY DESTINATION "lib")
IF (ZSTD_BUILD_STATIC)
INSTALL(TARGETS libzstd_static ARCHIVE DESTINATION "lib")

1
build/cmake/lib/pkgconfig.cmake Normal file
View File

@ -0,0 +1 @@
CONFIGURE_FILE("${IN}" "${OUT}" @ONLY)

45
build/cmake/programs/CMakeLists.txt
View File

@ -31,27 +31,29 @@ ENDIF (MSVC)
ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources})
TARGET_LINK_LIBRARIES(zstd libzstd_shared)
ADD_CUSTOM_COMMAND(TARGET zstd POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink zstd zstdcat COMMENT "Creating zstdcat symlink")
ADD_CUSTOM_COMMAND(TARGET zstd POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink zstd unzstd COMMENT "Creating unzstd symlink")
ADD_CUSTOM_TARGET(zstdcat ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdcat DEPENDS zstd COMMENT "Creating zstdcat symlink")
ADD_CUSTOM_TARGET(unzstd ALL ${CMAKE_COMMAND} -E create_symlink zstd unzstd DEPENDS zstd COMMENT "Creating unzstd symlink")
INSTALL(TARGETS zstd RUNTIME DESTINATION "bin")
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat DESTINATION "bin")
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd DESTINATION "bin")
IF (UNIX)
FILE(COPY ${PROGRAMS_DIR}/zstd.1 DESTINATION .)
ADD_CUSTOM_COMMAND(TARGET zstd POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink zstd.1 zstdcat.1 COMMENT "Creating zstdcat.1 symlink")
ADD_CUSTOM_COMMAND(TARGET zstd POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink zstd.1 unzstd.1 COMMENT "Creating unzstd.1 symlink")
ADD_CUSTOM_TARGET(zstd.1 ALL
${CMAKE_COMMAND} -E copy ${PROGRAMS_DIR}/zstd.1 .
COMMENT "Copying manpage zstd.1")
ADD_CUSTOM_TARGET(zstdcat.1 ALL ${CMAKE_COMMAND} -E create_symlink zstd.1 zstdcat.1 DEPENDS zstd.1 COMMENT "Creating zstdcat.1 symlink")
ADD_CUSTOM_TARGET(unzstd.1 ALL ${CMAKE_COMMAND} -E create_symlink zstd.1 unzstd.1 DEPENDS zstd.1 COMMENT "Creating unzstd.1 symlink")
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstd.1 DESTINATION "share/man/man1")
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat.1 DESTINATION "share/man/man1")
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd.1 DESTINATION "share/man/man1")
ADD_EXECUTABLE(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c)
TARGET_LINK_LIBRARIES(zstd-frugal libzstd_shared)
SET_TARGET_PROPERTIES(zstd-frugal PROPERTIES COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT")
SET_PROPERTY(TARGET zstd-frugal APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT")
ENDIF (UNIX)
IF (ZSTD_MULTITHREAD_SUPPORT)
SET_TARGET_PROPERTIES(zstd PROPERTIES COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
SET_PROPERTY(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
SET(THREADS_PREFER_PTHREAD_FLAG ON)
FIND_PACKAGE(Threads REQUIRED)
@ -61,6 +63,33 @@ IF (ZSTD_MULTITHREAD_SUPPORT)
MESSAGE(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads")
ENDIF()
ADD_CUSTOM_COMMAND(TARGET zstd POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink zstd zstdmt COMMENT "Creating zstdmt symlink")
ADD_CUSTOM_TARGET(zstdmt ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdmt DEPENDS zstd COMMENT "Creating zstdmt symlink")
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdmt DESTINATION "bin")
ENDIF (ZSTD_MULTITHREAD_SUPPORT)
OPTION(ZSTD_ZLIB_SUPPORT "ZLIB SUPPORT" OFF)
OPTION(ZSTD_LZMA_SUPPORT "LZMA SUPPORT" OFF)
IF (ZSTD_ZLIB_SUPPORT)
FIND_PACKAGE(ZLIB REQUIRED)
IF (ZLIB_FOUND)
INCLUDE_DIRECTORIES(${ZLIB_INCLUDE_DIRS})
TARGET_LINK_LIBRARIES(zstd ${ZLIB_LIBRARIES})
SET_PROPERTY(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_GZCOMPRESS;ZSTD_GZDECOMPRESS")
ELSE ()
MESSAGE(SEND_ERROR "zlib library is missing")
ENDIF ()
ENDIF ()
IF (ZSTD_LZMA_SUPPORT)
FIND_PACKAGE(LibLZMA REQUIRED)
IF (LIBLZMA_FOUND)
INCLUDE_DIRECTORIES(${LIBLZMA_INCLUDE_DIRS})
TARGET_LINK_LIBRARIES(zstd ${LIBLZMA_LIBRARIES})
SET_PROPERTY(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_LZMACOMPRESS;ZSTD_LZMADECOMPRESS")
ELSE ()
MESSAGE(SEND_ERROR "lzma library is missing")
ENDIF ()
ENDIF ()

2
circle.yml
View File

@ -3,7 +3,7 @@ dependencies:
- sudo dpkg --add-architecture i386
- sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update
- sudo apt-get -y install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
- sudo apt-get -y install libstdc++-6-dev clang gcc g++ gcc-5 gcc-6
- sudo apt-get -y install libstdc++-6-dev clang gcc g++ gcc-5 gcc-6 zlib1g-dev liblzma-dev
- sudo apt-get -y install linux-libc-dev:i386 libc6-dev-i386
test:

12
lib/Makefile
View File

@ -71,6 +71,9 @@ libzstd.a: $(ZSTD_OBJ)
@echo compiling static library
@$(AR) $(ARFLAGS) $@ $^
libzstd.a-mt: CPPFLAGS += -DZSTD_MULTHREAD
libzstd.a-mt: libzstd.a
$(LIBZSTD): LDFLAGS += -shared -fPIC -fvisibility=hidden
$(LIBZSTD): $(ZSTD_FILES)
@echo compiling dynamic library $(LIBVER)
@ -86,10 +89,17 @@ endif
libzstd : $(LIBZSTD)
libzstd-mt : CPPFLAGS += -DZSTD_MULTITHREAD
libzstd-mt : libzstd
lib: libzstd.a libzstd
lib-release: DEBUGFLAGS :=
lib-mt: CPPFLAGS += -DZSTD_MULTITHREAD
lib-mt: lib
lib-release lib-release-mt: DEBUGFLAGS :=
lib-release: lib
lib-release-mt: lib-mt
clean:
@$(RM) -r *.dSYM # Mac OS-X specific

8
lib/README.md
View File

@ -22,6 +22,14 @@ Some additional API may be useful if you're looking into advanced features :
They are not "stable", their definition may change in the future.
Only static linking is allowed.
#### ZSTDMT API
To enable multithreaded compression within the library, invoke `make lib-mt` target.
Prototypes are defined in header file `compress/zstdmt_compress.h`.
When linking a program that uses ZSTDMT API against libzstd.a on a POSIX system,
`-pthread` flag must be provided to the compiler and linker.
Note : ZSTDMT prototypes can still be used with a library built without multithread support,
but in this case, they will be single threaded only.
#### Modular build

20
lib/compress/zstd_compress.c
View File

@ -535,8 +535,10 @@ static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc,
{ HUF_repeat repeat = zc->flagStaticHufTable;
int const preferRepeat = zc->params.cParams.strategy < ZSTD_lazy ? srcSize <= 1024 : 0;
if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1;
cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat)
: HUF_compress4X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat);
cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11,
zc->tmpCounters, sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat)
: HUF_compress4X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11,
zc->tmpCounters, sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat);
if (repeat != HUF_repeat_none) { hType = set_repeat; } /* reused the existing table */
else { zc->flagStaticHufTable = HUF_repeat_check; } /* now have a table to reuse */
}
@ -881,7 +883,14 @@ static unsigned ZSTD_NbCommonBytes (register size_t val)
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_ctzll((U64)val) >> 3);
# else
static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2,
0, 3, 1, 3, 1, 4, 2, 7,
0, 2, 3, 6, 1, 5, 3, 5,
1, 3, 4, 4, 2, 5, 6, 7,
7, 0, 1, 2, 3, 3, 4, 6,
2, 6, 5, 5, 3, 4, 5, 6,
7, 1, 2, 4, 6, 4, 4, 5,
7, 2, 6, 5, 7, 6, 7, 7 };
return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
# endif
} else { /* 32 bits */
@ -892,7 +901,10 @@ static unsigned ZSTD_NbCommonBytes (register size_t val)
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_ctz((U32)val) >> 3);
# else
static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0,
3, 2, 2, 1, 3, 2, 0, 1,
3, 3, 1, 2, 2, 2, 2, 0,
3, 1, 2, 0, 1, 0, 1, 1 };
return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
# endif
}

4
programs/Makefile
View File

@ -93,7 +93,7 @@ THREAD_MSG := ==> building with threading support
THREAD_CPP := -DZSTD_MULTITHREAD
THREAD_LD := -pthread
else
THREAD_MSG := NO_THREAD_MSG
THREAD_MSG := $(NO_THREAD_MSG)
endif
# zlib detection
@ -161,7 +161,7 @@ zstd-nolegacy : clean_decomp_o
zstd-nomt : THREAD_CPP :=
zstd-nomt : THREAD_LD :=
zstd-nomt : THREAD_MSG := NO_THREAD_MSG
zstd-nomt : THREAD_MSG := $(NO_THREAD_MSG)
zstd-nomt : zstd
zstd-pgo : MOREFLAGS = -fprofile-generate