From 77ef7847f7e531acc6626204a628e1969301d021 Mon Sep 17 00:00:00 2001 From: Hans Johnson Date: Mon, 24 Dec 2018 08:48:04 -0600 Subject: [PATCH 001/178] Improve setting the default build type The CMAKE_BUILD_TYPE variable is a CACHE variable and should be set in a way that persists and is documented in the CACHE. Also set the default values for the gui to ease selection of types. These changes provide better support for GUI configurators that support cmake. --- build/cmake/CMakeLists.txt | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt index c9dab0d5dbc..d0f943b5841 100644 --- a/build/cmake/CMakeLists.txt +++ b/build/cmake/CMakeLists.txt @@ -12,9 +12,12 @@ cmake_minimum_required(VERSION 2.8.9) project(zstd) set(ZSTD_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..") -if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) - message(STATUS "No build type selected, defaulting to Release") - set(CMAKE_BUILD_TYPE "Release") +# Set a default build type if none was specified +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + message(STATUS "Setting build type to 'Release' as none was specified.") + set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE) + # Set the possible values of build type for cmake-gui + set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") endif() list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules") From 1a279ae85ab0863aa49ca65fde7ae1a772144a56 Mon Sep 17 00:00:00 2001 From: Hans Johnson Date: Mon, 24 Dec 2018 08:37:07 -0600 Subject: [PATCH 002/178] BUG: list sub-command REMOVE_DUPLICATES requires list to be present When compiling without c++ enabled, some variables are not present. This is a check enforced in recent Cmake versions. CMake Error at CMakeModules/AddZstdCompilationFlags.cmake:54 (list): list sub-command REMOVE_DUPLICATES requires list to be present. Call Stack (most recent call first): CMakeLists.txt:53 (ADD_ZSTD_COMPILATION_FLAGS) --- .../cmake/CMakeModules/AddZstdCompilationFlags.cmake | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake b/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake index 5e65cedb5a2..6cdf2b3afb0 100644 --- a/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake +++ b/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake @@ -50,9 +50,11 @@ macro(ADD_ZSTD_COMPILATION_FLAGS) CMAKE_C_FLAGS_MINSIZEREL CMAKE_C_FLAGS_RELWITHDEBINFO CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO) - separate_arguments(${flag_var}) - list(REMOVE_DUPLICATES ${flag_var}) - string(REPLACE ";" " " ${flag_var} "${${flag_var}}") + if( ${flag_var} ) + separate_arguments(${flag_var}) + list(REMOVE_DUPLICATES ${flag_var}) + string(REPLACE ";" " " ${flag_var} "${${flag_var}}") + endif() endforeach () if (MSVC AND ZSTD_USE_STATIC_RUNTIME) @@ -60,7 +62,9 @@ macro(ADD_ZSTD_COMPILATION_FLAGS) CMAKE_C_FLAGS_MINSIZEREL CMAKE_C_FLAGS_RELWITHDEBINFO CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO) - string(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}") + if ( ${flag_var} ) + string(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}") + endif() endforeach () endif () From 889a4927849947ed9544bfb09a9aff616a1979c9 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Fri, 28 Dec 2018 10:07:05 +0700 Subject: [PATCH 003/178] travis: Use ninja from github --- .travis.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3ad59d2b4fc..35197990596 100644 --- a/.travis.yml +++ b/.travis.yml @@ -179,14 +179,17 @@ matrix: # meson dedicated test - name: Xenial (Meson + clang) + env: ALLOW_FAILURES=true dist: xenial language: cpp compiler: clang - before_install: + install: - sudo apt-get install -qq liblz4-dev valgrind tree - - curl -o ~/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' - - python3 ~/get-pip.py --user - - pip3 install --user meson ninja + - travis_retry curl -o ~/ninja.zip -L 'https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip' + && unzip ~/ninja.zip -d ~/.local/bin + - travis_retry curl -o ~/get-pip.py -L 'https://bootstrap.pypa.io/get-pip.py' + && python3 ~/get-pip.py --user + && pip3 install --user meson script: - meson --buildtype=debug -Db_lundef=false @@ -198,4 +201,4 @@ matrix: - DESTDIR=./staging ninja install - tree ./staging allow_failures: - - name: Xenial (Meson + clang) + - env: ALLOW_FAILURES=true From 226cdffd698fa8e42551934968e9c8597c23b396 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Fri, 28 Dec 2018 11:17:11 +0700 Subject: [PATCH 004/178] meson: Update build guide [skip ci] --- build/meson/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/meson/README.md b/build/meson/README.md index dae503fef57..d79ed49696f 100644 --- a/build/meson/README.md +++ b/build/meson/README.md @@ -7,7 +7,7 @@ modern software development tools and practices, such as unit tests, coverage reports, Valgrind, CCache and the like. This Meson build system is provided with no guarantee and maintained -by Dima Krasner . +by Dima Krasner \. It outputs one `libzstd`, either shared or static, depending on `default_library` option. @@ -17,7 +17,7 @@ It outputs one `libzstd`, either shared or static, depending on `cd` to this meson directory (`build/meson`) ```sh -meson --buildtype=release -D with-contrib=true -D with-tests=true -D with-contrib=true builddir +meson --buildtype=release -Dbuild_{programs,contrib}=true builddir cd builddir ninja # to build ninja install # to install From 787a72cdfe2f50c64dc08c991c289f38d680ad86 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Fri, 28 Dec 2018 11:20:33 +0700 Subject: [PATCH 005/178] meson: Correct generating pkgconf after Meson v0.49.0 --- build/meson/lib/meson.build | 4 ++-- build/meson/meson.build | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/build/meson/lib/meson.build b/build/meson/lib/meson.build index 8854d43706f..a02bd2d7a0e 100644 --- a/build/meson/lib/meson.build +++ b/build/meson/lib/meson.build @@ -116,9 +116,9 @@ libzstd = library('zstd', libzstd_dep = declare_dependency(link_with: libzstd, include_directories: libzstd_includes) -pkgconfig.generate(name: 'libzstd', +pkgconfig.generate(libzstd, + name: 'libzstd', filebase: 'libzstd', - libraries: [libzstd], description: 'fast lossless compression algorithm library', version: zstd_libversion, url: 'http://www.zstd.net/') diff --git a/build/meson/meson.build b/build/meson/meson.build index 99173f50c8c..6543158eb1b 100644 --- a/build/meson/meson.build +++ b/build/meson/meson.build @@ -11,9 +11,11 @@ project('zstd', ['c', 'cpp'], license: ['BSD', 'GPLv2'], - default_options : ['c_std=c99', + default_options : [ + 'c_std=c99', 'cpp_std=c++11', - 'buildtype=release'], + 'buildtype=release' + ], version: '1.3.8', meson_version: '>=0.47.0') From 142076ceae05138a6c1640f23731560ae3d69fcb Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Fri, 28 Dec 2018 11:55:26 +0700 Subject: [PATCH 006/178] travis: Remove deprecated sudo field --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 35197990596..b3a654378d8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ language: c dist: trusty -sudo: required git: depth: 1 From 8b2210411a02a8ec537ee64d71ece7b9a48d1149 Mon Sep 17 00:00:00 2001 From: "Dmitry V. Levin" Date: Thu, 27 Dec 2018 12:42:44 +0000 Subject: [PATCH 007/178] contrib/pzstd/Makefile: fix build of tests Apparently, Options.o cannot be linked in without $(PROGDIR)/util.o --- contrib/pzstd/Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/contrib/pzstd/Makefile b/contrib/pzstd/Makefile index 1d434dbcd45..8d2b1932e91 100644 --- a/contrib/pzstd/Makefile +++ b/contrib/pzstd/Makefile @@ -190,13 +190,15 @@ $(ZSTDDIR)/libzstd.a: $(ZSTD_FILES) CFLAGS="$(ALL_CFLAGS)" LDFLAGS="$(ALL_LDFLAGS)" $(MAKE) -C $(ZSTDDIR) libzstd.a # Rules to build the tests -test/RoundTripTest$(EXT): test/RoundTripTest.o $(PROGDIR)/datagen.o Options.o \ +test/RoundTripTest$(EXT): test/RoundTripTest.o $(PROGDIR)/datagen.o \ + $(PROGDIR)/util.o Options.o \ Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a $(LD_COMMAND) test/%Test$(EXT): PZSTD_LDFLAGS += $(GTEST_LIB) test/%Test$(EXT): LIBS += -lgtest -lgtest_main -test/%Test$(EXT): test/%Test.o $(PROGDIR)/datagen.o Options.o Pzstd.o \ +test/%Test$(EXT): test/%Test.o $(PROGDIR)/datagen.o \ + $(PROGDIR)/util.o Options.o Pzstd.o \ SkippableFrame.o $(ZSTDDIR)/libzstd.a $(LD_COMMAND) From 97d1de3d22c5ad53a465d76a5708a4ba90d0ed8b Mon Sep 17 00:00:00 2001 From: Hans Johnson Date: Mon, 24 Dec 2018 08:18:26 -0600 Subject: [PATCH 008/178] Provide forward compatible cmake paradigms Automatically extract version information from the zstd.h file. Use naming of variables consisent with modern cmake and https://semver.org/ (Semantic Versioning 2.0.0, MAJOR, MINOR, PATCH) Modern versions of cmake provide consistent paradigms for configuring project external interface values. This set of changes provide a back port of some of cmake 3+ paradigms back to cmake 2.8.9. Set and allow use of the current cmake policies for newer versions of cmake when available to allow for modern compiler features to be utilized when available. NOTE: The intent is that future modifications to cmake will enable (conditional on cmake version support) the ability to support modern linkage, and target export mechanisms. Those future changes will make incorporating zstd into other packages much easier. This patch also allows the more rigourous error checking of commmon cmake errors to be preformed by cmake (i.e. more stringent syntax checking and create errors for common hard to find misuses of cmake variables). This patch also provides support for modern compiler support options by cmake (like enabling interprocedural optimization if link time optimizations are known to be supported by the compiler envirionment. IPO can be supported by setting the CMAKE_INTERPROCEDURAL_OPTIMIZATION variable for newer versions of cmake. --- build/cmake/CMakeLists.txt | 43 +++++++++++++++++-- .../CMakeModules/GetZstdLibraryVersion.cmake | 5 ++- build/cmake/lib/CMakeLists.txt | 10 +---- 3 files changed, 45 insertions(+), 13 deletions(-) diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt index 2340f0edac0..833730716f0 100644 --- a/build/cmake/CMakeLists.txt +++ b/build/cmake/CMakeLists.txt @@ -7,10 +7,48 @@ # in the COPYING file in the root directory of this source tree). # ################################################################ -cmake_minimum_required(VERSION 2.8.9) +cmake_minimum_required(VERSION 2.8.9 FATAL_ERROR) + +# As of 2018-12-26 ZSTD has been validated to build with cmake version 3.13.2 new policies. +# Set and use the newest cmake policies that are validated to work +set(ZSTD_MAX_VALIDATED_CMAKE_VERSION "3.13.2") +if("${CMAKE_MAJOR_VERSION}" LESS 3) # Cmake version 2 does not understand the VERSION_LESS_EQUAL operator + set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}") +else() + if("${CMAKE_VERSION}" VERSION_LESS_EQUAL "${ZSTD_MAX_VALIDATED_CMAKE_VERSION}") + set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}") + else() + set(ZSTD_CMAKE_POLICY_VERSION "${ZSTD_MAX_VALIDATED_CMAKE_VERSION}") + endif() +endif() +cmake_policy(VERSION ${ZSTD_CMAKE_POLICY_VERSION}) -project(zstd) +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules") set(ZSTD_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..") +set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib) +# Parse version +include(GetZstdLibraryVersion) +GetZstdLibraryVersion(${LIBRARY_DIR}/zstd.h zstd_VERSION_MAJOR zstd_VERSION_MINOR zstd_VERSION_PATCH) + +if( CMAKE_MAJOR_VERSION LESS 3 ) + ## Provide cmake 3+ behavior for older versions of cmake + project(zstd) + set(PROJECT_VERSION_MAJOR ${zstd_VERSION_MAJOR}) + set(PROJECT_VERSION_MINOR ${zstd_VERSION_MINOR}) + set(PROJECT_VERSION_PATCH ${zstd_VERSION_PATCH}) + set(PROJECT_VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}") + enable_language(C) # Main library is in C + enable_language(CXX) # Testing contributed code also utilizes CXX +else() + project(zstd + VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}" + LANGUAGES C # Main library is in C + CXX # Testing contributed code also utilizes CXX + ) +endif() +message(STATUS "ZSTD VERSION: ${zstd_VERSION}") +set(zstd_HOMEPAGE_URL "http://www.zstd.net") +set(zstd_DESCRIPTION "Zstandard is a real-time compression algorithm, providing high compression ratios.") # Set a default build type if none was specified if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) @@ -20,7 +58,6 @@ if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") endif() -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules") include(GNUInstallDirs) #----------------------------------------------------------------------------- diff --git a/build/cmake/CMakeModules/GetZstdLibraryVersion.cmake b/build/cmake/CMakeModules/GetZstdLibraryVersion.cmake index af07723a57a..e8ed6064c96 100644 --- a/build/cmake/CMakeModules/GetZstdLibraryVersion.cmake +++ b/build/cmake/CMakeModules/GetZstdLibraryVersion.cmake @@ -1,9 +1,10 @@ -function(GetZstdLibraryVersion _header _major _minor _release) +function(GetZstdLibraryVersion _header _major _minor _patch) # Read file content file(READ ${_header} CONTENT) string(REGEX MATCH ".*define ZSTD_VERSION_MAJOR *([0-9]+).*define ZSTD_VERSION_MINOR *([0-9]+).*define ZSTD_VERSION_RELEASE *([0-9]+)" VERSION_REGEX "${CONTENT}") set(${_major} ${CMAKE_MATCH_1} PARENT_SCOPE) set(${_minor} ${CMAKE_MATCH_2} PARENT_SCOPE) - set(${_release} ${CMAKE_MATCH_3} PARENT_SCOPE) + set(${_patch} ${CMAKE_MATCH_3} PARENT_SCOPE) endfunction() + diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt index 6ed6c305eec..b99bb699c5a 100644 --- a/build/cmake/lib/CMakeLists.txt +++ b/build/cmake/lib/CMakeLists.txt @@ -18,14 +18,8 @@ if(NOT ZSTD_BUILD_SHARED AND NOT ZSTD_BUILD_STATIC) endif() # Define library directory, where sources and header files are located -set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib) include_directories(${LIBRARY_DIR} ${LIBRARY_DIR}/common) -# Parse version -include(GetZstdLibraryVersion) -GetZstdLibraryVersion(${LIBRARY_DIR}/zstd.h LIBVER_MAJOR LIBVER_MINOR LIBVER_RELEASE) -message(STATUS "ZSTD VERSION ${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}") - set(Sources ${LIBRARY_DIR}/common/entropy_common.c ${LIBRARY_DIR}/common/fse_decompress.c @@ -155,7 +149,7 @@ if (ZSTD_BUILD_SHARED) libzstd_shared PROPERTIES OUTPUT_NAME zstd - SOVERSION ${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}) + SOVERSION ${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}) endif () if (ZSTD_BUILD_STATIC) @@ -170,7 +164,7 @@ if (UNIX) set(PREFIX "${CMAKE_INSTALL_PREFIX}") set(LIBDIR "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}") set(INCLUDEDIR "${CMAKE_INSTALL_PREFIX}/include") - set(VERSION "${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}") + set(VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}") add_custom_target(libzstd.pc ALL ${CMAKE_COMMAND} -DIN="${LIBRARY_DIR}/libzstd.pc.in" -DOUT="libzstd.pc" -DPREFIX="${PREFIX}" -DLIBDIR="${LIBDIR}" -DINCLUDEDIR="${INCLUDEDIR}" -DVERSION="${VERSION}" From f9e4f89252f5cddb7a02a4e106e9280807c2bcd8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 2 Jan 2019 12:14:36 -0800 Subject: [PATCH 009/178] improved comments for adjustCParams() and getCParams() --- lib/compress/zstd_compress.c | 32 +++++++++++++++++--------------- lib/zstd.h | 16 ++++++++++------ 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index c2c9d3bc553..ef55f032388 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -888,10 +888,11 @@ static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) } /** ZSTD_adjustCParams_internal() : - optimize `cPar` for a given input (`srcSize` and `dictSize`). - mostly downsizing to reduce memory consumption and initialization latency. - Both `srcSize` and `dictSize` are optional (use 0 if unknown). - Note : cPar is assumed validated. Use ZSTD_checkCParams() to ensure this condition. */ + * optimize `cPar` for a specified input (`srcSize` and `dictSize`). + * mostly downsize to reduce memory consumption and initialization latency. + * `srcSize` can be ZSTD_CONTENTSIZE_UNKNOWN when not known. + * note : for the time being, `srcSize==0` means "unknown" too, for compatibility with older convention. + * condition : cPar is presumed validated (can be checked using ZSTD_checkCParams()). */ static ZSTD_compressionParameters ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, unsigned long long srcSize, @@ -901,7 +902,7 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, static const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1); assert(ZSTD_checkCParams(cPar)==0); - if (dictSize && (srcSize+1<2) /* srcSize unknown */ ) + if (dictSize && (srcSize+1<2) /* ZSTD_CONTENTSIZE_UNKNOWN and 0 mean "unknown" */ ) srcSize = minSrcSize; /* presumed small when there is a dictionary */ else if (srcSize == 0) srcSize = ZSTD_CONTENTSIZE_UNKNOWN; /* 0 == unknown : presumed large */ @@ -922,7 +923,7 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, } if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) - cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */ + cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* minimum wlog required for valid frame header */ return cPar; } @@ -932,7 +933,7 @@ ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize) { - cPar = ZSTD_clampCParams(cPar); + cPar = ZSTD_clampCParams(cPar); /* resulting cPar is necessarily valid (all parameters within range) */ return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize); } @@ -4151,7 +4152,7 @@ int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } int ZSTD_minCLevel(void) { return (int)-ZSTD_TARGETLENGTH_MAX; } static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = { -{ /* "default" - guarantees a monotonically increasing memory budget */ +{ /* "default" - for any srcSize > 256 KB */ /* W, C, H, S, L, TL, strat */ { 19, 12, 13, 1, 6, 1, ZSTD_fast }, /* base for negative levels */ { 19, 13, 14, 1, 7, 0, ZSTD_fast }, /* level 1 */ @@ -4258,13 +4259,13 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV }; /*! ZSTD_getCParams() : -* @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize. -* Size values are optional, provide 0 if not known or unused */ + * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize. + * Size values are optional, provide 0 if not known or unused */ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) { size_t const addedSize = srcSizeHint ? 0 : 500; - U64 const rSize = srcSizeHint+dictSize ? srcSizeHint+dictSize+addedSize : (U64)-1; - U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */ + U64 const rSize = srcSizeHint+dictSize ? srcSizeHint+dictSize+addedSize : ZSTD_CONTENTSIZE_UNKNOWN; /* intentional overflow for srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN */ + U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); int row = compressionLevel; DEBUGLOG(5, "ZSTD_getCParams (cLevel=%i)", compressionLevel); if (compressionLevel == 0) row = ZSTD_CLEVEL_DEFAULT; /* 0 == default */ @@ -4272,13 +4273,14 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l if (compressionLevel > ZSTD_MAX_CLEVEL) row = ZSTD_MAX_CLEVEL; { ZSTD_compressionParameters cp = ZSTD_defaultCParameters[tableID][row]; if (compressionLevel < 0) cp.targetLength = (unsigned)(-compressionLevel); /* acceleration factor */ - return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize); + return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize); /* refine parameters based on srcSize & dictSize */ } } /*! ZSTD_getParams() : -* same as ZSTD_getCParams(), but @return a `ZSTD_parameters` object (instead of `ZSTD_compressionParameters`). -* All fields of `ZSTD_frameParameters` are set to default (0) */ + * same idea as ZSTD_getCParams() + * @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`). + * Fields of `ZSTD_frameParameters` are set to default values */ ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) { ZSTD_parameters params; ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSizeHint, dictSize); diff --git a/lib/zstd.h b/lib/zstd.h index b18fc8a44bf..237635cbaf3 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1226,22 +1226,26 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictS ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); /*! ZSTD_getCParams() : -* @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. -* `estimatedSrcSize` value is optional, select 0 if not known */ + * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. + * `estimatedSrcSize` value is optional, select 0 if not known */ ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); /*! ZSTD_getParams() : -* same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`. -* All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */ + * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`. + * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */ ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); /*! ZSTD_checkCParams() : -* Ensure param values remain within authorized range */ + * Ensure param values remain within authorized range. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */ ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); /*! ZSTD_adjustCParams() : * optimize params for a given `srcSize` and `dictSize`. - * both values are optional, select `0` if unknown. */ + * `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN. + * `dictSize` must be `0` when there is no dictionary. + * cPar can be invalid : all parameters will be clamped within valid range in the @return struct. + * This function never fails (wide contract) */ ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); /*! ZSTD_compress_advanced() : From 9448a3790dec147ac05a3ba02bcbb52a5b9b090a Mon Sep 17 00:00:00 2001 From: Erik Webb Date: Thu, 3 Jan 2019 15:26:20 -0500 Subject: [PATCH 010/178] Check CMake minor version support for VERSION_LESS_EQUAL VERSION_LESS_EQUAL is only available to CMake 3.7+. This adds additional logic to check that CMAKE_MINOR_VERSION is at least 7. Fixes #1489 --- build/cmake/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt index 833730716f0..f72f3f458cd 100644 --- a/build/cmake/CMakeLists.txt +++ b/build/cmake/CMakeLists.txt @@ -12,9 +12,9 @@ cmake_minimum_required(VERSION 2.8.9 FATAL_ERROR) # As of 2018-12-26 ZSTD has been validated to build with cmake version 3.13.2 new policies. # Set and use the newest cmake policies that are validated to work set(ZSTD_MAX_VALIDATED_CMAKE_VERSION "3.13.2") -if("${CMAKE_MAJOR_VERSION}" LESS 3) # Cmake version 2 does not understand the VERSION_LESS_EQUAL operator +if("${CMAKE_MAJOR_VERSION}" LESS 3) # Cmake version <3.7 does not understand the VERSION_LESS_EQUAL operator set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}") -else() +elseif(("${CMAKE_MAJOR_VERSION}" EQUAL 3 AND "${CMAKE_MINOR_VERSION}" GREATER 6) OR ("${CMAKE_MAJOR_VERSION}" GREATER 3)) if("${CMAKE_VERSION}" VERSION_LESS_EQUAL "${ZSTD_MAX_VALIDATED_CMAKE_VERSION}") set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}") else() From fe82637069bfdf09c9fcb45076379fd79291d09a Mon Sep 17 00:00:00 2001 From: Conrad Meyer Date: Fri, 4 Jan 2019 11:57:12 -0800 Subject: [PATCH 011/178] Fix #1425 - Use physical core count API on FreeBSD Similar to Apple, use the native physical core count sysctl, when available. This is a little repetitive (it's basically the __APPLE__ method plus the otherBSD method concatenated together) but seemed clearer than any way that would totally eliminate repetition. The __FreeBSD_version check only tests the version of the FreeBSD kernel that zstd is compiled on; importantly, it may be run on a different version. So the compile-time check is a little naive and needs to be able to fallback to work on older versions of FreeBSD. For a similar reason, it may make sense to simply eliminate the __FreeBSD_version check entirely. The tradeoff is that a spurious sysctlbyname would be issued when -T0 is used on older kernels. --- programs/util.c | 38 +++++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/programs/util.c b/programs/util.c index 34634318c63..49eea148ec1 100644 --- a/programs/util.c +++ b/programs/util.c @@ -640,10 +640,42 @@ int UTIL_countPhysicalCores(void) } } -#elif defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__) +#elif defined(__FreeBSD__) -/* Use apple-provided syscall - * see: man 3 sysctl */ +#include +#include + +/* Use physical core sysctl when available + * see: man 4 smp, man 3 sysctl */ +int UTIL_countPhysicalCores(void) +{ + static int numPhysicalCores = 0; /* freebsd sysctl is native int sized */ + if (numPhysicalCores != 0) return numPhysicalCores; + +#if __FreeBSD_version >= 1300008 + { size_t size = sizeof(numPhysicalCores); + int ret = sysctlbyname("kern.smp.cores", &numPhysicalCores, &size, NULL, 0); + if (ret == 0) return numPhysicalCores; + if (errno != ENOENT) { + perror("zstd: can't get number of physical cpus"); + exit(1); + } + /* sysctl not present, fall through to older sysconf method */ + } +#endif + + numPhysicalCores = (int)sysconf(_SC_NPROCESSORS_ONLN); + if (numPhysicalCores == -1) { + /* value not queryable, fall back on 1 */ + numPhysicalCores = 1; + } + return numPhysicalCores; +} + +#elif defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__) + +/* Use POSIX sysconf + * see: man 3 sysconf */ int UTIL_countPhysicalCores(void) { static int numPhysicalCores = 0; From 7b6a8840c5654ad0aa89c067b309c0a9579d8c27 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Sun, 6 Jan 2019 21:22:49 +0700 Subject: [PATCH 012/178] Fix #1428 - zstdgrep returned 1 on match and unmatch - Use ZCAT for testing zstdgrep in case of non-install yet - tests: Add file test for zstdgrep --- programs/zstdgrep | 24 ++++++++++-------------- tests/Makefile | 8 ++++++-- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/programs/zstdgrep b/programs/zstdgrep index a10e0710a29..cb804b8bead 100755 --- a/programs/zstdgrep +++ b/programs/zstdgrep @@ -22,8 +22,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -grep=grep -zcat=zstdcat +grep=${GREP:-grep} +zcat=${ZCAT:-zstdcat} endofopts=0 pattern_found=0 @@ -31,12 +31,13 @@ grep_args="" hyphen=0 silent=0 -prg=$(basename "$0") +prog=${0##*/} # handle being called 'zegrep' or 'zfgrep' -case "${prg}" in - *zegrep) grep_args="-E";; - *zfgrep) grep_args="-F";; +case $prog in + *egrep*) prog=zegrep; grep_args='-E';; + *fgrep*) prog=zfgrep; grep_args='-F';; + *) prog=zstdgrep;; esac # skip all options and pass them on to grep taking care of options @@ -47,7 +48,7 @@ while [ "$#" -gt 0 ] && [ "${endofopts}" -eq 0 ]; do # from GNU grep-2.5.1 -- keep in sync! -[ABCDXdefm]) if [ "$#" -lt 2 ]; then - printf '%s: missing argument for %s flag\n' "${prg}" "$1" >&2 + printf '%s: missing argument for %s flag\n' "${prog}" "$1" >&2 exit 1 fi case "$1" in @@ -94,7 +95,7 @@ if [ "${pattern_found}" -lt 1 ]; then elif [ "${hyphen}" -gt 0 ]; then pattern="-" else - printf '%s: missing pattern\n' "${prg}" >&2 + printf '%s: missing pattern\n' "${prog}" >&2 exit 1 fi fi @@ -113,16 +114,11 @@ else if [ "${silent}" -lt 1 ] && [ "$#" -gt 1 ]; then grep_args="-H ${grep_args}" fi - CUR_EXIT_CODE=0 - EXIT_CODE=1 set -f while [ "$#" -gt 0 ]; do # shellcheck disable=SC2086 "${zcat}" -fq -- "$1" | "${grep}" --label="${1}" ${grep_args} -- "${pattern}" - - CUR_EXIT_CODE=$? - if [ "${CUR_EXIT_CODE}" -eq 0 ] && [ "${EXIT_CODE}" -ne 1 ]; then - EXIT_CODE=0 - fi + [ "$?" -ne 0 ] && EXIT_CODE=1 shift done set +f diff --git a/tests/Makefile b/tests/Makefile index 25bd5c84ef9..2daf0970f07 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -353,8 +353,12 @@ test-gzstd: gzstd $(RM) *.gz *.zst README2.md gz_zstd zstd_gz hello.txt test-zstdgrep: gzstd - @echo a | $(PRGDIR)/zstd | $(PRGDIR)/zstdgrep a - @echo a | $(PRGDIR)/zstd | $(PRGDIR)/zstdgrep b && return 1 || return 0 + -[ -f /tmp/zstdcat ] || ln -s $(PWD)/$(PRGDIR)/zstd /tmp/zstdcat + echo a | $(PRGDIR)/zstd | env ZCAT=/tmp/zstdcat $(PRGDIR)/zstdgrep a + echo a | $(PRGDIR)/zstd | env ZCAT=/tmp/zstdcat $(PRGDIR)/zstdgrep b && return 1 || return 0 + -echo 'hello world' > test.txt && $(PRGDIR)/zstd test.txt + env ZCAT=/tmp/zstdcat $(PRGDIR)/zstdgrep hello test.txt.zst + env ZCAT=/tmp/zstdcat $(PRGDIR)/zstdgrep weird test.txt.zst && return 1 || return 0 test-fullbench: fullbench datagen $(QEMU_SYS) ./fullbench -i1 From de7e3be7fad12b335c9dc29ba213e98d74890a7c Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Sun, 6 Jan 2019 23:28:38 +0700 Subject: [PATCH 013/178] Fix potential leak of 'outBuff' (reported by scan-build) --- tests/legacy.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/legacy.c b/tests/legacy.c index e1cf82f2f9d..4234e8fbd70 100644 --- a/tests/legacy.c +++ b/tests/legacy.c @@ -87,6 +87,7 @@ static int testStreamingAPI(void) } if (stream == NULL) { DISPLAY("ERROR: Could not create dstream\n"); + free(outBuff); return 1; } From 260ff2f6b7d590345bde6825a1ad0dc7b9aac9b1 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Sun, 6 Jan 2019 23:42:24 +0700 Subject: [PATCH 014/178] tests/legagy.c: More fixes --- tests/legacy.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/legacy.c b/tests/legacy.c index 4234e8fbd70..b749567f440 100644 --- a/tests/legacy.c +++ b/tests/legacy.c @@ -74,6 +74,7 @@ static int testSimpleAPI(void) static int testStreamingAPI(void) { + int error_code = 0; size_t const outBuffSize = ZSTD_DStreamOutSize(); char* const outBuff = malloc(outBuffSize); ZSTD_DStream* const stream = ZSTD_createDStream(); @@ -97,13 +98,15 @@ static int testStreamingAPI(void) size_t const ret = ZSTD_initDStream(stream); if (ZSTD_isError(ret)) { DISPLAY("ERROR: ZSTD_initDStream: %s\n", ZSTD_getErrorName(ret)); - return 1; + error_code = 1; + break; } } { size_t const ret = ZSTD_decompressStream(stream, &output, &input); if (ZSTD_isError(ret)) { DISPLAY("ERROR: ZSTD_decompressStream: %s\n", ZSTD_getErrorName(ret)); - return 1; + error_code = 1; + break; } if (ret == 0) { @@ -112,7 +115,8 @@ static int testStreamingAPI(void) if (memcmp(outBuff, EXPECTED + outputPos, output.pos) != 0) { DISPLAY("ERROR: Wrong decoded output produced\n"); - return 1; + error_code = 1; + break; } outputPos += output.pos; if (input.pos == input.size && output.pos < output.size) { @@ -122,8 +126,8 @@ static int testStreamingAPI(void) free(outBuff); ZSTD_freeDStream(stream); - DISPLAY("Streaming API OK\n"); - return 0; + if (error_code == 0) DISPLAY("Streaming API OK\n"); + return error_code; } int main(void) From 62c0dd7affbec18a2277d420c498874120628375 Mon Sep 17 00:00:00 2001 From: Hans Johnson Date: Tue, 8 Jan 2019 08:43:04 -0600 Subject: [PATCH 015/178] ENH: Simplify conditional logic Based on excellent comment in #1489, the logic to set cmake policies was simplified. This will provide an easier mechanism for maintaining the conditional logic across many versions of cmake. --- build/cmake/CMakeLists.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt index f72f3f458cd..33c05aee856 100644 --- a/build/cmake/CMakeLists.txt +++ b/build/cmake/CMakeLists.txt @@ -11,15 +11,15 @@ cmake_minimum_required(VERSION 2.8.9 FATAL_ERROR) # As of 2018-12-26 ZSTD has been validated to build with cmake version 3.13.2 new policies. # Set and use the newest cmake policies that are validated to work -set(ZSTD_MAX_VALIDATED_CMAKE_VERSION "3.13.2") -if("${CMAKE_MAJOR_VERSION}" LESS 3) # Cmake version <3.7 does not understand the VERSION_LESS_EQUAL operator +set(ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION "3") +set(ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION "13") #Policies never changed at PATCH level +if("${CMAKE_MAJOR_VERSION}" LESS 3) set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}") -elseif(("${CMAKE_MAJOR_VERSION}" EQUAL 3 AND "${CMAKE_MINOR_VERSION}" GREATER 6) OR ("${CMAKE_MAJOR_VERSION}" GREATER 3)) - if("${CMAKE_VERSION}" VERSION_LESS_EQUAL "${ZSTD_MAX_VALIDATED_CMAKE_VERSION}") +elseif( "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}" EQUAL "${CMAKE_MAJOR_VERSION}" AND + "${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}" GREATER "${CMAKE_MINOR_VERSION}") set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}") - else() - set(ZSTD_CMAKE_POLICY_VERSION "${ZSTD_MAX_VALIDATED_CMAKE_VERSION}") - endif() +else() + set(ZSTD_CMAKE_POLICY_VERSION "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}.${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}.0") endif() cmake_policy(VERSION ${ZSTD_CMAKE_POLICY_VERSION}) From 6ff9d5e881383ca2abcf51fa3aafc53c21021f11 Mon Sep 17 00:00:00 2001 From: Li-Wen Hsu Date: Wed, 16 Jan 2019 04:55:43 +0800 Subject: [PATCH 016/178] Include unistd.h on unix platforms for explicit function declaration --- zlibWrapper/gzguts.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zlibWrapper/gzguts.h b/zlibWrapper/gzguts.h index 05bf4d9f4c3..b639b4be85f 100644 --- a/zlibWrapper/gzguts.h +++ b/zlibWrapper/gzguts.h @@ -38,6 +38,8 @@ #ifdef _WIN32 # include +#else +# include #endif #if defined(__TURBOC__) || defined(_MSC_VER) || defined(_WIN32) From 4fa585aee375867c99787467a0258fe10a3d4bd5 Mon Sep 17 00:00:00 2001 From: Karl Ostmo Date: Mon, 14 Jan 2019 17:58:46 -0800 Subject: [PATCH 017/178] fix --list on truncated files fseek() doesn't indicate when it moves past the end of a file. Consequently, if a file is truncated within its last block, the error would't be detected. This PR adds a test scenario that induces this situation using a small compressed file of only one block in size. This test is added to tests/playTests.sh Check is implemented by ensuring that the filehandle position is equal to the filesize upon exit. --- programs/fileio.c | 52 +++++++++++++++++++++++++++++++++------------- tests/playTests.sh | 13 ++++++++++++ 2 files changed, 51 insertions(+), 14 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 9fb795ed3f2..8538285bdb2 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -237,10 +237,13 @@ void FIO_addAbortHandler() ***************************************************************/ #if defined(_MSC_VER) && _MSC_VER >= 1400 # define LONG_SEEK _fseeki64 +# define LONG_TELL _ftelli64 #elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ # define LONG_SEEK fseeko +# define LONG_TELL ftello #elif defined(__MINGW32__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) && defined(__MSVCRT__) # define LONG_SEEK fseeko64 +# define LONG_TELL ftello64 #elif defined(_WIN32) && !defined(__DJGPP__) # include static int LONG_SEEK(FILE* file, __int64 offset, int origin) { @@ -261,6 +264,7 @@ void FIO_addAbortHandler() } #else # define LONG_SEEK fseek +# define LONG_TELL ftell #endif @@ -2142,7 +2146,13 @@ typedef struct { U32 nbFiles; } fileInfo_t; -typedef enum { info_success=0, info_frame_error=1, info_not_zstd=2, info_file_error=3 } InfoError; +typedef enum { + info_success=0, + info_frame_error=1, + info_not_zstd=2, + info_file_error=3, + info_truncated_input=4, +} InfoError; #define ERROR_IF(c,n,...) { \ if (c) { \ @@ -2164,6 +2174,12 @@ FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile) && (numBytesRead == 0) && (info->compressedSize > 0) && (info->compressedSize != UTIL_FILESIZE_UNKNOWN) ) { + unsigned long long file_position = (unsigned long long) LONG_TELL(srcFile); + unsigned long long file_size = (unsigned long long) info->compressedSize; + ERROR_IF(file_position != file_size, info_truncated_input, + "Error: seeked to position %llu, which is beyond file size of %llu\n", + file_position, + file_size); break; /* correct end of file => success */ } ERROR_IF(feof(srcFile), info_not_zstd, "Error: reached end of file with incomplete frame"); @@ -2332,20 +2348,28 @@ FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLevel) fileInfo_t info; memset(&info, 0, sizeof(info)); { InfoError const error = getFileInfo(&info, inFileName); - if (error == info_frame_error) { - /* display error, but provide output */ - DISPLAYLEVEL(1, "Error while parsing %s \n", inFileName); - } - else if (error == info_not_zstd) { - DISPLAYOUT("File %s not compressed by zstd \n", inFileName); - if (displayLevel > 2) DISPLAYOUT("\n"); - return 1; - } - else if (error == info_file_error) { - /* error occurred while opening the file */ - if (displayLevel > 2) DISPLAYOUT("\n"); - return 1; + switch (error) { + case info_frame_error: + /* display error, but provide output */ + DISPLAYLEVEL(1, "Error while parsing \"%s\" \n", inFileName); + break; + case info_not_zstd: + DISPLAYOUT("File \"%s\" not compressed by zstd \n", inFileName); + if (displayLevel > 2) DISPLAYOUT("\n"); + return 1; + case info_file_error: + /* error occurred while opening the file */ + if (displayLevel > 2) DISPLAYOUT("\n"); + return 1; + case info_truncated_input: + DISPLAYOUT("File \"%s\" is truncated \n", inFileName); + if (displayLevel > 2) DISPLAYOUT("\n"); + return 1; + case info_success: + default: + break; } + displayInfo(inFileName, &info, displayLevel); *total = FIO_addFInfo(*total, info); assert(error == info_success || error == info_frame_error); diff --git a/tests/playTests.sh b/tests/playTests.sh index ef4861b5291..8342455a1e7 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -809,6 +809,19 @@ $ZSTD --list tmp* && die "-l must fail on non-zstd file" $ZSTD -lv tmp1* && die "-l must fail on non-zstd file" $ZSTD --list -v tmp2 tmp12.zst && die "-l must fail on non-zstd file" +$ECHO "test : detect truncated compressed file " +TEST_DATA_FILE=truncatable-input.txt +FULL_COMPRESSED_FILE=${TEST_DATA_FILE}.zst +TRUNCATED_COMPRESSED_FILE=truncated-input.txt.zst +./datagen -g50000 > $TEST_DATA_FILE +$ZSTD -f $TEST_DATA_FILE -o $FULL_COMPRESSED_FILE +head -c 100 $FULL_COMPRESSED_FILE > $TRUNCATED_COMPRESSED_FILE +$ZSTD --list $TRUNCATED_COMPRESSED_FILE && die "-l must fail on truncated file" + +rm $TEST_DATA_FILE +rm $FULL_COMPRESSED_FILE +rm $TRUNCATED_COMPRESSED_FILE + $ECHO "\n===> zstd --list/-l errors when presented with stdin / no files" $ZSTD -l && die "-l must fail on empty list of files" $ZSTD -l - && die "-l does not work on stdin" From 1828c2619c32bdf7f6411dadcc85d19f0a281445 Mon Sep 17 00:00:00 2001 From: Li-Wen Hsu Date: Fri, 11 Jan 2019 04:00:27 +0800 Subject: [PATCH 018/178] Add Cirrus-CI config for FreeBSD builds --- .cirrus.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .cirrus.yml diff --git a/.cirrus.yml b/.cirrus.yml new file mode 100644 index 00000000000..777d3eca109 --- /dev/null +++ b/.cirrus.yml @@ -0,0 +1,15 @@ +env: + CIRRUS_CLONE_DEPTH: 1 + ARCH: amd64 + +task: + freebsd_instance: + matrix: + image: freebsd-12-0-release-amd64 + image: freebsd-11-2-release-amd64 + install_script: + - sed -i.bak -e 's,pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly,pkg+http://pkg.FreeBSD.org/\${ABI}/latest,' /etc/pkg/FreeBSD.conf + - pkg upgrade -y + - pkg install -y gmake + script: + - CFLAGS="-Werror" gmake -j all From 281c7970ac36e26e087a047a9aa337c7b735a162 Mon Sep 17 00:00:00 2001 From: Li-Wen Hsu Date: Thu, 17 Jan 2019 03:32:46 +0800 Subject: [PATCH 019/178] Add Cirrus-CI build status badge --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 4b6d19e7e72..e3f19c5a352 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ a list of known ports and bindings is provided on [Zstandard homepage](http://ww [![Build Status][travisDevBadge]][travisLink] [![Build status][AppveyorDevBadge]][AppveyorLink] [![Build status][CircleDevBadge]][CircleLink] +[![Build status][CirrusDevBadge]][CirrusLink] [travisDevBadge]: https://travis-ci.org/facebook/zstd.svg?branch=dev "Continuous Integration test suite" [travisLink]: https://travis-ci.org/facebook/zstd @@ -21,6 +22,8 @@ a list of known ports and bindings is provided on [Zstandard homepage](http://ww [AppveyorLink]: https://ci.appveyor.com/project/YannCollet/zstd-p0yf0 [CircleDevBadge]: https://circleci.com/gh/facebook/zstd/tree/dev.svg?style=shield "Short test suite" [CircleLink]: https://circleci.com/gh/facebook/zstd +[CirrusDevBadge]: https://api.cirrus-ci.com/github/facebook/zstd.svg +[CirrusLink]: https://cirrus-ci.com/github/facebook/zstd ## Benchmarks From a1394399b4d750bc74af426460177166e82c7339 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 19 Jan 2019 23:38:20 -0800 Subject: [PATCH 020/178] fixed minor conversion warnings in examples/ --- doc/zstd_manual.html | 16 ++++++++++------ examples/multiple_simple_compression.c | 8 ++++---- examples/utils.h | 4 ++-- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index 6dfa6d997cb..c7962e7de0c 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -1011,22 +1011,26 @@

Streaming decompression functions

size_t ZSTD_initDSt
 


 
 
ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
-
   @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
-   `estimatedSrcSize` value is optional, select 0 if not known 
+
 @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
+ `estimatedSrcSize` value is optional, select 0 if not known 
 


 
 
ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
-
   same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`.
-   All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 
+
  same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`.
+  All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 
 


 
 
size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
-
   Ensure param values remain within authorized range 
+
  Ensure param values remain within authorized range.
+ @return 0 on success, or an error code (can be checked with ZSTD_isError()) 
 


 
 
ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
 
  optimize params for a given `srcSize` and `dictSize`.
-  both values are optional, select `0` if unknown. 
+ `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN.
+ `dictSize` must be `0` when there is no dictionary.
+  cPar can be invalid : all parameters will be clamped within valid range in the @return struct.
+  This function never fails (wide contract) 
 


 
 
size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
diff --git a/examples/multiple_simple_compression.c b/examples/multiple_simple_compression.c
index 65c775bfa1d..b9bb29a9de4 100644
--- a/examples/multiple_simple_compression.c
+++ b/examples/multiple_simple_compression.c
@@ -28,7 +28,7 @@ typedef struct {
  * allocate memory for buffers big enough to compress all files
  * as well as memory for output file name (ofn)
  */
-static resources createResources_orDie(int argc, const char** argv, char **ofn, int* ofnBufferLen)
+static resources createResources_orDie(int argc, const char** argv, char **ofn, size_t* ofnBufferLen)
 {
     size_t maxFilenameLength=0;
     size_t maxFileSize = 0;
@@ -94,14 +94,14 @@ int main(int argc, const char** argv)
 
     /* memory allocation for outFilename and resources */
     char* outFilename;
-    int outFilenameBufferLen;
-    resources const ress = createResources_orDie(argc, argv, &outFilename, &outFilenameBufferLen); 
+    size_t outFilenameBufferLen;
+    resources const ress = createResources_orDie(argc, argv, &outFilename, &outFilenameBufferLen);
 
     /* compress files with shared context, input and output buffers */
     int argNb;
     for (argNb = 1; argNb < argc; argNb++) {
         const char* const inFilename = argv[argNb];
-        int inFilenameLen = strlen(inFilename);
+        size_t const inFilenameLen = strlen(inFilename);
         assert(inFilenameLen + 5 <= outFilenameBufferLen);
         memcpy(outFilename, inFilename, inFilenameLen);
         memcpy(outFilename+inFilenameLen, ".zst", 5);
diff --git a/examples/utils.h b/examples/utils.h
index 6d13604579f..77c7a4f0ca9 100644
--- a/examples/utils.h
+++ b/examples/utils.h
@@ -56,7 +56,7 @@ static size_t fsize_orDie(const char *filename)
      * 2. if off_t -> size_t type conversion results in discrepancy,
      *    the file size is too large for type size_t.
      */
-    if ((fileSize < 0) || (fileSize != (off_t)size)) { 
+    if ((fileSize < 0) || (fileSize != (off_t)size)) {
         fprintf(stderr, "%s : filesize too large \n", filename);
         exit(ERROR_largeFile);
     }
@@ -150,7 +150,7 @@ static void* malloc_orDie(size_t size)
  * @return If successful this function will load file into buffer and
  * return file size, otherwise it will printout an error to stderr and exit.
  */
-static size_t loadFile_orDie(const char* fileName, void* buffer, int bufferSize)
+static size_t loadFile_orDie(const char* fileName, void* buffer, size_t bufferSize)
 {
     size_t const fileSize = fsize_orDie(fileName);
     assert(fileSize <= bufferSize);

From 2a6aa6be5f2f76593036d5a41aa27ae43c56b1dd Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sat, 19 Jan 2019 23:40:41 -0800
Subject: [PATCH 021/178] updated clang tests

target clangbuild
---
 Makefile | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/Makefile b/Makefile
index ed78d195613..f2ec8c9b107 100644
--- a/Makefile
+++ b/Makefile
@@ -156,7 +156,7 @@ list:
 	    done \
 	} | column -t -s $$'\t'
 
-.PHONY: install clangtest armtest usan asan uasan
+.PHONY: install armtest usan asan uasan
 install:
 	@$(MAKE) -C $(ZSTDDIR) $@
 	@$(MAKE) -C $(PRGDIR) $@
@@ -188,7 +188,7 @@ gcc7build: clean
 .PHONY: clangbuild
 clangbuild: clean
 	clang -v
-	CXX=clang++ CC=clang $(MAKE) all MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation"
+	CXX=clang++ CC=clang CFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation" $(MAKE) all
 
 m32build: clean
 	gcc -v
@@ -232,10 +232,6 @@ gcc6test: clean
 	gcc-6 -v
 	$(MAKE) all CC=gcc-6 MOREFLAGS="-Werror"
 
-clangtest: clean
-	clang -v
-	$(MAKE) all CXX=clang++ CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation"
-
 armtest: clean
 	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
 	$(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static" FUZZER_FLAGS=--no-big-tests

From 4909a341db27f123d920041dcbd5e4a51aa0b38f Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sat, 19 Jan 2019 23:44:09 -0800
Subject: [PATCH 022/178] added clang test in travis

---
 .travis.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index b3a654378d8..64e01f8b4d8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -30,12 +30,14 @@ matrix:
       script:
         - make test
 
-    - name: gcc-6 + gcc-7 compilation
+    - name: clang + gcc-6 + gcc-7 compilation
       script:
         - make gcc6install gcc7install
         - CC=gcc-6 CFLAGS=-Werror make -j all
         - make clean
         - CC=gcc-7 CFLAGS=-Werror make -j all
+        - make clean
+        - CXX=clang++ CC=clang CFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation" make all
 
     - name: gcc-8 + ASan + UBSan + Test Zstd
       script:

From 54bd39abdfdd73d89a9aa102a249abf0aa72a685 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sun, 20 Jan 2019 11:13:01 -0800
Subject: [PATCH 023/178] Revert "added clang test in travis"

This reverts commit 4909a341db27f123d920041dcbd5e4a51aa0b38f.
---
 .travis.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 64e01f8b4d8..b3a654378d8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -30,14 +30,12 @@ matrix:
       script:
         - make test
 
-    - name: clang + gcc-6 + gcc-7 compilation
+    - name: gcc-6 + gcc-7 compilation
       script:
         - make gcc6install gcc7install
         - CC=gcc-6 CFLAGS=-Werror make -j all
         - make clean
         - CC=gcc-7 CFLAGS=-Werror make -j all
-        - make clean
-        - CXX=clang++ CC=clang CFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation" make all
 
     - name: gcc-8 + ASan + UBSan + Test Zstd
       script:

From 5e220bf4b5705776a0c16877db81b4b4849d7f4a Mon Sep 17 00:00:00 2001
From: Karl Ostmo 
Date: Tue, 22 Jan 2019 17:31:13 -0800
Subject: [PATCH 024/178] Remove global parameters, pass into public functions
 instead

---
 programs/fileio.c  | 398 ++++++++++++++++++++++++++++-----------------
 programs/fileio.h  |  60 ++++---
 programs/zstdcli.c |  94 ++++++-----
 3 files changed, 329 insertions(+), 223 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 8538285bdb2..590726ea0c1 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -79,23 +79,29 @@
 /*-*************************************
 *  Macros
 ***************************************/
+
+struct FIO_display_prefs_s {
+    int displayLevel;   /* 0 : no display;  1: errors;  2: + result + interaction + warnings;  3: + progression;  4: + information */
+    U32 noProgress;
+};
+
+static FIO_display_prefs_t g_display_prefs = {2, 0};
+
 #define DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
 #define DISPLAYOUT(...)      fprintf(stdout, __VA_ARGS__)
-#define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
-static int g_displayLevel = 2;   /* 0 : no display;  1: errors;  2: + result + interaction + warnings;  3: + progression;  4: + information */
-void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
+#define DISPLAYLEVEL(l, ...) { if (g_display_prefs.displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
 
 static const U64 g_refreshRate = SEC_TO_MICRO / 6;
 static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
 
-#define READY_FOR_UPDATE() (!g_noProgress && UTIL_clockSpanMicro(g_displayClock) > g_refreshRate)
+#define READY_FOR_UPDATE() (!g_display_prefs.noProgress && UTIL_clockSpanMicro(g_displayClock) > g_refreshRate)
 #define DELAY_NEXT_UPDATE() { g_displayClock = UTIL_getTime(); }
 #define DISPLAYUPDATE(l, ...) {                              \
-        if (g_displayLevel>=l && !g_noProgress) {            \
-            if (READY_FOR_UPDATE() || (g_displayLevel>=4)) { \
+        if (g_display_prefs.displayLevel>=l && !g_display_prefs.noProgress) {            \
+            if (READY_FOR_UPDATE() || (g_display_prefs.displayLevel>=4)) { \
                 DELAY_NEXT_UPDATE();                         \
                 DISPLAY(__VA_ARGS__);                        \
-                if (g_displayLevel>=4) fflush(stderr);       \
+                if (g_display_prefs.displayLevel>=4) fflush(stderr);       \
     }   }   }
 
 #undef MIN  /* in case it would be already defined */
@@ -269,97 +275,171 @@ void FIO_addAbortHandler()
 
 
 /*-*************************************
-*  Local Parameters - Not thread safe
+*  Parameters: Typedefs
 ***************************************/
-static FIO_compressionType_t g_compressionType = FIO_zstdCompression;
-void FIO_setCompressionType(FIO_compressionType_t compressionType) { g_compressionType = compressionType; }
-static U32 g_overwrite = 0;
-void FIO_overwriteMode(void) { g_overwrite = 1; }
-static U32 g_sparseFileSupport = ZSTD_SPARSE_DEFAULT;   /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
-void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport = sparse; }
-static U32 g_dictIDFlag = 1;
-void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; }
-static U32 g_checksumFlag = 1;
-void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; }
-static U32 g_removeSrcFile = 0;
-void FIO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); }
-static unsigned g_memLimit = 0;
-void FIO_setMemLimit(unsigned memLimit) { g_memLimit = memLimit; }
-static unsigned g_nbWorkers = 1;
-void FIO_setNbWorkers(unsigned nbWorkers) {
+
+struct FIO_prefs_s {
+
+    /* Algorithm preferences */
+    FIO_compressionType_t compressionType;
+    U32 sparseFileSupport;   /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
+    U32 dictIDFlag;
+    U32 checksumFlag;
+    U32 blockSize;
+    unsigned overlapLog;
+    U32 adaptiveMode;
+    U32 rsyncable;
+    int minAdaptLevel;
+    int maxAdaptLevel;
+    U32 ldmFlag;
+    U32 ldmHashLog;
+    U32 ldmMinMatch;
+    U32 ldmBucketSizeLog;
+    U32 ldmHashRateLog;
+
+    /* IO preferences */
+    U32 removeSrcFile;
+    U32 overwrite;
+
+    /* Computation resources preferences */
+    unsigned memLimit;
+    unsigned nbWorkers;
+};
+
+
+/*-*************************************
+*  Parameters: Initialization
+***************************************/
+
+#define FIO_OVERLAP_LOG_NOTSET 9999
+#define FIO_LDM_PARAM_NOTSET 9999
+
+
+FIO_prefs_t* FIO_createPreferences(void)
+{
+    FIO_prefs_t* const ret = (FIO_prefs_t*)malloc(sizeof(FIO_prefs_t));
+    if (!ret) EXM_THROW(21, "Allocation error : not enough memory");
+
+    ret->compressionType = FIO_zstdCompression;
+    ret->overwrite = 0;
+    ret->sparseFileSupport = ZSTD_SPARSE_DEFAULT;
+    ret->dictIDFlag = 1;
+    ret->checksumFlag = 1;
+    ret->removeSrcFile = 0;
+    ret->memLimit = 0;
+    ret->nbWorkers = 1;
+    ret->blockSize = 0;
+    ret->overlapLog = FIO_OVERLAP_LOG_NOTSET;
+    ret->adaptiveMode = 0;
+    ret->rsyncable = 0;
+    ret->minAdaptLevel = -50;   /* initializing this value requires a constant, so ZSTD_minCLevel() doesn't work */
+    ret->maxAdaptLevel = 22;   /* initializing this value requires a constant, so ZSTD_maxCLevel() doesn't work */
+    ret->ldmFlag = 0;
+    ret->ldmHashLog = 0;
+    ret->ldmMinMatch = 0;
+    ret->ldmBucketSizeLog = FIO_LDM_PARAM_NOTSET;
+    ret->ldmHashRateLog = FIO_LDM_PARAM_NOTSET;
+    return ret;
+}
+
+void FIO_freePreferences(FIO_prefs_t* const prefs)
+{
+    free(prefs);
+}
+
+
+/*-*************************************
+*  Parameters: Display Options
+***************************************/
+
+void FIO_setNotificationLevel(unsigned level) { g_display_prefs.displayLevel=level; }
+
+void FIO_setNoProgress(unsigned noProgress) { g_display_prefs.noProgress = noProgress; }
+
+
+/*-*************************************
+*  Parameters: Setters
+***************************************/
+
+void FIO_setCompressionType(FIO_prefs_t* const prefs, FIO_compressionType_t compressionType) { prefs->compressionType = compressionType; }
+
+void FIO_overwriteMode(FIO_prefs_t* const prefs) { prefs->overwrite = 1; }
+
+void FIO_setSparseWrite(FIO_prefs_t* const prefs, unsigned sparse) { prefs->sparseFileSupport = sparse; }
+
+void FIO_setDictIDFlag(FIO_prefs_t* const prefs, unsigned dictIDFlag) { prefs->dictIDFlag = dictIDFlag; }
+
+void FIO_setChecksumFlag(FIO_prefs_t* const prefs, unsigned checksumFlag) { prefs->checksumFlag = checksumFlag; }
+
+void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, unsigned flag) { prefs->removeSrcFile = (flag>0); }
+
+void FIO_setMemLimit(FIO_prefs_t* const prefs, unsigned memLimit) { prefs->memLimit = memLimit; }
+
+void FIO_setNbWorkers(FIO_prefs_t* const prefs, unsigned nbWorkers) {
 #ifndef ZSTD_MULTITHREAD
     if (nbWorkers > 0) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n");
 #endif
-    g_nbWorkers = nbWorkers;
+    prefs->nbWorkers = nbWorkers;
 }
-static U32 g_blockSize = 0;
-void FIO_setBlockSize(unsigned blockSize) {
-    if (blockSize && g_nbWorkers==0)
+
+void FIO_setBlockSize(FIO_prefs_t* const prefs, unsigned blockSize) {
+    if (blockSize && prefs->nbWorkers==0)
         DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n");
-    g_blockSize = blockSize;
+    prefs->blockSize = blockSize;
 }
-#define FIO_OVERLAP_LOG_NOTSET 9999
-static unsigned g_overlapLog = FIO_OVERLAP_LOG_NOTSET;
-void FIO_setOverlapLog(unsigned overlapLog){
-    if (overlapLog && g_nbWorkers==0)
+
+void FIO_setOverlapLog(FIO_prefs_t* const prefs, unsigned overlapLog){
+    if (overlapLog && prefs->nbWorkers==0)
         DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n");
-    g_overlapLog = overlapLog;
+    prefs->overlapLog = overlapLog;
 }
-static U32 g_adaptiveMode = 0;
-void FIO_setAdaptiveMode(unsigned adapt) {
-    if ((adapt>0) && (g_nbWorkers==0))
+
+void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, unsigned adapt) {
+    if ((adapt>0) && (prefs->nbWorkers==0))
         EXM_THROW(1, "Adaptive mode is not compatible with single thread mode \n");
-    g_adaptiveMode = adapt;
+    prefs->adaptiveMode = adapt;
 }
-static U32 g_rsyncable = 0;
-void FIO_setRsyncable(unsigned rsyncable) {
-    if ((rsyncable>0) && (g_nbWorkers==0))
+
+void FIO_setRsyncable(FIO_prefs_t* const prefs, unsigned rsyncable) {
+    if ((rsyncable>0) && (prefs->nbWorkers==0))
         EXM_THROW(1, "Rsyncable mode is not compatible with single thread mode \n");
-    g_rsyncable = rsyncable;
+    prefs->rsyncable = rsyncable;
 }
-static int g_minAdaptLevel = -50;   /* initializing this value requires a constant, so ZSTD_minCLevel() doesn't work */
-void FIO_setAdaptMin(int minCLevel)
+
+void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel)
 {
 #ifndef ZSTD_NOCOMPRESS
     assert(minCLevel >= ZSTD_minCLevel());
 #endif
-    g_minAdaptLevel = minCLevel;
+    prefs->minAdaptLevel = minCLevel;
 }
-static int g_maxAdaptLevel = 22;   /* initializing this value requires a constant, so ZSTD_maxCLevel() doesn't work */
-void FIO_setAdaptMax(int maxCLevel)
+
+void FIO_setAdaptMax(FIO_prefs_t* const prefs, int maxCLevel)
 {
-    g_maxAdaptLevel = maxCLevel;
+    prefs->maxAdaptLevel = maxCLevel;
 }
 
-static U32 g_ldmFlag = 0;
-void FIO_setLdmFlag(unsigned ldmFlag) {
-    g_ldmFlag = (ldmFlag>0);
-}
-static U32 g_ldmHashLog = 0;
-void FIO_setLdmHashLog(unsigned ldmHashLog) {
-    g_ldmHashLog = ldmHashLog;
-}
-static U32 g_ldmMinMatch = 0;
-void FIO_setLdmMinMatch(unsigned ldmMinMatch) {
-    g_ldmMinMatch = ldmMinMatch;
+void FIO_setLdmFlag(FIO_prefs_t* const prefs, unsigned ldmFlag) {
+    prefs->ldmFlag = (ldmFlag>0);
 }
 
-#define FIO_LDM_PARAM_NOTSET 9999
-static U32 g_ldmBucketSizeLog = FIO_LDM_PARAM_NOTSET;
-void FIO_setLdmBucketSizeLog(unsigned ldmBucketSizeLog) {
-    g_ldmBucketSizeLog = ldmBucketSizeLog;
+void FIO_setLdmHashLog(FIO_prefs_t* const prefs, unsigned ldmHashLog) {
+    prefs->ldmHashLog = ldmHashLog;
 }
 
-static U32 g_ldmHashRateLog = FIO_LDM_PARAM_NOTSET;
-void FIO_setLdmHashRateLog(unsigned ldmHashRateLog) {
-    g_ldmHashRateLog = ldmHashRateLog;
+void FIO_setLdmMinMatch(FIO_prefs_t* const prefs, unsigned ldmMinMatch) {
+    prefs->ldmMinMatch = ldmMinMatch;
 }
-static U32 g_noProgress = 0;
-void FIO_setNoProgress(unsigned noProgress) {
-    g_noProgress = noProgress;
+
+void FIO_setLdmBucketSizeLog(FIO_prefs_t* const prefs, unsigned ldmBucketSizeLog) {
+    prefs->ldmBucketSizeLog = ldmBucketSizeLog;
 }
 
 
+void FIO_setLdmHashRateLog(FIO_prefs_t* const prefs, unsigned ldmHashRateLog) {
+    prefs->ldmHashRateLog = ldmHashRateLog;
+}
+
 
 /*-*************************************
 *  Functions
@@ -414,14 +494,14 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
 /** FIO_openDstFile() :
  *  condition : `dstFileName` must be non-NULL.
  * @result : FILE* to `dstFileName`, or NULL if it fails */
-static FILE* FIO_openDstFile(const char* srcFileName, const char* dstFileName)
+static FILE* FIO_openDstFile(FIO_prefs_t* const prefs, const char* srcFileName, const char* dstFileName)
 {
     assert(dstFileName != NULL);
     if (!strcmp (dstFileName, stdoutmark)) {
         DISPLAYLEVEL(4,"Using stdout for output \n");
         SET_BINARY_MODE(stdout);
-        if (g_sparseFileSupport == 1) {
-            g_sparseFileSupport = 0;
+        if (prefs->sparseFileSupport == 1) {
+            prefs->sparseFileSupport = 0;
             DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n");
         }
         return stdout;
@@ -451,8 +531,8 @@ static FILE* FIO_openDstFile(const char* srcFileName, const char* dstFileName)
 #endif
     }
 
-    if (g_sparseFileSupport == 1) {
-        g_sparseFileSupport = ZSTD_SPARSE_DEFAULT;
+    if (prefs->sparseFileSupport == 1) {
+        prefs->sparseFileSupport = ZSTD_SPARSE_DEFAULT;
     }
 
     if (UTIL_isRegularFile(dstFileName)) {
@@ -464,8 +544,8 @@ static FILE* FIO_openDstFile(const char* srcFileName, const char* dstFileName)
         }
         if (fCheck != NULL) {  /* dst file exists, authorization prompt */
             fclose(fCheck);
-            if (!g_overwrite) {
-                if (g_displayLevel <= 1) {
+            if (!prefs->overwrite) {
+                if (g_display_prefs.displayLevel <= 1) {
                     /* No interaction possible */
                     DISPLAY("zstd: %s already exists; not overwritten  \n",
                             dstFileName);
@@ -543,7 +623,8 @@ typedef struct {
     ZSTD_CStream* cctx;
 } cRess_t;
 
-static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
+static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
+                                    const char* dictFileName, int cLevel,
                                     U64 srcSize,
                                     ZSTD_compressionParameters comprParams) {
     cRess_t ress;
@@ -567,23 +648,23 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
         if (dictFileName && (dictBuffer==NULL))
             EXM_THROW(32, "allocation error : can't create dictBuffer");
 
-        if (g_adaptiveMode && !g_ldmFlag && !comprParams.windowLog)
+        if (prefs->adaptiveMode && !prefs->ldmFlag && !comprParams.windowLog)
             comprParams.windowLog = ADAPT_WINDOWLOG_DEFAULT;
 
         CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_contentSizeFlag, 1) );  /* always enable content size when available (note: supposed to be default) */
-        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_dictIDFlag, g_dictIDFlag) );
-        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_checksumFlag, g_checksumFlag) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_dictIDFlag, prefs->dictIDFlag) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_checksumFlag, prefs->checksumFlag) );
         /* compression level */
         CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, cLevel) );
         /* long distance matching */
-        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_enableLongDistanceMatching, g_ldmFlag) );
-        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashLog, g_ldmHashLog) );
-        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmMinMatch, g_ldmMinMatch) );
-        if (g_ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) {
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmBucketSizeLog, g_ldmBucketSizeLog) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_enableLongDistanceMatching, prefs->ldmFlag) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashLog, prefs->ldmHashLog) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmMinMatch, prefs->ldmMinMatch) );
+        if (prefs->ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) {
+            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmBucketSizeLog, prefs->ldmBucketSizeLog) );
         }
-        if (g_ldmHashRateLog != FIO_LDM_PARAM_NOTSET) {
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashRateLog, g_ldmHashRateLog) );
+        if (prefs->ldmHashRateLog != FIO_LDM_PARAM_NOTSET) {
+            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashRateLog, prefs->ldmHashRateLog) );
         }
         /* compression parameters */
         CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_windowLog, comprParams.windowLog) );
@@ -595,14 +676,14 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
         CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_strategy, comprParams.strategy) );
         /* multi-threading */
 #ifdef ZSTD_MULTITHREAD
-        DISPLAYLEVEL(5,"set nb workers = %u \n", g_nbWorkers);
-        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_nbWorkers, g_nbWorkers) );
-        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, g_blockSize) );
-        if (g_overlapLog != FIO_OVERLAP_LOG_NOTSET) {
-            DISPLAYLEVEL(3,"set overlapLog = %u \n", g_overlapLog);
-            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_overlapLog, g_overlapLog) );
+        DISPLAYLEVEL(5,"set nb workers = %u \n", prefs->nbWorkers);
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_nbWorkers, prefs->nbWorkers) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, prefs->blockSize) );
+        if (prefs->overlapLog != FIO_OVERLAP_LOG_NOTSET) {
+            DISPLAYLEVEL(3,"set overlapLog = %u \n", prefs->overlapLog);
+            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_overlapLog, prefs->overlapLog) );
         }
-        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_rsyncable, g_rsyncable) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_rsyncable, prefs->rsyncable) );
 #endif
         /* dictionary */
         CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) );  /* set the value temporarily for dictionary loading, to adapt compression parameters */
@@ -880,7 +961,8 @@ FIO_compressLz4Frame(cRess_t* ress,
 
 
 static unsigned long long
-FIO_compressZstdFrame(const cRess_t* ressPtr,
+FIO_compressZstdFrame(FIO_prefs_t* const prefs,
+                      const cRess_t* ressPtr,
                       const char* srcFileName, U64 fileSize,
                       int compressionLevel, U64* readsize)
 {
@@ -951,7 +1033,7 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
                 double const cShare = (double)zfp.produced / (zfp.consumed + !zfp.consumed/*avoid div0*/) * 100;
 
                 /* display progress notifications */
-                if (g_displayLevel >= 3) {
+                if (g_display_prefs.displayLevel >= 3) {
                     DISPLAYUPDATE(3, "\r(L%i) Buffered :%4u MB - Consumed :%4u MB - Compressed :%4u MB => %.2f%% ",
                                 compressionLevel,
                                 (unsigned)((zfp.ingested - zfp.consumed) >> 20),
@@ -967,7 +1049,7 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
                 }
 
                 /* adaptive mode : statistics measurement and speed correction */
-                if (g_adaptiveMode) {
+                if (prefs->adaptiveMode) {
 
                     /* check output speed */
                     if (zfp.currentJobID > 1) {  /* only possible if nbWorkers >= 1 */
@@ -975,7 +1057,7 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
                         unsigned long long newlyProduced = zfp.produced - previous_zfp_update.produced;
                         unsigned long long newlyFlushed = zfp.flushed - previous_zfp_update.flushed;
                         assert(zfp.produced >= previous_zfp_update.produced);
-                        assert(g_nbWorkers >= 1);
+                        assert(prefs->nbWorkers >= 1);
 
                         /* test if compression is blocked
                          * either because output is slow and all buffers are full
@@ -1004,7 +1086,7 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
                         DISPLAYLEVEL(6, "compression level adaptation check \n")
 
                         /* check input speed */
-                        if (zfp.currentJobID > g_nbWorkers+1) {   /* warm up period, to fill all workers */
+                        if (zfp.currentJobID > prefs->nbWorkers+1) {   /* warm up period, to fill all workers */
                             if (inputBlocked <= 0) {
                                 DISPLAYLEVEL(6, "input is never blocked => input is slower than ingestion \n");
                                 speedChange = slower;
@@ -1036,14 +1118,14 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
                             DISPLAYLEVEL(6, "slower speed , higher compression \n")
                             compressionLevel ++;
                             if (compressionLevel > ZSTD_maxCLevel()) compressionLevel = ZSTD_maxCLevel();
-                            if (compressionLevel > g_maxAdaptLevel) compressionLevel = g_maxAdaptLevel;
+                            if (compressionLevel > prefs->maxAdaptLevel) compressionLevel = prefs->maxAdaptLevel;
                             compressionLevel += (compressionLevel == 0);   /* skip 0 */
                             ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel);
                         }
                         if (speedChange == faster) {
                             DISPLAYLEVEL(6, "faster speed , lighter compression \n")
                             compressionLevel --;
-                            if (compressionLevel < g_minAdaptLevel) compressionLevel = g_minAdaptLevel;
+                            if (compressionLevel < prefs->minAdaptLevel) compressionLevel = prefs->minAdaptLevel;
                             compressionLevel -= (compressionLevel == 0);   /* skip 0 */
                             ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel);
                         }
@@ -1073,7 +1155,8 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
  *            1 : missing or pb opening srcFileName
  */
 static int
-FIO_compressFilename_internal(cRess_t ress,
+FIO_compressFilename_internal(FIO_prefs_t* const prefs,
+                              cRess_t ress,
                               const char* dstFileName, const char* srcFileName,
                               int compressionLevel)
 {
@@ -1083,10 +1166,10 @@ FIO_compressFilename_internal(cRess_t ress,
     DISPLAYLEVEL(5, "%s: %u bytes \n", srcFileName, (unsigned)fileSize);
 
     /* compression format selection */
-    switch (g_compressionType) {
+    switch (prefs->compressionType) {
         default:
         case FIO_zstdCompression:
-            compressedfilesize = FIO_compressZstdFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize);
+            compressedfilesize = FIO_compressZstdFrame(prefs, &ress, srcFileName, fileSize, compressionLevel, &readsize);
             break;
 
         case FIO_gzipCompression:
@@ -1102,7 +1185,7 @@ FIO_compressFilename_internal(cRess_t ress,
         case FIO_xzCompression:
         case FIO_lzmaCompression:
 #ifdef ZSTD_LZMACOMPRESS
-            compressedfilesize = FIO_compressLzmaFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize, g_compressionType==FIO_lzmaCompression);
+            compressedfilesize = FIO_compressLzmaFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize, prefs->compressionType==FIO_lzmaCompression);
 #else
             (void)compressionLevel;
             EXM_THROW(20, "zstd: %s: file cannot be compressed as xz/lzma (zstd compiled without ZSTD_LZMACOMPRESS) -- ignored \n",
@@ -1142,7 +1225,8 @@ FIO_compressFilename_internal(cRess_t ress,
  *  @return : 0 : compression completed correctly,
  *            1 : pb
  */
-static int FIO_compressFilename_dstFile(cRess_t ress,
+static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs,
+                                        cRess_t ress,
                                         const char* dstFileName,
                                         const char* srcFileName,
                                         int compressionLevel)
@@ -1157,7 +1241,7 @@ static int FIO_compressFilename_dstFile(cRess_t ress,
     if (ress.dstFile == NULL) {
         closeDstFile = 1;
         DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s", dstFileName);
-        ress.dstFile = FIO_openDstFile(srcFileName, dstFileName);
+        ress.dstFile = FIO_openDstFile(prefs, srcFileName, dstFileName);
         if (ress.dstFile==NULL) return 1;  /* could not open dstFileName */
         /* Must only be added after FIO_openDstFile() succeeds.
          * Otherwise we may delete the destination file if it already exists,
@@ -1170,7 +1254,7 @@ static int FIO_compressFilename_dstFile(cRess_t ress,
             transfer_permissions = 1;
     }
 
-    result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, compressionLevel);
+    result = FIO_compressFilename_internal(prefs, ress, dstFileName, srcFileName, compressionLevel);
 
     if (closeDstFile) {
         FILE* const dstFile = ress.dstFile;
@@ -1203,7 +1287,8 @@ static int FIO_compressFilename_dstFile(cRess_t ress,
  *            1 : missing or pb opening srcFileName
  */
 static int
-FIO_compressFilename_srcFile(cRess_t ress,
+FIO_compressFilename_srcFile(FIO_prefs_t* const prefs,
+                             cRess_t ress,
                              const char* dstFileName,
                              const char* srcFileName,
                              int compressionLevel)
@@ -1219,11 +1304,11 @@ FIO_compressFilename_srcFile(cRess_t ress,
     ress.srcFile = FIO_openSrcFile(srcFileName);
     if (ress.srcFile == NULL) return 1;   /* srcFile could not be opened */
 
-    result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName, compressionLevel);
+    result = FIO_compressFilename_dstFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
 
     fclose(ress.srcFile);
     ress.srcFile = NULL;
-    if ( g_removeSrcFile   /* --rm */
+    if ( prefs->removeSrcFile   /* --rm */
       && result == 0       /* success */
       && strcmp(srcFileName, stdinmark)   /* exception : don't erase stdin */
       ) {
@@ -1238,7 +1323,8 @@ FIO_compressFilename_srcFile(cRess_t ress,
 }
 
 
-int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
+int FIO_compressFilename(FIO_prefs_t* const prefs,
+                         const char* dstFileName, const char* srcFileName,
                          const char* dictFileName, int compressionLevel,
                          ZSTD_compressionParameters comprParams)
 {
@@ -1246,8 +1332,8 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
     U64 const fileSize = UTIL_getFileSize(srcFileName);
     U64 const srcSize = (fileSize == UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : fileSize;
 
-    cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams);
-    int const result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, compressionLevel);
+    cRess_t const ress = FIO_createCResources(prefs, dictFileName, compressionLevel, srcSize, comprParams);
+    int const result = FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
 
     double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC;
     DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds);
@@ -1292,7 +1378,8 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix)
  * into one destination (outFileName)
  * or into one file each (outFileName == NULL, but suffix != NULL).
  */
-int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles,
+int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
+                                  const char** inFileNamesTable, unsigned nbFiles,
                                   const char* outFileName, const char* suffix,
                                   const char* dictFileName, int compressionLevel,
                                   ZSTD_compressionParameters comprParams)
@@ -1301,19 +1388,19 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
     U64 const firstFileSize = UTIL_getFileSize(inFileNamesTable[0]);
     U64 const firstSrcSize = (firstFileSize == UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : firstFileSize;
     U64 const srcSize = (nbFiles != 1) ? ZSTD_CONTENTSIZE_UNKNOWN : firstSrcSize ;
-    cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams);
+    cRess_t ress = FIO_createCResources(prefs, dictFileName, compressionLevel, srcSize, comprParams);
 
     /* init */
     assert(outFileName != NULL || suffix != NULL);
 
     if (outFileName != NULL) {   /* output into a single destination (stdout typically) */
-        ress.dstFile = FIO_openDstFile(NULL, outFileName);
+        ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName);
         if (ress.dstFile == NULL) {  /* could not open outFileName */
             error = 1;
         } else {
             unsigned u;
             for (u=0; umemLimit) );
     ress.srcBufferSize = ZSTD_DStreamInSize();
     ress.srcBuffer = malloc(ress.srcBufferSize);
     ress.dstBufferSize = ZSTD_DStreamOutSize();
@@ -1387,7 +1474,7 @@ static void FIO_freeDResources(dRess_t ress)
 
 /** FIO_fwriteSparse() :
 *   @return : storedSkips, to be provided to next call to FIO_fwriteSparse() of LZ4IO_fwriteSparseEnd() */
-static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips)
+static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips)
 {
     const size_t* const bufferT = (const size_t*)buffer;   /* Buffer is supposed malloc'ed, hence aligned on size_t */
     size_t bufferSizeT = bufferSize / sizeof(size_t);
@@ -1395,7 +1482,7 @@ static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSi
     const size_t* ptrT = bufferT;
     static const size_t segmentSizeT = (32 KB) / sizeof(size_t);   /* 0-test re-attempted every 32 KB */
 
-    if (!g_sparseFileSupport) {  /* normal write */
+    if (!prefs->sparseFileSupport) {  /* normal write */
         size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file);
         if (sizeCheck != bufferSize)
             EXM_THROW(70, "Write error : %s (cannot write decoded block)",
@@ -1456,10 +1543,10 @@ static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSi
     return storedSkips;
 }
 
-static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
+static void FIO_fwriteSparseEnd(FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips)
 {
     if (storedSkips>0) {
-        assert(g_sparseFileSupport > 0);  /* storedSkips>0 implies sparse support is enabled */
+        assert(prefs->sparseFileSupport > 0);  /* storedSkips>0 implies sparse support is enabled */
         if (LONG_SEEK(file, storedSkips-1, SEEK_CUR) != 0)
             EXM_THROW(69, "Final skip error (sparse file support)");
         /* last zero must be explicitly written,
@@ -1473,7 +1560,7 @@ static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
 
 /** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode
     @return : 0 (no error) */
-static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_t bufferSize, size_t alreadyLoaded)
+static unsigned FIO_passThrough(FIO_prefs_t* const prefs, FILE* foutput, FILE* finput, void* buffer, size_t bufferSize, size_t alreadyLoaded)
 {
     size_t const blockSize = MIN(64 KB, bufferSize);
     size_t readFromInput = 1;
@@ -1488,10 +1575,10 @@ static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_
 
     while (readFromInput) {
         readFromInput = fread(buffer, 1, blockSize, finput);
-        storedSkips = FIO_fwriteSparse(foutput, buffer, readFromInput, storedSkips);
+        storedSkips = FIO_fwriteSparse(prefs, foutput, buffer, readFromInput, storedSkips);
     }
 
-    FIO_fwriteSparseEnd(foutput, storedSkips);
+    FIO_fwriteSparseEnd(prefs, foutput, storedSkips);
     return 0;
 }
 
@@ -1510,7 +1597,7 @@ static unsigned FIO_highbit64(unsigned long long v)
 
 /* FIO_zstdErrorHelp() :
  * detailed error message when requested window size is too large */
-static void FIO_zstdErrorHelp(dRess_t* ress, size_t err, char const* srcFileName)
+static void FIO_zstdErrorHelp(FIO_prefs_t* const prefs, dRess_t* ress, size_t err, char const* srcFileName)
 {
     ZSTD_frameHeader header;
 
@@ -1523,9 +1610,9 @@ static void FIO_zstdErrorHelp(dRess_t* ress, size_t err, char const* srcFileName
     if (err == 0) {
         unsigned long long const windowSize = header.windowSize;
         unsigned const windowLog = FIO_highbit64(windowSize) + ((windowSize & (windowSize - 1)) != 0);
-        assert(g_memLimit > 0);
+        assert(prefs->memLimit > 0);
         DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u\n",
-                        srcFileName, windowSize, g_memLimit);
+                        srcFileName, windowSize, prefs->memLimit);
         if (windowLog <= ZSTD_WINDOWLOG_MAX) {
             unsigned const windowMB = (unsigned)((windowSize >> 20) + ((windowSize & ((1 MB) - 1)) != 0));
             assert(windowSize < (U64)(1ULL << 52));   /* ensure now overflow for windowMB */
@@ -1542,7 +1629,9 @@ static void FIO_zstdErrorHelp(dRess_t* ress, size_t err, char const* srcFileName
  *  @return : size of decoded zstd frame, or an error code
 */
 #define FIO_ERROR_FRAME_DECODING   ((unsigned long long)(-2))
-static unsigned long long FIO_decompressZstdFrame(dRess_t* ress,
+static unsigned long long FIO_decompressZstdFrame(
+                                       FIO_prefs_t* const prefs,
+                                       dRess_t* ress,
                                        FILE* finput,
                                        const char* srcFileName,
                                        U64 alreadyDecoded)
@@ -1571,12 +1660,12 @@ static unsigned long long FIO_decompressZstdFrame(dRess_t* ress,
         if (ZSTD_isError(readSizeHint)) {
             DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n",
                             srcFileName, ZSTD_getErrorName(readSizeHint));
-            FIO_zstdErrorHelp(ress, readSizeHint, srcFileName);
+            FIO_zstdErrorHelp(prefs, ress, readSizeHint, srcFileName);
             return FIO_ERROR_FRAME_DECODING;
         }
 
         /* Write block */
-        storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, storedSkips);
+        storedSkips = FIO_fwriteSparse(prefs, ress->dstFile, ress->dstBuffer, outBuff.pos, storedSkips);
         frameSize += outBuff.pos;
         DISPLAYUPDATE(2, "\r%-20.20s : %u MB...     ",
                          srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) );
@@ -1607,7 +1696,7 @@ static unsigned long long FIO_decompressZstdFrame(dRess_t* ress,
                 ress->srcBufferLoaded += readSize;
     }   }   }
 
-    FIO_fwriteSparseEnd(ress->dstFile, storedSkips);
+    FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips);
 
     return frameSize;
 }
@@ -1838,7 +1927,7 @@ static unsigned long long FIO_decompressLz4Frame(dRess_t* ress,
  * @return : 0 : OK
  *           1 : error
  */
-static int FIO_decompressFrames(dRess_t ress, FILE* srcFile,
+static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* srcFile,
                         const char* dstFileName, const char* srcFileName)
 {
     unsigned readSomething = 0;
@@ -1866,7 +1955,7 @@ static int FIO_decompressFrames(dRess_t ress, FILE* srcFile,
             return 1;
         }
         if (ZSTD_isFrame(buf, ress.srcBufferLoaded)) {
-            unsigned long long const frameSize = FIO_decompressZstdFrame(&ress, srcFile, srcFileName, filesize);
+            unsigned long long const frameSize = FIO_decompressZstdFrame(prefs, &ress, srcFile, srcFileName, filesize);
             if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
             filesize += frameSize;
         } else if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */
@@ -1897,8 +1986,8 @@ static int FIO_decompressFrames(dRess_t ress, FILE* srcFile,
             DISPLAYLEVEL(1, "zstd: %s: lz4 file cannot be uncompressed (zstd compiled without HAVE_LZ4) -- ignored \n", srcFileName);
             return 1;
 #endif
-        } else if ((g_overwrite) && !strcmp (dstFileName, stdoutmark)) {  /* pass-through mode */
-            return FIO_passThrough(ress.dstFile, srcFile,
+        } else if ((prefs->overwrite) && !strcmp (dstFileName, stdoutmark)) {  /* pass-through mode */
+            return FIO_passThrough(prefs, ress.dstFile, srcFile,
                                    ress.srcBuffer, ress.srcBufferSize, ress.srcBufferLoaded);
         } else {
             DISPLAYLEVEL(1, "zstd: %s: unsupported format \n", srcFileName);
@@ -1919,7 +2008,8 @@ static int FIO_decompressFrames(dRess_t ress, FILE* srcFile,
     @return : 0 : OK
               1 : operation aborted
 */
-static int FIO_decompressDstFile(dRess_t ress, FILE* srcFile,
+static int FIO_decompressDstFile(FIO_prefs_t* const prefs,
+                                 dRess_t ress, FILE* srcFile,
                                  const char* dstFileName, const char* srcFileName)
 {
     int result;
@@ -1930,7 +2020,7 @@ static int FIO_decompressDstFile(dRess_t ress, FILE* srcFile,
     if (ress.dstFile == NULL) {
         releaseDstFile = 1;
 
-        ress.dstFile = FIO_openDstFile(srcFileName, dstFileName);
+        ress.dstFile = FIO_openDstFile(prefs, srcFileName, dstFileName);
         if (ress.dstFile==0) return 1;
 
         /* Must only be added after FIO_openDstFile() succeeds.
@@ -1945,7 +2035,7 @@ static int FIO_decompressDstFile(dRess_t ress, FILE* srcFile,
     }
 
 
-    result = FIO_decompressFrames(ress, srcFile, dstFileName, srcFileName);
+    result = FIO_decompressFrames(prefs, ress, srcFile, dstFileName, srcFileName);
 
     if (releaseDstFile) {
         FILE* const dstFile = ress.dstFile;
@@ -1978,7 +2068,7 @@ static int FIO_decompressDstFile(dRess_t ress, FILE* srcFile,
     @return : 0 : OK
               1 : error
 */
-static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const char* srcFileName)
+static int FIO_decompressSrcFile(FIO_prefs_t* const prefs, dRess_t ress, const char* dstFileName, const char* srcFileName)
 {
     FILE* srcFile;
     int result;
@@ -1992,14 +2082,14 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
     if (srcFile==NULL) return 1;
     ress.srcBufferLoaded = 0;
 
-    result = FIO_decompressDstFile(ress, srcFile, dstFileName, srcFileName);
+    result = FIO_decompressDstFile(prefs, ress, srcFile, dstFileName, srcFileName);
 
     /* Close file */
     if (fclose(srcFile)) {
         DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));  /* error should not happen */
         return 1;
     }
-    if ( g_removeSrcFile  /* --rm */
+    if ( prefs->removeSrcFile  /* --rm */
       && (result==0)      /* decompression successful */
       && strcmp(srcFileName, stdinmark) ) /* not stdin */ {
         /* We must clear the handler, since after this point calling it would
@@ -2016,12 +2106,13 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
 
 
 
-int FIO_decompressFilename(const char* dstFileName, const char* srcFileName,
+int FIO_decompressFilename(FIO_prefs_t* const prefs,
+                           const char* dstFileName, const char* srcFileName,
                            const char* dictFileName)
 {
-    dRess_t const ress = FIO_createDResources(dictFileName);
+    dRess_t const ress = FIO_createDResources(prefs, dictFileName);
 
-    int const decodingError = FIO_decompressSrcFile(ress, dstFileName, srcFileName);
+    int const decodingError = FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName);
 
     FIO_freeDResources(ress);
     return decodingError;
@@ -2098,19 +2189,20 @@ FIO_determineDstName(const char* srcFileName)
 
 
 int
-FIO_decompressMultipleFilenames(const char* srcNamesTable[], unsigned nbFiles,
+FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
+                                const char* srcNamesTable[], unsigned nbFiles,
                                 const char* outFileName,
                                 const char* dictFileName)
 {
     int error = 0;
-    dRess_t ress = FIO_createDResources(dictFileName);
+    dRess_t ress = FIO_createDResources(prefs, dictFileName);
 
     if (outFileName) {
         unsigned u;
-        ress.dstFile = FIO_openDstFile(NULL, outFileName);
+        ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName);
         if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", outFileName);
         for (u=0; u use stdin and stdout */
@@ -1092,25 +1094,25 @@ int main(int argCount, const char* argv[])
     FIO_setNotificationLevel(g_displayLevel);
     if (operation==zom_compress) {
 #ifndef ZSTD_NOCOMPRESS
-        FIO_setNbWorkers(nbWorkers);
-        FIO_setBlockSize((U32)blockSize);
-        if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(g_overlapLog);
-        FIO_setLdmFlag(ldmFlag);
-        FIO_setLdmHashLog(g_ldmHashLog);
-        FIO_setLdmMinMatch(g_ldmMinMatch);
-        if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(g_ldmBucketSizeLog);
-        if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(g_ldmHashRateLog);
-        FIO_setAdaptiveMode(adapt);
-        FIO_setAdaptMin(adaptMin);
-        FIO_setAdaptMax(adaptMax);
-        FIO_setRsyncable(rsyncable);
+        FIO_setNbWorkers(prefs, nbWorkers);
+        FIO_setBlockSize(prefs, (U32)blockSize);
+        if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, g_overlapLog);
+        FIO_setLdmFlag(prefs, ldmFlag);
+        FIO_setLdmHashLog(prefs, g_ldmHashLog);
+        FIO_setLdmMinMatch(prefs, g_ldmMinMatch);
+        if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, g_ldmBucketSizeLog);
+        if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, g_ldmHashRateLog);
+        FIO_setAdaptiveMode(prefs, adapt);
+        FIO_setAdaptMin(prefs, adaptMin);
+        FIO_setAdaptMax(prefs, adaptMax);
+        FIO_setRsyncable(prefs, rsyncable);
         if (adaptMin > cLevel) cLevel = adaptMin;
         if (adaptMax < cLevel) cLevel = adaptMax;
 
         if ((filenameIdx==1) && outFileName)
-          operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel, compressionParams);
+          operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams);
         else
-          operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams);
+          operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams);
 #else
         (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; /* not used when ZSTD_NOCOMPRESS set */
         DISPLAY("Compression not supported \n");
@@ -1124,17 +1126,19 @@ int main(int argCount, const char* argv[])
                 memLimit = (U32)1 << (compressionParams.windowLog & 31);
             }
         }
-        FIO_setMemLimit(memLimit);
+        FIO_setMemLimit(prefs, memLimit);
         if (filenameIdx==1 && outFileName)
-            operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName);
+            operationResult = FIO_decompressFilename(prefs, outFileName, filenameTable[0], dictFileName);
         else
-            operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, outFileName, dictFileName);
+            operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outFileName, dictFileName);
 #else
         DISPLAY("Decompression not supported \n");
 #endif
     }
 
 _end:
+    FIO_freePreferences(prefs);
+
     if (main_pause) waitEnter();
 #ifdef UTIL_HAS_CREATEFILELIST
     if (extendedFileList)

From 6b2f26791e3cb015f881c5646c46842cfcb2de0d Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 25 Jan 2019 12:18:33 -0800
Subject: [PATCH 025/178] updated benchfn.h code comments

---
 programs/benchfn.h | 45 +++++++++++++++++++++++----------------------
 1 file changed, 23 insertions(+), 22 deletions(-)

diff --git a/programs/benchfn.h b/programs/benchfn.h
index 3ca36e3623d..aa3c3e9391c 100644
--- a/programs/benchfn.h
+++ b/programs/benchfn.h
@@ -58,30 +58,31 @@ typedef size_t (*BMK_initFn_t)(void* initPayload);
 typedef unsigned (*BMK_errorFn_t)(size_t);
 
 
-/* BMK_benchFunction() parameters are provided through following structure.
- * This is preferable for readability,
- * as the number of parameters required is pretty large.
+/* BMK_benchFunction() parameters are provided via the following structure.
+ * A structure is preferable for readability,
+ * as the number of parameters required is fairly large.
  * No initializer is provided, because it doesn't make sense to provide some "default" :
- * all parameters should be specified by the caller */
+ * all parameters must be specified by the caller.
+ * optional parameters are labelled explicitly, and accept value NULL when not used */
 typedef struct {
-    BMK_benchFn_t benchFn;   /* the function to benchmark, over the set of blocks */
-    void* benchPayload;      /* pass custom parameters to benchFn  :
-                              * (*benchFn)(srcBuffers[i], srcSizes[i], dstBuffers[i], dstCapacities[i], benchPayload) */
-    BMK_initFn_t initFn;     /* (*initFn)(initPayload) is run once per run, at the beginning. */
-    void* initPayload;       /* Both arguments can be NULL, in which case nothing is run. */
-    BMK_errorFn_t errorFn;   /* errorFn will check each return value of benchFn over each block, to determine if it failed or not.
-                              * errorFn can be NULL, in which case no check is performed.
-                              * errorFn must return 0 when benchFn was successful, and >= 1 if it detects an error.
-                              * Execution is stopped as soon as an error is detected.
-                              * the triggering return value can be retrieved using BMK_extract_errorResult(). */
-    size_t blockCount;       /* number of blocks to operate benchFn on.
-                              * It's also the size of all array parameters :
-                              * srcBuffers, srcSizes, dstBuffers, dstCapacities, blockResults */
-    const void *const * srcBuffers; /* array of buffers to be operated on by benchFn */
-    const size_t* srcSizes;  /* array of the sizes of srcBuffers buffers */
-    void *const * dstBuffers;/* array of buffers to be written into by benchFn */
-    const size_t* dstCapacities; /* array of the capacities of dstBuffers buffers */
-    size_t* blockResults;    /* Optional: store the return value of benchFn for each block. Use NULL if this result is not requested. */
+    BMK_benchFn_t benchFn;    /* the function to benchmark, over the set of blocks */
+    void* benchPayload;       /* pass custom parameters to benchFn  :
+                               * (*benchFn)(srcBuffers[i], srcSizes[i], dstBuffers[i], dstCapacities[i], benchPayload) */
+    BMK_initFn_t initFn;      /* (*initFn)(initPayload) is run once per run, at the beginning. */
+    void* initPayload;        /* Both arguments can be NULL, in which case nothing is run. */
+    BMK_errorFn_t errorFn;    /* errorFn will check each return value of benchFn over each block, to determine if it failed or not.
+                               * errorFn can be NULL, in which case no check is performed.
+                               * errorFn must return 0 when benchFn was successful, and >= 1 if it detects an error.
+                               * Execution is stopped as soon as an error is detected.
+                               * the triggering return value can be retrieved using BMK_extract_errorResult(). */
+    size_t blockCount;        /* number of blocks to operate benchFn on.
+                               * It's also the size of all array parameters :
+                               * srcBuffers, srcSizes, dstBuffers, dstCapacities, blockResults */
+    const void *const * srcBuffers; /* read-only array of buffers to be operated on by benchFn */
+    const size_t* srcSizes;   /* read-only array containing sizes of srcBuffers */
+    void *const * dstBuffers; /* array of buffers to be written into by benchFn. This array is not optional, it must be provided even if unused by benchfn. */
+    const size_t* dstCapacities; /* read-only array containing capacities of dstBuffers. This array must be present. */
+    size_t* blockResults;     /* Optional: store the return value of benchFn for each block. Use NULL if this result is not requested. */
 } BMK_benchParams_t;
 
 

From f75ad2edcf3525597137a1c9da35e58ecd279e94 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 25 Jan 2019 14:22:25 -0800
Subject: [PATCH 026/178] added ability to create timedFnState on stack

---
 programs/benchfn.c | 19 ++++++++++++++-----
 programs/benchfn.h | 17 ++++++++++++++++-
 2 files changed, 30 insertions(+), 6 deletions(-)

diff --git a/programs/benchfn.c b/programs/benchfn.c
index f5118d0871f..a27d91a070c 100644
--- a/programs/benchfn.c
+++ b/programs/benchfn.c
@@ -148,7 +148,7 @@ BMK_runOutcome_t BMK_benchFunction(BMK_benchParams_t p,
 
         {   U64 const totalTime = UTIL_clockSpanNano(clockStart);
             BMK_runTime_t rt;
-            rt.nanoSecPerRun = totalTime / nbLoops;
+            rt.nanoSecPerRun = (double)totalTime / nbLoops;
             rt.sumOfReturn = dstSize;
             return BMK_setValid_runTime(rt);
     }   }
@@ -178,6 +178,15 @@ void BMK_freeTimedFnState(BMK_timedFnState_t* state) {
     free(state);
 }
 
+BMK_timedFnState_t* BMK_initStatic_timedFnState(void* buffer, size_t size, unsigned total_ms, unsigned run_ms)
+{
+    BMK_timedFnState_t* const r = (BMK_timedFnState_t*)buffer;
+    if (size < sizeof(struct BMK_timedFnState_s)) return NULL;
+    if ((size_t)buffer % 8) return NULL;  /* must be aligned on 8-bytes boundaries */
+    BMK_resetTimedFnState(r, total_ms, run_ms);
+    return r;
+}
+
 void BMK_resetTimedFnState(BMK_timedFnState_t* timedFnState, unsigned total_ms, unsigned run_ms)
 {
     if (!total_ms) total_ms = 1 ;
@@ -186,7 +195,7 @@ void BMK_resetTimedFnState(BMK_timedFnState_t* timedFnState, unsigned total_ms,
     timedFnState->timeSpent_ns = 0;
     timedFnState->timeBudget_ns = (U64)total_ms * TIMELOOP_NANOSEC / 1000;
     timedFnState->runBudget_ns = (U64)run_ms * TIMELOOP_NANOSEC / 1000;
-    timedFnState->fastestRun.nanoSecPerRun = (U64)(-1LL);
+    timedFnState->fastestRun.nanoSecPerRun = (double)TIMELOOP_NANOSEC * 2000000000;  /* hopefully large enough : must be larger than any potential measurement */
     timedFnState->fastestRun.sumOfReturn = (size_t)(-1LL);
     timedFnState->nbLoops = 1;
     timedFnState->coolTime = UTIL_getTime();
@@ -231,13 +240,13 @@ BMK_runOutcome_t BMK_benchTimedFn(BMK_timedFnState_t* cont,
         }
 
         {   BMK_runTime_t const newRunTime = BMK_extract_runTime(runResult);
-            U64 const loopDuration_ns = newRunTime.nanoSecPerRun * cont->nbLoops;
+            double const loopDuration_ns = newRunTime.nanoSecPerRun * cont->nbLoops;
 
-            cont->timeSpent_ns += loopDuration_ns;
+            cont->timeSpent_ns += (unsigned long long)loopDuration_ns;
 
             /* estimate nbLoops for next run to last approximately 1 second */
             if (loopDuration_ns > (runBudget_ns / 50)) {
-                U64 const fastestRun_ns = MIN(bestRunTime.nanoSecPerRun, newRunTime.nanoSecPerRun);
+                double const fastestRun_ns = MIN(bestRunTime.nanoSecPerRun, newRunTime.nanoSecPerRun);
                 cont->nbLoops = (U32)(runBudget_ns / fastestRun_ns) + 1;
             } else {
                 /* previous run was too short : blindly increase workload by x multiplier */
diff --git a/programs/benchfn.h b/programs/benchfn.h
index aa3c3e9391c..737dfa9956e 100644
--- a/programs/benchfn.h
+++ b/programs/benchfn.h
@@ -31,7 +31,7 @@ extern "C" {
 /* BMK_runTime_t: valid result return type */
 
 typedef struct {
-    unsigned long long nanoSecPerRun;  /* time per iteration (over all blocks) */
+    double nanoSecPerRun;  /* time per iteration (over all blocks) */
     size_t sumOfReturn;         /* sum of return values */
 } BMK_runTime_t;
 
@@ -160,6 +160,21 @@ void BMK_resetTimedFnState(BMK_timedFnState_t* timedFnState, unsigned total_ms,
 void BMK_freeTimedFnState(BMK_timedFnState_t* state);
 
 
+/* BMK_timedFnState_shell and BMK_initStatic_timedFnState() :
+ * Makes it possible to statically allocate a BMK_timedFnState_t on stack.
+ * BMK_timedFnState_shell is only there to allocate space,
+ * never ever access its members.
+ * BMK_timedFnState_t() actually accepts any buffer.
+ * It will check if provided buffer is large enough and is correctly aligned,
+ * and will return NULL if conditions are not respected.
+ */
+#define BMK_TIMEDFNSTATE_SIZE 56
+typedef union {
+    char never_access_space[BMK_TIMEDFNSTATE_SIZE];
+    long long alignment_enforcer;  /* must be aligned on 8-bytes boundaries */
+} BMK_timedFnState_shell;
+BMK_timedFnState_t* BMK_initStatic_timedFnState(void* buffer, size_t size, unsigned total_ms, unsigned run_ms);
+
 
 #endif   /* BENCH_FN_H_23876 */
 

From 07e047873e87713b2cf663559a909d1542b59fba Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 25 Jan 2019 14:42:44 -0800
Subject: [PATCH 027/178] fixed fileio.c compilation with LZ4 enabled

was broken by #1505.

I'm surprised it passed CI tests.
LZ4 tests are part of the "Extended" tests on Travis CI,
which are run on "master" and in "cron" jobs.

Since latest cron job did not failed,
especially this one : https://travis-ci.org/facebook/zstd/jobs/484365040
it suggests cron jobs are no longer using `dev` branch.

To be investigated
---
 programs/fileio.c | 83 ++++++++++++++++++++++++++---------------------
 programs/fileio.h | 22 ++++++-------
 2 files changed, 57 insertions(+), 48 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 590726ea0c1..dd47a1e32ed 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -195,7 +195,7 @@ static void ABRThandler(int sig) {
     const char* name;
     void* addrlist[MAX_STACK_FRAMES];
     char** symbollist;
-    U32 addrlen, i;
+    int addrlen, i;
 
     switch (sig) {
         case SIGABRT: name = "SIGABRT"; break;
@@ -283,19 +283,19 @@ struct FIO_prefs_s {
     /* Algorithm preferences */
     FIO_compressionType_t compressionType;
     U32 sparseFileSupport;   /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
-    U32 dictIDFlag;
-    U32 checksumFlag;
-    U32 blockSize;
-    unsigned overlapLog;
+    int dictIDFlag;
+    int checksumFlag;
+    int blockSize;
+    int overlapLog;
     U32 adaptiveMode;
-    U32 rsyncable;
+    int rsyncable;
     int minAdaptLevel;
     int maxAdaptLevel;
-    U32 ldmFlag;
-    U32 ldmHashLog;
-    U32 ldmMinMatch;
-    U32 ldmBucketSizeLog;
-    U32 ldmHashRateLog;
+    int ldmFlag;
+    int ldmHashLog;
+    int ldmMinMatch;
+    int ldmBucketSizeLog;
+    int ldmHashRateLog;
 
     /* IO preferences */
     U32 removeSrcFile;
@@ -303,7 +303,7 @@ struct FIO_prefs_s {
 
     /* Computation resources preferences */
     unsigned memLimit;
-    unsigned nbWorkers;
+    int nbWorkers;
 };
 
 
@@ -352,7 +352,7 @@ void FIO_freePreferences(FIO_prefs_t* const prefs)
 *  Parameters: Display Options
 ***************************************/
 
-void FIO_setNotificationLevel(unsigned level) { g_display_prefs.displayLevel=level; }
+void FIO_setNotificationLevel(int level) { g_display_prefs.displayLevel=level; }
 
 void FIO_setNoProgress(unsigned noProgress) { g_display_prefs.noProgress = noProgress; }
 
@@ -367,28 +367,28 @@ void FIO_overwriteMode(FIO_prefs_t* const prefs) { prefs->overwrite = 1; }
 
 void FIO_setSparseWrite(FIO_prefs_t* const prefs, unsigned sparse) { prefs->sparseFileSupport = sparse; }
 
-void FIO_setDictIDFlag(FIO_prefs_t* const prefs, unsigned dictIDFlag) { prefs->dictIDFlag = dictIDFlag; }
+void FIO_setDictIDFlag(FIO_prefs_t* const prefs, int dictIDFlag) { prefs->dictIDFlag = dictIDFlag; }
 
-void FIO_setChecksumFlag(FIO_prefs_t* const prefs, unsigned checksumFlag) { prefs->checksumFlag = checksumFlag; }
+void FIO_setChecksumFlag(FIO_prefs_t* const prefs, int checksumFlag) { prefs->checksumFlag = checksumFlag; }
 
 void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, unsigned flag) { prefs->removeSrcFile = (flag>0); }
 
 void FIO_setMemLimit(FIO_prefs_t* const prefs, unsigned memLimit) { prefs->memLimit = memLimit; }
 
-void FIO_setNbWorkers(FIO_prefs_t* const prefs, unsigned nbWorkers) {
+void FIO_setNbWorkers(FIO_prefs_t* const prefs, int nbWorkers) {
 #ifndef ZSTD_MULTITHREAD
     if (nbWorkers > 0) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n");
 #endif
     prefs->nbWorkers = nbWorkers;
 }
 
-void FIO_setBlockSize(FIO_prefs_t* const prefs, unsigned blockSize) {
+void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize) {
     if (blockSize && prefs->nbWorkers==0)
         DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n");
     prefs->blockSize = blockSize;
 }
 
-void FIO_setOverlapLog(FIO_prefs_t* const prefs, unsigned overlapLog){
+void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog){
     if (overlapLog && prefs->nbWorkers==0)
         DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n");
     prefs->overlapLog = overlapLog;
@@ -400,7 +400,7 @@ void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, unsigned adapt) {
     prefs->adaptiveMode = adapt;
 }
 
-void FIO_setRsyncable(FIO_prefs_t* const prefs, unsigned rsyncable) {
+void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable) {
     if ((rsyncable>0) && (prefs->nbWorkers==0))
         EXM_THROW(1, "Rsyncable mode is not compatible with single thread mode \n");
     prefs->rsyncable = rsyncable;
@@ -423,20 +423,20 @@ void FIO_setLdmFlag(FIO_prefs_t* const prefs, unsigned ldmFlag) {
     prefs->ldmFlag = (ldmFlag>0);
 }
 
-void FIO_setLdmHashLog(FIO_prefs_t* const prefs, unsigned ldmHashLog) {
+void FIO_setLdmHashLog(FIO_prefs_t* const prefs, int ldmHashLog) {
     prefs->ldmHashLog = ldmHashLog;
 }
 
-void FIO_setLdmMinMatch(FIO_prefs_t* const prefs, unsigned ldmMinMatch) {
+void FIO_setLdmMinMatch(FIO_prefs_t* const prefs, int ldmMinMatch) {
     prefs->ldmMinMatch = ldmMinMatch;
 }
 
-void FIO_setLdmBucketSizeLog(FIO_prefs_t* const prefs, unsigned ldmBucketSizeLog) {
+void FIO_setLdmBucketSizeLog(FIO_prefs_t* const prefs, int ldmBucketSizeLog) {
     prefs->ldmBucketSizeLog = ldmBucketSizeLog;
 }
 
 
-void FIO_setLdmHashRateLog(FIO_prefs_t* const prefs, unsigned ldmHashRateLog) {
+void FIO_setLdmHashRateLog(FIO_prefs_t* const prefs, int ldmHashRateLog) {
     prefs->ldmHashRateLog = ldmHashRateLog;
 }
 
@@ -667,12 +667,12 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
             CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashRateLog, prefs->ldmHashRateLog) );
         }
         /* compression parameters */
-        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_windowLog, comprParams.windowLog) );
-        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_chainLog, comprParams.chainLog) );
-        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_hashLog, comprParams.hashLog) );
-        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_searchLog, comprParams.searchLog) );
-        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_minMatch, comprParams.minMatch) );
-        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetLength, comprParams.targetLength) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_windowLog, (int)comprParams.windowLog) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_chainLog, (int)comprParams.chainLog) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_hashLog, (int)comprParams.hashLog) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_searchLog, (int)comprParams.searchLog) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_minMatch, (int)comprParams.minMatch) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetLength, (int)comprParams.targetLength) );
         CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_strategy, comprParams.strategy) );
         /* multi-threading */
 #ifdef ZSTD_MULTITHREAD
@@ -859,15 +859,19 @@ FIO_compressLzmaFrame(cRess_t* ress,
 #endif
 
 #ifdef ZSTD_LZ4COMPRESS
+
 #if LZ4_VERSION_NUMBER <= 10600
 #define LZ4F_blockLinked blockLinked
 #define LZ4F_max64KB max64KB
 #endif
+
 static int FIO_LZ4_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); }
+
 static unsigned long long
 FIO_compressLz4Frame(cRess_t* ress,
                      const char* srcFileName, U64 const srcFileSize,
-                     int compressionLevel, U64* readsize)
+                     int compressionLevel, int checksumFlag,
+                     U64* readsize)
 {
     const size_t blockSize = FIO_LZ4_GetBlockSize_FromBlockId(LZ4F_max64KB);
     unsigned long long inFileSize = 0, outFileSize = 0;
@@ -887,7 +891,7 @@ FIO_compressLz4Frame(cRess_t* ress,
     prefs.compressionLevel = compressionLevel;
     prefs.frameInfo.blockMode = LZ4F_blockLinked;
     prefs.frameInfo.blockSizeID = LZ4F_max64KB;
-    prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)g_checksumFlag;
+    prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)checksumFlag;
 #if LZ4_VERSION_NUMBER >= 10600
     prefs.frameInfo.contentSize = (srcFileSize==UTIL_FILESIZE_UNKNOWN) ? 0 : srcFileSize;
 #endif
@@ -1086,7 +1090,7 @@ FIO_compressZstdFrame(FIO_prefs_t* const prefs,
                         DISPLAYLEVEL(6, "compression level adaptation check \n")
 
                         /* check input speed */
-                        if (zfp.currentJobID > prefs->nbWorkers+1) {   /* warm up period, to fill all workers */
+                        if (zfp.currentJobID > (unsigned)(prefs->nbWorkers+1)) {   /* warm up period, to fill all workers */
                             if (inputBlocked <= 0) {
                                 DISPLAYLEVEL(6, "input is never blocked => input is slower than ingestion \n");
                                 speedChange = slower;
@@ -1195,7 +1199,7 @@ FIO_compressFilename_internal(FIO_prefs_t* const prefs,
 
         case FIO_lz4Compression:
 #ifdef ZSTD_LZ4COMPRESS
-            compressedfilesize = FIO_compressLz4Frame(&ress, srcFileName, fileSize, compressionLevel, &readsize);
+            compressedfilesize = FIO_compressLz4Frame(&ress, srcFileName, fileSize, compressionLevel, prefs->checksumFlag, &readsize);
 #else
             (void)compressionLevel;
             EXM_THROW(20, "zstd: %s: file cannot be compressed as lz4 (zstd compiled without ZSTD_LZ4COMPRESS) -- ignored \n",
@@ -1535,7 +1539,7 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
                 if (seekResult)
                     EXM_THROW(74, "Sparse skip error ; try --no-sparse");
                 storedSkips = 0;
-                {   size_t const sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file);
+                {   size_t const sizeCheck = fwrite(restPtr, 1, (size_t)(restEnd - restPtr), file);
                     if (sizeCheck != (size_t)(restEnd - restPtr))
                         EXM_THROW(75, "Write error : cannot write decoded end of block");
     }   }   }   }
@@ -1560,7 +1564,10 @@ static void FIO_fwriteSparseEnd(FIO_prefs_t* const prefs, FILE* file, unsigned s
 
 /** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode
     @return : 0 (no error) */
-static unsigned FIO_passThrough(FIO_prefs_t* const prefs, FILE* foutput, FILE* finput, void* buffer, size_t bufferSize, size_t alreadyLoaded)
+static int FIO_passThrough(FIO_prefs_t* const prefs,
+                           FILE* foutput, FILE* finput,
+                           void* buffer, size_t bufferSize,
+                           size_t alreadyLoaded)
 {
     size_t const blockSize = MIN(64 KB, bufferSize);
     size_t readFromInput = 1;
@@ -1987,8 +1994,10 @@ static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* sr
             return 1;
 #endif
         } else if ((prefs->overwrite) && !strcmp (dstFileName, stdoutmark)) {  /* pass-through mode */
-            return FIO_passThrough(prefs, ress.dstFile, srcFile,
-                                   ress.srcBuffer, ress.srcBufferSize, ress.srcBufferLoaded);
+            return FIO_passThrough(prefs,
+                                   ress.dstFile, srcFile,
+                                   ress.srcBuffer, ress.srcBufferSize,
+                                   ress.srcBufferLoaded);
         } else {
             DISPLAYLEVEL(1, "zstd: %s: unsupported format \n", srcFileName);
             return 1;
diff --git a/programs/fileio.h b/programs/fileio.h
index 8eb0404bdfb..b20570bcbc8 100644
--- a/programs/fileio.h
+++ b/programs/fileio.h
@@ -57,23 +57,23 @@ void FIO_overwriteMode(FIO_prefs_t* const prefs);
 void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, unsigned adapt);
 void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel);
 void FIO_setAdaptMax(FIO_prefs_t* const prefs, int maxCLevel);
-void FIO_setBlockSize(FIO_prefs_t* const prefs, unsigned blockSize);
-void FIO_setChecksumFlag(FIO_prefs_t* const prefs, unsigned checksumFlag);
-void FIO_setDictIDFlag(FIO_prefs_t* const prefs, unsigned dictIDFlag);
-void FIO_setLdmBucketSizeLog(FIO_prefs_t* const prefs, unsigned ldmBucketSizeLog);
+void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize);
+void FIO_setChecksumFlag(FIO_prefs_t* const prefs, int checksumFlag);
+void FIO_setDictIDFlag(FIO_prefs_t* const prefs, int dictIDFlag);
+void FIO_setLdmBucketSizeLog(FIO_prefs_t* const prefs, int ldmBucketSizeLog);
 void FIO_setLdmFlag(FIO_prefs_t* const prefs, unsigned ldmFlag);
-void FIO_setLdmHashRateLog(FIO_prefs_t* const prefs, unsigned ldmHashRateLog);
-void FIO_setLdmHashLog(FIO_prefs_t* const prefs, unsigned ldmHashLog);
-void FIO_setLdmMinMatch(FIO_prefs_t* const prefs, unsigned ldmMinMatch);
+void FIO_setLdmHashRateLog(FIO_prefs_t* const prefs, int ldmHashRateLog);
+void FIO_setLdmHashLog(FIO_prefs_t* const prefs, int ldmHashLog);
+void FIO_setLdmMinMatch(FIO_prefs_t* const prefs, int ldmMinMatch);
 void FIO_setMemLimit(FIO_prefs_t* const prefs, unsigned memLimit);
-void FIO_setNbWorkers(FIO_prefs_t* const prefs, unsigned nbWorkers);
-void FIO_setOverlapLog(FIO_prefs_t* const prefs, unsigned overlapLog);
+void FIO_setNbWorkers(FIO_prefs_t* const prefs, int nbWorkers);
+void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog);
 void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, unsigned flag);
 void FIO_setSparseWrite(FIO_prefs_t* const prefs, unsigned sparse);  /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
-void FIO_setRsyncable(FIO_prefs_t* const prefs, unsigned rsyncable);
+void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable);
 
 void FIO_setNoProgress(unsigned noProgress);
-void FIO_setNotificationLevel(unsigned level);
+void FIO_setNotificationLevel(int level);
 
 /*-*************************************
 *  Single File functions

From b8701102e08329e7b2b2ccde817d24e147c3e565 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 25 Jan 2019 15:11:50 -0800
Subject: [PATCH 028/178] fixed benchzstd to use new version of benchfn

returning a double type
---
 programs/benchzstd.c | 42 +++++++++++++++++++++++++-----------------
 programs/benchzstd.h | 22 +++++++++++-----------
 2 files changed, 36 insertions(+), 28 deletions(-)

diff --git a/programs/benchzstd.c b/programs/benchzstd.c
index 0be74a96504..09f39cef51a 100644
--- a/programs/benchzstd.c
+++ b/programs/benchzstd.c
@@ -159,9 +159,13 @@ typedef struct {
 #define MIN(a,b)    ((a) < (b) ? (a) : (b))
 #define MAX(a,b)    ((a) > (b) ? (a) : (b))
 
-static void BMK_initCCtx(ZSTD_CCtx* ctx,
-    const void* dictBuffer, size_t dictBufferSize, int cLevel,
-    const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) {
+static void
+BMK_initCCtx(ZSTD_CCtx* ctx,
+            const void* dictBuffer, size_t dictBufferSize,
+            int cLevel,
+            const ZSTD_compressionParameters* comprParams,
+            const BMK_advancedParams_t* adv)
+{
     ZSTD_CCtx_reset(ctx, ZSTD_reset_session_and_parameters);
     if (adv->nbWorkers==1) {
         CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_nbWorkers, 0));
@@ -174,12 +178,12 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx,
     CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmHashLog, adv->ldmHashLog));
     CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmBucketSizeLog, adv->ldmBucketSizeLog));
     CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmHashRateLog, adv->ldmHashRateLog));
-    CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_windowLog, comprParams->windowLog));
-    CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_hashLog, comprParams->hashLog));
-    CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_chainLog, comprParams->chainLog));
-    CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_searchLog, comprParams->searchLog));
-    CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_minMatch, comprParams->minMatch));
-    CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_targetLength, comprParams->targetLength));
+    CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_windowLog, (int)comprParams->windowLog));
+    CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_hashLog, (int)comprParams->hashLog));
+    CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_chainLog, (int)comprParams->chainLog));
+    CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_searchLog, (int)comprParams->searchLog));
+    CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_minMatch, (int)comprParams->minMatch));
+    CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_targetLength, (int)comprParams->targetLength));
     CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_strategy, comprParams->strategy));
     CHECK_Z(ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize));
 }
@@ -444,7 +448,7 @@ BMK_benchMemAdvancedNoAlloc(
                     cSize = cResult.sumOfReturn;
                     ratio = (double)srcSize / cSize;
                     {   BMK_benchResult_t newResult;
-                        newResult.cSpeed = ((U64)srcSize * TIMELOOP_NANOSEC / cResult.nanoSecPerRun);
+                        newResult.cSpeed = (U64)((double)srcSize * TIMELOOP_NANOSEC / cResult.nanoSecPerRun);
                         benchResult.cSize = cSize;
                         if (newResult.cSpeed > benchResult.cSpeed)
                             benchResult.cSpeed = newResult.cSpeed;
@@ -468,7 +472,7 @@ BMK_benchMemAdvancedNoAlloc(
                 }
 
                 {   BMK_runTime_t const dResult = BMK_extract_runTime(dOutcome);
-                    U64 const newDSpeed = (srcSize * TIMELOOP_NANOSEC / dResult.nanoSecPerRun);
+                    U64 const newDSpeed = (U64)((double)srcSize * TIMELOOP_NANOSEC / dResult.nanoSecPerRun);
                     if (newDSpeed > benchResult.dSpeed)
                         benchResult.dSpeed = newDSpeed;
                 }
@@ -505,17 +509,21 @@ BMK_benchMemAdvancedNoAlloc(
                         pos = (U32)(u - bacc);
                         bNb = pos / (128 KB);
                         DISPLAY("(sample %u, block %u, pos %u) \n", segNb, bNb, pos);
-                        if (u>5) {
-                            int n;
+                        {   size_t const lowest = (u>5) ? 5 : u;
+                            size_t n;
                             DISPLAY("origin: ");
-                            for (n=-5; n<0; n++) DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]);
+                            for (n=lowest; n>0; n--)
+                                DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u-n]);
                             DISPLAY(" :%02X:  ", ((const BYTE*)srcBuffer)[u]);
-                            for (n=1; n<3; n++) DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]);
+                            for (n=1; n<3; n++)
+                                DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]);
                             DISPLAY(" \n");
                             DISPLAY("decode: ");
-                            for (n=-5; n<0; n++) DISPLAY("%02X ", resultBuffer[u+n]);
+                            for (n=lowest; n>0; n++)
+                                DISPLAY("%02X ", resultBuffer[u-n]);
                             DISPLAY(" :%02X:  ", resultBuffer[u]);
-                            for (n=1; n<3; n++) DISPLAY("%02X ", resultBuffer[u+n]);
+                            for (n=1; n<3; n++)
+                                DISPLAY("%02X ", resultBuffer[u+n]);
                             DISPLAY(" \n");
                         }
                         break;
diff --git a/programs/benchzstd.h b/programs/benchzstd.h
index 3a8b893e7ee..fd7c54f5a54 100644
--- a/programs/benchzstd.h
+++ b/programs/benchzstd.h
@@ -105,17 +105,17 @@ typedef enum {
 } BMK_mode_t;
 
 typedef struct {
-    BMK_mode_t mode;            /* 0: all, 1: compress only 2: decode only */
-    unsigned nbSeconds;         /* default timing is in nbSeconds */
-    size_t blockSize;           /* Maximum size of each block*/
-    unsigned nbWorkers;         /* multithreading */
-    unsigned realTime;          /* real time priority */
-    int additionalParam;        /* used by python speed benchmark */
-    unsigned ldmFlag;           /* enables long distance matching */
-    unsigned ldmMinMatch;       /* below: parameters for long distance matching, see zstd.1.md */
-    unsigned ldmHashLog;
-    unsigned ldmBucketSizeLog;
-    unsigned ldmHashRateLog;
+    BMK_mode_t mode;        /* 0: all, 1: compress only 2: decode only */
+    unsigned nbSeconds;     /* default timing is in nbSeconds */
+    size_t blockSize;       /* Maximum size of each block*/
+    int nbWorkers;          /* multithreading */
+    unsigned realTime;      /* real time priority */
+    int additionalParam;    /* used by python speed benchmark */
+    int ldmFlag;            /* enables long distance matching */
+    int ldmMinMatch;        /* below: parameters for long distance matching, see zstd.1.md */
+    int ldmHashLog;
+    int ldmBucketSizeLog;
+    int ldmHashRateLog;
 } BMK_advancedParams_t;
 
 /* returns default parameters used by nonAdvanced functions */

From cee31bbd257263c540b9925790f54689698c7578 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 25 Jan 2019 15:38:08 -0800
Subject: [PATCH 029/178] updated fullbench and paramgrill

to use new benchfn
returning a double.
---
 tests/fullbench.c  | 51 ++++++++++++++++++++++------------------------
 tests/paramgrill.c | 44 +++++++++++++++++++++++----------------
 2 files changed, 50 insertions(+), 45 deletions(-)

diff --git a/tests/fullbench.c b/tests/fullbench.c
index 8644a2e3a13..c46535178e3 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -67,12 +67,6 @@ static const size_t g_sampleSize = 10000000;
 static unsigned g_nbIterations = NBLOOPS;
 static double g_compressibility = COMPRESSIBILITY_DEFAULT;
 
-static void BMK_SetNbIterations(unsigned nbLoops)
-{
-    g_nbIterations = nbLoops;
-    DISPLAY("- %i iterations -\n", g_nbIterations);
-}
-
 
 /*_*******************************************************
 *  Private functions
@@ -316,9 +310,9 @@ static size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize,
 /*_*******************************************************
 *  Bench functions
 *********************************************************/
-static size_t benchMem(unsigned benchNb,
-                       const void* src, size_t srcSize,
-                       int cLevel, ZSTD_compressionParameters cparams)
+static int benchMem(unsigned benchNb,
+                    const void* src, size_t srcSize,
+                    int cLevel, ZSTD_compressionParameters cparams)
 {
     size_t dstBuffSize = ZSTD_compressBound(srcSize);
     BYTE*  dstBuff;
@@ -395,22 +389,22 @@ static size_t benchMem(unsigned benchNb,
           cparams->minMatch, cparams->targetLength, cparams->strategy); */
 
     ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_compressionLevel, cLevel);
-    ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_windowLog, cparams.windowLog);
-    ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_hashLog, cparams.hashLog);
-    ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_chainLog, cparams.chainLog);
-    ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_searchLog, cparams.searchLog);
-    ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_minMatch, cparams.minMatch);
-    ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_targetLength, cparams.targetLength);
+    ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_windowLog, (int)cparams.windowLog);
+    ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_hashLog, (int)cparams.hashLog);
+    ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_chainLog, (int)cparams.chainLog);
+    ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_searchLog, (int)cparams.searchLog);
+    ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_minMatch, (int)cparams.minMatch);
+    ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_targetLength, (int)cparams.targetLength);
     ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_strategy, cparams.strategy);
 
 
     ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_compressionLevel, cLevel);
-    ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_windowLog, cparams.windowLog);
-    ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_hashLog, cparams.hashLog);
-    ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_chainLog, cparams.chainLog);
-    ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_searchLog, cparams.searchLog);
-    ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_minMatch, cparams.minMatch);
-    ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_targetLength, cparams.targetLength);
+    ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_windowLog, (int)cparams.windowLog);
+    ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_hashLog, (int)cparams.hashLog);
+    ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_chainLog, (int)cparams.chainLog);
+    ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_searchLog, (int)cparams.searchLog);
+    ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_minMatch, (int)cparams.minMatch);
+    ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_targetLength, (int)cparams.targetLength);
     ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_strategy, cparams.strategy);
 
     /* Preparation */
@@ -469,8 +463,9 @@ static size_t benchMem(unsigned benchNb,
             iend = ip + ZSTD_blockHeaderSize + cBlockSize;   /* End of first block */
             ip += ZSTD_blockHeaderSize;                      /* skip block header */
             ZSTD_decompressBegin(g_zdc);
-            ip += ZSTD_decodeLiteralsBlock(g_zdc, ip, iend-ip);   /* skip literal segment */
-            g_cSize = iend-ip;
+            assert(iend > ip);
+            ip += ZSTD_decodeLiteralsBlock(g_zdc, ip, (size_t)(iend-ip));   /* skip literal segment */
+            g_cSize = (size_t)(iend-ip);
             memcpy(buff2, ip, g_cSize);   /* copy rest of block (it starts by SeqHeader) */
             srcSize = srcSize > 128 KB ? 128 KB : srcSize;   /* speed relative to block */
             break;
@@ -654,7 +649,9 @@ static unsigned readU32FromChar(const char** stringPtr)
     while ((**stringPtr >='0') && (**stringPtr <='9')) {
         unsigned const max = (((unsigned)(-1)) / 10) - 1;
         if (result > max) ERROR_OUT(errorMsg);
-        result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
+        result *= 10;
+        result += (unsigned)(**stringPtr - '0');
+        (*stringPtr)++ ;
     }
     if ((**stringPtr=='K') || (**stringPtr=='M')) {
         unsigned const maxK = ((unsigned)(-1)) >> 10;
@@ -671,7 +668,7 @@ static unsigned readU32FromChar(const char** stringPtr)
     return result;
 }
 
-static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
+static int longCommandWArg(const char** stringPtr, const char* longCommand)
 {
     size_t const comSize = strlen(longCommand);
     int const result = !strncmp(*stringPtr, longCommand, comSize);
@@ -772,7 +769,7 @@ int main(int argc, const char** argv)
                     /* Modify Nb Iterations */
                 case 'i':
                     argument++;
-                    BMK_SetNbIterations((int)readU32FromChar(&argument));
+                    g_nbIterations = readU32FromChar(&argument);
                     break;
 
                     /* Select compressibility of synthetic sample */
@@ -782,7 +779,7 @@ int main(int argc, const char** argv)
                     break;
                 case 'l':
                     argument++;
-                    cLevel = readU32FromChar(&argument);
+                    cLevel = (int)readU32FromChar(&argument);
                     cparams = ZSTD_getCParams(cLevel, 0, 0);
                     break;
 
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 415551b9578..88b7259c865 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -141,7 +141,7 @@ static const char* g_shortParamNames[NUM_PARAMS] =
 /* maps value from { 0 to rangetable[param] - 1 } to valid paramvalues */
 static U32 rangeMap(varInds_t param, int ind)
 {
-    ind = MAX(MIN(ind, (int)rangetable[param] - 1), 0);
+    U32 const uind = (U32)MAX(MIN(ind, (int)rangetable[param] - 1), 0);
     switch(param) {
         case wlog_ind: /* using default: triggers -Wswitch-enum */
         case clog_ind:
@@ -149,11 +149,11 @@ static U32 rangeMap(varInds_t param, int ind)
         case slog_ind:
         case mml_ind:
         case strt_ind:
-            return mintable[param] + ind;
+            return mintable[param] + uind;
         case tlen_ind:
-            return tlen_table[ind];
+            return tlen_table[uind];
         case fadt_ind: /* 0, 1, 2 -> -1, 0, 1 */
-            return ind - 1;
+            return uind - 1;
         case NUM_PARAMS:
         default:;
     }
@@ -173,7 +173,7 @@ static int invRangeMap(varInds_t param, U32 value)
         case slog_ind:
         case mml_ind:
         case strt_ind:
-            return value - mintable[param];
+            return (int)(value - mintable[param]);
         case tlen_ind: /* bin search */
         {
             int lo = 0;
@@ -493,13 +493,15 @@ static void
 paramVariation(paramValues_t* ptr, memoTable_t* mtAll, const U32 nbChanges)
 {
     paramValues_t p;
-    U32 validated = 0;
+    int validated = 0;
     while (!validated) {
         U32 i;
         p = *ptr;
         for (i = 0 ; i < nbChanges ; i++) {
             const U32 changeID = (U32)FUZ_rand(&g_rand) % (mtAll[p.vals[strt_ind]].varLen << 1);
-            paramVaryOnce(mtAll[p.vals[strt_ind]].varArray[changeID >> 1], ((changeID & 1) << 1) - 1, &p);
+            paramVaryOnce(mtAll[p.vals[strt_ind]].varArray[changeID >> 1],
+                          (int)((changeID & 1) << 1) - 1,
+                          &p);
         }
         validated = paramValid(p);
     }
@@ -511,7 +513,7 @@ static paramValues_t randomParams(void)
 {
     varInds_t v; paramValues_t p;
     for(v = 0; v < NUM_PARAMS; v++) {
-        p.vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]);
+        p.vals[v] = rangeMap(v, (int)(FUZ_rand(&g_rand) % rangetable[v]));
     }
     return p;
 }
@@ -1638,7 +1640,7 @@ BMK_benchMemInvertible( buffers_t buf, contexts_t ctx,
                 return bOut;
             }
             {   BMK_runTime_t const rResult = BMK_extract_runTime(cOutcome);
-                bResult.cSpeed = (srcSize * TIMELOOP_NANOSEC) / rResult.nanoSecPerRun;
+                bResult.cSpeed = (unsigned long long)((double)srcSize * TIMELOOP_NANOSEC / rResult.nanoSecPerRun);
                 bResult.cSize = rResult.sumOfReturn;
             }
             compressionCompleted = BMK_isCompleted_TimedFn(timeStateCompress);
@@ -1656,7 +1658,7 @@ BMK_benchMemInvertible( buffers_t buf, contexts_t ctx,
                 return bOut;
             }
             {   BMK_runTime_t const rResult = BMK_extract_runTime(dOutcome);
-                bResult.dSpeed = (srcSize * TIMELOOP_NANOSEC) / rResult.nanoSecPerRun;
+                bResult.dSpeed = (unsigned long long)((double)srcSize * TIMELOOP_NANOSEC / rResult.nanoSecPerRun);
             }
             decompressionCompleted = BMK_isCompleted_TimedFn(timeStateDecompress);
         }
@@ -2199,7 +2201,9 @@ static winnerInfo_t climbOnce(const constraint_t target,
                 for (offset = -1; offset <= 1; offset += 2) {
                     CHECKTIME(winnerInfo);
                     candidateInfo.params = cparam;
-                    paramVaryOnce(mtAll[cparam.vals[strt_ind]].varArray[i], offset, &candidateInfo.params);
+                    paramVaryOnce(mtAll[cparam.vals[strt_ind]].varArray[i],
+                                  offset,
+                                  &candidateInfo.params);
 
                     if(paramValid(candidateInfo.params)) {
                         int res;
@@ -2351,7 +2355,7 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy)
  * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio)
  */
 
-static int g_maxTries = 5;
+static unsigned g_maxTries = 5;
 #define TRY_DECAY 1
 
 static int
@@ -2561,7 +2565,7 @@ optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles,
  * @return 0 and doesn't modify *stringPtr otherwise.
  * from zstdcli.c
  */
-static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
+static int longCommandWArg(const char** stringPtr, const char* longCommand)
 {
     size_t const comSize = strlen(longCommand);
     int const result = !strncmp(*stringPtr, longCommand, comSize);
@@ -2588,7 +2592,10 @@ static unsigned readU32FromChar(const char** stringPtr)
     while ((**stringPtr >='0') && (**stringPtr <='9')) {
         unsigned const max = (((unsigned)(-1)) / 10) - 1;
         if (result > max) errorOut(errorMsg);
-        result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
+        result *= 10;
+        assert(**stringPtr >= '0');
+        result += (unsigned)(**stringPtr - '0');
+        (*stringPtr)++ ;
     }
     if ((**stringPtr=='K') || (**stringPtr=='M')) {
         unsigned const maxK = ((unsigned)(-1)) >> 10;
@@ -2726,7 +2733,7 @@ int main(int argc, const char** argv)
                 PARSE_SUB_ARGS("strict=", "stc=", g_strictness);
                 PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries);
                 PARSE_SUB_ARGS("memoLimitLog=", "memLog=", memoTableLog);
-                if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelOpt = readU32FromChar(&argument); g_optmode = 1; if (argument[0]==',') { argument++; continue; } else break; }
+                if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelOpt = (int)readU32FromChar(&argument); g_optmode = 1; if (argument[0]==',') { argument++; continue; } else break; }
                 if (longCommandWArg(&argument, "speedForRatio=") || longCommandWArg(&argument, "speedRatio=")) { g_ratioMultiplier = readDoubleFromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
 
                 DISPLAY("invalid optimization parameter \n");
@@ -2743,7 +2750,7 @@ int main(int argc, const char** argv)
             g_singleRun = 1;
             for ( ; ;) {
                 if(parse_params(&argument, &g_params)) { if(argument[0] == ',') { argument++; continue; } else break; }
-                if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelRun = readU32FromChar(&argument); g_params = emptyParams(); if (argument[0]==',') { argument++; continue; } else break; }
+                if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelRun = (int)readU32FromChar(&argument); g_params = emptyParams(); if (argument[0]==',') { argument++; continue; } else break; }
 
                 DISPLAY("invalid compression parameter \n");
                 return 1;
@@ -2855,7 +2862,7 @@ int main(int argc, const char** argv)
                             continue;
                         case 'L':
                             {   argument++;
-                                cLevelRun = readU32FromChar(&argument);
+                                cLevelRun = (int)readU32FromChar(&argument);
                                 g_params = emptyParams();
                                 continue;
                             }
@@ -2944,7 +2951,8 @@ int main(int argc, const char** argv)
             }
         } else {
             if (g_optimizer) {
-                result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, cLevelOpt, cLevelRun, memoTableLog);
+                assert(filenamesStart < argc);
+                result = optimizeForSize(argv+filenamesStart, (size_t)(argc-filenamesStart), dictFileName, target, paramTarget, cLevelOpt, cLevelRun, memoTableLog);
             } else {
                 result = benchFiles(argv+filenamesStart, argc-filenamesStart, dictFileName, cLevelRun);
             }

From 54fa31f03b0a4eb9d7ac38943282de7f4da19c4f Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Wed, 5 Dec 2018 16:23:18 -0800
Subject: [PATCH 030/178] Add RETURN_ERROR_IF Macro That Logs Debug Information
 When Check Fails

---
 lib/common/zstd_internal.h | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index edeb74b9c3a..73fc7705b94 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -56,6 +56,14 @@ extern "C" {
 #define CHECK_F(f) { size_t const errcod = f; if (ERR_isError(errcod)) return errcod; }  /* check and Forward error code */
 #define CHECK_E(f, e) { size_t const errcod = f; if (ERR_isError(errcod)) return ERROR(e); }  /* check and send Error code */
 
+#define RETURN_ERROR_IF(cond, err, ...) \
+  if (cond) { \
+    RAWLOG(3, "%s:%d: check %s failed, returning %s", __FILE__, __LINE__, ZSTD_QUOTE(cond), ZSTD_QUOTE(ERROR(err))); \
+    RAWLOG(3, ": " __VA_ARGS__); \
+    RAWLOG(3, "\n"); \
+    return ERROR(err); \
+  }
+
 
 /*-*************************************
 *  Common constants

From ea031f4ea2cba65a9bb4d76d01b71597320db108 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Wed, 5 Dec 2018 17:17:11 -0800
Subject: [PATCH 031/178] Convert Checks in zstd_decompress_block.c to
 RETURN_ERROR_IF

---
 lib/decompress/zstd_decompress_block.c | 86 +++++++++++++-------------
 1 file changed, 43 insertions(+), 43 deletions(-)

diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c
index 32baad9fbb5..b1932e5dede 100644
--- a/lib/decompress/zstd_decompress_block.c
+++ b/lib/decompress/zstd_decompress_block.c
@@ -56,14 +56,15 @@ static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
 size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
                           blockProperties_t* bpPtr)
 {
-    if (srcSize < ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
+    RETURN_ERROR_IF(srcSize < ZSTD_blockHeaderSize, srcSize_wrong);
+
     {   U32 const cBlockHeader = MEM_readLE24(src);
         U32 const cSize = cBlockHeader >> 3;
         bpPtr->lastBlock = cBlockHeader & 1;
         bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3);
         bpPtr->origSize = cSize;   /* only useful for RLE */
         if (bpPtr->blockType == bt_rle) return 1;
-        if (bpPtr->blockType == bt_reserved) return ERROR(corruption_detected);
+        RETURN_ERROR_IF(bpPtr->blockType == bt_reserved, corruption_detected);
         return cSize;
     }
 }
@@ -78,7 +79,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
 size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
                           const void* src, size_t srcSize)   /* note : srcSize < BLOCKSIZE */
 {
-    if (srcSize < MIN_CBLOCK_SIZE) return ERROR(corruption_detected);
+    RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected);
 
     {   const BYTE* const istart = (const BYTE*) src;
         symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3);
@@ -86,11 +87,11 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
         switch(litEncType)
         {
         case set_repeat:
-            if (dctx->litEntropy==0) return ERROR(dictionary_corrupted);
+            RETURN_ERROR_IF(dctx->litEntropy==0, dictionary_corrupted);
             /* fall-through */
 
         case set_compressed:
-            if (srcSize < 5) return ERROR(corruption_detected);   /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3 */
+            RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3");
             {   size_t lhSize, litSize, litCSize;
                 U32 singleStream=0;
                 U32 const lhlCode = (istart[0] >> 2) & 3;
@@ -118,8 +119,8 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
                     litCSize = (lhc >> 22) + (istart[4] << 10);
                     break;
                 }
-                if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected);
-                if (litCSize + lhSize > srcSize) return ERROR(corruption_detected);
+                RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected);
+                RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected);
 
                 /* prefetch huffman table if cold */
                 if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) {
@@ -157,7 +158,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
                     }
                 }
 
-                if (HUF_isError(hufSuccess)) return ERROR(corruption_detected);
+                RETURN_ERROR_IF(HUF_isError(hufSuccess), corruption_detected);
 
                 dctx->litPtr = dctx->litBuffer;
                 dctx->litSize = litSize;
@@ -187,7 +188,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
                 }
 
                 if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) {  /* risk reading beyond src buffer with wildcopy */
-                    if (litSize+lhSize > srcSize) return ERROR(corruption_detected);
+                    RETURN_ERROR_IF(litSize+lhSize > srcSize, corruption_detected);
                     memcpy(dctx->litBuffer, istart+lhSize, litSize);
                     dctx->litPtr = dctx->litBuffer;
                     dctx->litSize = litSize;
@@ -216,17 +217,17 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
                 case 3:
                     lhSize = 3;
                     litSize = MEM_readLE24(istart) >> 4;
-                    if (srcSize<4) return ERROR(corruption_detected);   /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */
+                    RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4");
                     break;
                 }
-                if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected);
+                RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected);
                 memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH);
                 dctx->litPtr = dctx->litBuffer;
                 dctx->litSize = litSize;
                 return lhSize+1;
             }
         default:
-            return ERROR(corruption_detected);   /* impossible */
+            RETURN_ERROR_IF(1, corruption_detected, "impossible");
         }
     }
 }
@@ -436,8 +437,8 @@ static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymb
     switch(type)
     {
     case set_rle :
-        if (!srcSize) return ERROR(srcSize_wrong);
-        if ( (*(const BYTE*)src) > max) return ERROR(corruption_detected);
+        RETURN_ERROR_IF(!srcSize, srcSize_wrong);
+        RETURN_ERROR_IF((*(const BYTE*)src) > max, corruption_detected);
         {   U32 const symbol = *(const BYTE*)src;
             U32 const baseline = baseValue[symbol];
             U32 const nbBits = nbAdditionalBits[symbol];
@@ -449,7 +450,7 @@ static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymb
         *DTablePtr = defaultTable;
         return 0;
     case set_repeat:
-        if (!flagRepeatTable) return ERROR(corruption_detected);
+        RETURN_ERROR_IF(!flagRepeatTable, corruption_detected);
         /* prefetch FSE table if used */
         if (ddictIsCold && (nbSeq > 24 /* heuristic */)) {
             const void* const pStart = *DTablePtr;
@@ -461,15 +462,15 @@ static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymb
         {   unsigned tableLog;
             S16 norm[MaxSeq+1];
             size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize);
-            if (FSE_isError(headerSize)) return ERROR(corruption_detected);
-            if (tableLog > maxLog) return ERROR(corruption_detected);
+            RETURN_ERROR_IF(FSE_isError(headerSize), corruption_detected);
+            RETURN_ERROR_IF(tableLog > maxLog, corruption_detected);
             ZSTD_buildFSETable(DTableSpace, norm, max, baseValue, nbAdditionalBits, tableLog);
             *DTablePtr = DTableSpace;
             return headerSize;
         }
-    default :   /* impossible */
+    default :
         assert(0);
-        return ERROR(GENERIC);
+        RETURN_ERROR_IF(1, GENERIC, "impossible");
     }
 }
 
@@ -483,28 +484,28 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
     DEBUGLOG(5, "ZSTD_decodeSeqHeaders");
 
     /* check */
-    if (srcSize < MIN_SEQUENCES_SIZE) return ERROR(srcSize_wrong);
+    RETURN_ERROR_IF(srcSize < MIN_SEQUENCES_SIZE, srcSize_wrong);
 
     /* SeqHead */
     nbSeq = *ip++;
     if (!nbSeq) {
         *nbSeqPtr=0;
-        if (srcSize != 1) return ERROR(srcSize_wrong);
+        RETURN_ERROR_IF(srcSize != 1, srcSize_wrong);
         return 1;
     }
     if (nbSeq > 0x7F) {
         if (nbSeq == 0xFF) {
-            if (ip+2 > iend) return ERROR(srcSize_wrong);
+            RETURN_ERROR_IF(ip+2 > iend, srcSize_wrong);
             nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2;
         } else {
-            if (ip >= iend) return ERROR(srcSize_wrong);
+            RETURN_ERROR_IF(ip >= iend, srcSize_wrong);
             nbSeq = ((nbSeq-0x80)<<8) + *ip++;
         }
     }
     *nbSeqPtr = nbSeq;
 
     /* FSE table descriptors */
-    if (ip+4 > iend) return ERROR(srcSize_wrong); /* minimum possible size */
+    RETURN_ERROR_IF(ip+4 > iend, srcSize_wrong); /* minimum possible size */
     {   symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6);
         symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3);
         symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3);
@@ -517,7 +518,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
                                                       LL_base, LL_bits,
                                                       LL_defaultDTable, dctx->fseEntropy,
                                                       dctx->ddictIsCold, nbSeq);
-            if (ZSTD_isError(llhSize)) return ERROR(corruption_detected);
+            RETURN_ERROR_IF(ZSTD_isError(llhSize), corruption_detected);
             ip += llhSize;
         }
 
@@ -527,7 +528,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
                                                       OF_base, OF_bits,
                                                       OF_defaultDTable, dctx->fseEntropy,
                                                       dctx->ddictIsCold, nbSeq);
-            if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected);
+            RETURN_ERROR_IF(ZSTD_isError(ofhSize), corruption_detected);
             ip += ofhSize;
         }
 
@@ -537,7 +538,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
                                                       ML_base, ML_bits,
                                                       ML_defaultDTable, dctx->fseEntropy,
                                                       dctx->ddictIsCold, nbSeq);
-            if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected);
+            RETURN_ERROR_IF(ZSTD_isError(mlhSize), corruption_detected);
             ip += mlhSize;
         }
     }
@@ -590,8 +591,8 @@ size_t ZSTD_execSequenceLast7(BYTE* op,
     const BYTE* match = oLitEnd - sequence.offset;
 
     /* check */
-    if (oMatchEnd>oend) return ERROR(dstSize_tooSmall);   /* last match must fit within dstBuffer */
-    if (iLitEnd > litLimit) return ERROR(corruption_detected);   /* try to read beyond literal buffer */
+    RETURN_ERROR_IF(oMatchEnd>oend, dstSize_tooSmall, "last match must fit within dstBuffer");
+    RETURN_ERROR_IF(iLitEnd > litLimit, corruption_detected, "try to read beyond literal buffer");
 
     /* copy literals */
     while (op < oLitEnd) *op++ = *(*litPtr)++;
@@ -599,7 +600,7 @@ size_t ZSTD_execSequenceLast7(BYTE* op,
     /* copy Match */
     if (sequence.offset > (size_t)(oLitEnd - base)) {
         /* offset beyond prefix */
-        if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected);
+        RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - vBase),corruption_detected);
         match = dictEnd - (base-match);
         if (match + sequence.matchLength <= dictEnd) {
             memmove(oLitEnd, match, sequence.matchLength);
@@ -631,8 +632,8 @@ size_t ZSTD_execSequence(BYTE* op,
     const BYTE* match = oLitEnd - sequence.offset;
 
     /* check */
-    if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
-    if (iLitEnd > litLimit) return ERROR(corruption_detected);   /* over-read beyond lit buffer */
+    RETURN_ERROR_IF(oMatchEnd>oend, dstSize_tooSmall, "last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend");
+    RETURN_ERROR_IF(iLitEnd > litLimit, corruption_detected, "over-read beyond lit buffer");
     if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd);
 
     /* copy Literals */
@@ -645,8 +646,7 @@ size_t ZSTD_execSequence(BYTE* op,
     /* copy Match */
     if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
         /* offset beyond prefix -> go into extDict */
-        if (sequence.offset > (size_t)(oLitEnd - virtualStart))
-            return ERROR(corruption_detected);
+        RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected);
         match = dictEnd + (match - prefixStart);
         if (match + sequence.matchLength <= dictEnd) {
             memmove(oLitEnd, match, sequence.matchLength);
@@ -712,8 +712,8 @@ size_t ZSTD_execSequenceLong(BYTE* op,
     const BYTE* match = sequence.match;
 
     /* check */
-    if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
-    if (iLitEnd > litLimit) return ERROR(corruption_detected);   /* over-read beyond lit buffer */
+    RETURN_ERROR_IF(oMatchEnd > oend, dstSize_tooSmall, "last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend");
+    RETURN_ERROR_IF(iLitEnd > litLimit, corruption_detected, "over-read beyond lit buffer");
     if (oLitEnd > oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, prefixStart, dictStart, dictEnd);
 
     /* copy Literals */
@@ -726,7 +726,7 @@ size_t ZSTD_execSequenceLong(BYTE* op,
     /* copy Match */
     if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
         /* offset beyond prefix */
-        if (sequence.offset > (size_t)(oLitEnd - dictStart)) return ERROR(corruption_detected);
+        RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - dictStart), corruption_detected);
         if (match + sequence.matchLength <= dictEnd) {
             memmove(oLitEnd, match, sequence.matchLength);
             return sequenceLength;
@@ -927,14 +927,14 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx,
 
         /* check if reached exact end */
         DEBUGLOG(5, "ZSTD_decompressSequences_body: after decode loop, remaining nbSeq : %i", nbSeq);
-        if (nbSeq) return ERROR(corruption_detected);
+        RETURN_ERROR_IF(nbSeq, corruption_detected);
         /* save reps for next block */
         { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); }
     }
 
     /* last literal segment */
     {   size_t const lastLLSize = litEnd - litPtr;
-        if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall);
+        RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall);
         memcpy(op, litPtr, lastLLSize);
         op += lastLLSize;
     }
@@ -1076,7 +1076,7 @@ ZSTD_decompressSequencesLong_body(
             sequences[seqNb] = ZSTD_decodeSequenceLong(&seqState, isLongOffset);
             PREFETCH_L1(sequences[seqNb].match); PREFETCH_L1(sequences[seqNb].match + sequences[seqNb].matchLength - 1); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */
         }
-        if (seqNb (size_t)(oend-op)) return ERROR(dstSize_tooSmall);
+        RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall);
         memcpy(op, litPtr, lastLLSize);
         op += lastLLSize;
     }
@@ -1240,7 +1240,7 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
     ZSTD_longOffset_e const isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (!frame || (dctx->fParams.windowSize > (1ULL << STREAM_ACCUMULATOR_MIN))));
     DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize);
 
-    if (srcSize >= ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong);
+    RETURN_ERROR_IF(srcSize >= ZSTD_BLOCKSIZE_MAX, srcSize_wrong);
 
     /* Decode literals section */
     {   size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize);

From c823237d7b72670e2a769800e8109d7c5db46bca Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Mon, 28 Jan 2019 12:22:52 -0500
Subject: [PATCH 032/178] Convert Checks in zstd_decompress.c to
 RETURN_ERROR_IF

---
 lib/decompress/zstd_decompress.c | 228 ++++++++++++++++---------------
 1 file changed, 115 insertions(+), 113 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index feef1ef67ab..ae976e11451 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -150,7 +150,7 @@ ZSTD_DCtx* ZSTD_createDCtx(void)
 size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
 {
     if (dctx==NULL) return 0;   /* support free on NULL */
-    if (dctx->staticSize) return ERROR(memory_allocation);   /* not compatible with static DCtx */
+    RETURN_ERROR_IF(dctx->staticSize, memory_allocation, "not compatible with static DCtx");
     {   ZSTD_customMem const cMem = dctx->customMem;
         ZSTD_freeDDict(dctx->ddictLocal);
         dctx->ddictLocal = NULL;
@@ -203,7 +203,7 @@ unsigned ZSTD_isFrame(const void* buffer, size_t size)
 static size_t ZSTD_frameHeaderSize_internal(const void* src, size_t srcSize, ZSTD_format_e format)
 {
     size_t const minInputSize = ZSTD_startingInputLength(format);
-    if (srcSize < minInputSize) return ERROR(srcSize_wrong);
+    RETURN_ERROR_IF(srcSize < minInputSize, srcSize_wrong);
 
     {   BYTE const fhd = ((const BYTE*)src)[minInputSize-1];
         U32 const dictID= fhd & 3;
@@ -238,7 +238,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s
 
     memset(zfhPtr, 0, sizeof(*zfhPtr));   /* not strictly necessary, but static analyzer do not understand that zfhPtr is only going to be read only if return value is zero, since they are 2 different signals */
     if (srcSize < minInputSize) return minInputSize;
-    if (src==NULL) return ERROR(GENERIC);   /* invalid parameter */
+    RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter");
 
     if ( (format != ZSTD_f_zstd1_magicless)
       && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) {
@@ -251,7 +251,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s
             zfhPtr->frameType = ZSTD_skippableFrame;
             return 0;
         }
-        return ERROR(prefix_unknown);
+        RETURN_ERROR_IF(1, prefix_unknown);
     }
 
     /* ensure there is enough `srcSize` to fully read/decode frame header */
@@ -269,14 +269,13 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s
         U64 windowSize = 0;
         U32 dictID = 0;
         U64 frameContentSize = ZSTD_CONTENTSIZE_UNKNOWN;
-        if ((fhdByte & 0x08) != 0)
-            return ERROR(frameParameter_unsupported); /* reserved bits, must be zero */
+        RETURN_ERROR_IF((fhdByte & 0x08) != 0, frameParameter_unsupported,
+                        "reserved bits, must be zero");
 
         if (!singleSegment) {
             BYTE const wlByte = ip[pos++];
             U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
-            if (windowLog > ZSTD_WINDOWLOG_MAX)
-                return ERROR(frameParameter_windowTooLarge);
+            RETURN_ERROR_IF(windowLog > ZSTD_WINDOWLOG_MAX, frameParameter_windowTooLarge);
             windowSize = (1ULL << windowLog);
             windowSize += (windowSize >> 3) * (wlByte&7);
         }
@@ -348,12 +347,11 @@ static size_t readSkippableFrameSize(void const* src, size_t srcSize)
     size_t const skippableHeaderSize = ZSTD_SKIPPABLEHEADERSIZE;
     U32 sizeU32;
 
-    if (srcSize < ZSTD_SKIPPABLEHEADERSIZE)
-        return ERROR(srcSize_wrong);
+    RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong);
 
     sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE);
-    if ((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32)
-        return ERROR(frameParameter_unsupported);
+    RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32,
+                    frameParameter_unsupported);
 
     return skippableHeaderSize + sizeU32;
 }
@@ -428,9 +426,9 @@ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t he
 {
     size_t const result = ZSTD_getFrameHeader_advanced(&(dctx->fParams), src, headerSize, dctx->format);
     if (ZSTD_isError(result)) return result;    /* invalid header */
-    if (result>0) return ERROR(srcSize_wrong);  /* headerSize too small */
-    if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID))
-        return ERROR(dictionary_wrong);
+    RETURN_ERROR_IF(result>0, srcSize_wrong, "headerSize too small");
+    RETURN_ERROR_IF(dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID),
+                    dictionary_wrong);
     if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0);
     return 0;
 }
@@ -459,7 +457,7 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
         /* Extract Frame Header */
         {   size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize);
             if (ZSTD_isError(ret)) return ret;
-            if (ret > 0) return ERROR(srcSize_wrong);
+            RETURN_ERROR_IF(ret > 0, srcSize_wrong);
         }
 
         ip += zfh.headerSize;
@@ -471,8 +469,8 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
             size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
             if (ZSTD_isError(cBlockSize)) return cBlockSize;
 
-            if (ZSTD_blockHeaderSize + cBlockSize > remainingSize)
-                return ERROR(srcSize_wrong);
+            RETURN_ERROR_IF(ZSTD_blockHeaderSize + cBlockSize > remainingSize,
+                            srcSize_wrong);
 
             ip += ZSTD_blockHeaderSize + cBlockSize;
             remainingSize -= ZSTD_blockHeaderSize + cBlockSize;
@@ -481,7 +479,7 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
         }
 
         if (zfh.checksumFlag) {   /* Final frame content checksum */
-            if (remainingSize < 4) return ERROR(srcSize_wrong);
+            RETURN_ERROR_IF(remainingSize < 4, srcSize_wrong);
             ip += 4;
         }
 
@@ -522,9 +520,9 @@ static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity,
     DEBUGLOG(5, "ZSTD_copyRawBlock");
     if (dst == NULL) {
         if (srcSize == 0) return 0;
-        return ERROR(dstBuffer_null);
+        RETURN_ERROR_IF(1, dstBuffer_null);
     }
-    if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall);
+    RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall);
     memcpy(dst, src, srcSize);
     return srcSize;
 }
@@ -535,9 +533,9 @@ static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity,
 {
     if (dst == NULL) {
         if (regenSize == 0) return 0;
-        return ERROR(dstBuffer_null);
+        RETURN_ERROR_IF(1, dstBuffer_null);
     }
-    if (regenSize > dstCapacity) return ERROR(dstSize_tooSmall);
+    RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall);
     memset(dst, b, regenSize);
     return regenSize;
 }
@@ -560,14 +558,15 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
     DEBUGLOG(4, "ZSTD_decompressFrame (srcSize:%i)", (int)*srcSizePtr);
 
     /* check */
-    if (remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN+ZSTD_blockHeaderSize)
-        return ERROR(srcSize_wrong);
+    RETURN_ERROR_IF(
+        remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN+ZSTD_blockHeaderSize,
+        srcSize_wrong);
 
     /* Frame Header */
     {   size_t const frameHeaderSize = ZSTD_frameHeaderSize(ip, ZSTD_FRAMEHEADERSIZE_PREFIX);
         if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize;
-        if (remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize)
-            return ERROR(srcSize_wrong);
+        RETURN_ERROR_IF(remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize,
+                        srcSize_wrong);
         CHECK_F( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) );
         ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize;
     }
@@ -581,7 +580,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
 
         ip += ZSTD_blockHeaderSize;
         remainingSrcSize -= ZSTD_blockHeaderSize;
-        if (cBlockSize > remainingSrcSize) return ERROR(srcSize_wrong);
+        RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong);
 
         switch(blockProperties.blockType)
         {
@@ -596,7 +595,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
             break;
         case bt_reserved :
         default:
-            return ERROR(corruption_detected);
+            RETURN_ERROR_IF(1, corruption_detected);
         }
 
         if (ZSTD_isError(decodedSize)) return decodedSize;
@@ -609,15 +608,15 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
     }
 
     if (dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) {
-        if ((U64)(op-ostart) != dctx->fParams.frameContentSize) {
-            return ERROR(corruption_detected);
-    }   }
+        RETURN_ERROR_IF((U64)(op-ostart) != dctx->fParams.frameContentSize,
+                        corruption_detected);
+    }
     if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */
         U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState);
         U32 checkRead;
-        if (remainingSrcSize<4) return ERROR(checksum_wrong);
+        RETURN_ERROR_IF(remainingSrcSize<4, checksum_wrong);
         checkRead = MEM_readLE32(ip);
-        if (checkRead != checkCalc) return ERROR(checksum_wrong);
+        RETURN_ERROR_IF(checkRead != checkCalc, checksum_wrong);
         ip += 4;
         remainingSrcSize -= 4;
     }
@@ -652,8 +651,8 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
             size_t decodedSize;
             size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
             if (ZSTD_isError(frameSize)) return frameSize;
-            /* legacy support is not compatible with static dctx */
-            if (dctx->staticSize) return ERROR(memory_allocation);
+            RETURN_ERROR_IF(dctx->staticSize, memory_allocation,
+                "legacy support is not compatible with static dctx");
 
             decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize);
             if (ZSTD_isError(decodedSize)) return decodedSize;
@@ -676,7 +675,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
                 size_t const skippableSize = readSkippableFrameSize(src, srcSize);
                 if (ZSTD_isError(skippableSize))
                     return skippableSize;
-                if (srcSize < skippableSize) return ERROR(srcSize_wrong);
+                RETURN_ERROR_IF(srcSize < skippableSize, srcSize_wrong);
 
                 src = (const BYTE *)src + skippableSize;
                 srcSize -= skippableSize;
@@ -695,19 +694,19 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
 
         {   const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity,
                                                     &src, &srcSize);
-            if ( (ZSTD_getErrorCode(res) == ZSTD_error_prefix_unknown)
-              && (moreThan1Frame==1) ) {
-                /* at least one frame successfully completed,
-                 * but following bytes are garbage :
-                 * it's more likely to be a srcSize error,
-                 * specifying more bytes than compressed size of frame(s).
-                 * This error message replaces ERROR(prefix_unknown),
-                 * which would be confusing, as the first header is actually correct.
-                 * Note that one could be unlucky, it might be a corruption error instead,
-                 * happening right at the place where we expect zstd magic bytes.
-                 * But this is _much_ less likely than a srcSize field error. */
-                return ERROR(srcSize_wrong);
-            }
+            RETURN_ERROR_IF(
+                (ZSTD_getErrorCode(res) == ZSTD_error_prefix_unknown)
+             && (moreThan1Frame==1),
+                srcSize_wrong,
+                "at least one frame successfully completed, but following "
+                "bytes are garbage: it's more likely to be a srcSize error, "
+                "specifying more bytes than compressed size of frame(s). This "
+                "error message replaces ERROR(prefix_unknown), which would be "
+                "confusing, as the first header is actually correct. Note that "
+                "one could be unlucky, it might be a corruption error instead, "
+                "happening right at the place where we expect zstd magic "
+                "bytes. But this is _much_ less likely than a srcSize field "
+                "error.");
             if (ZSTD_isError(res)) return res;
             assert(res <= dstCapacity);
             dst = (BYTE*)dst + res;
@@ -716,7 +715,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
         moreThan1Frame = 1;
     }  /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */
 
-    if (srcSize) return ERROR(srcSize_wrong); /* input not entirely consumed */
+    RETURN_ERROR_IF(srcSize, srcSize_wrong, "input not entirely consumed");
 
     return (BYTE*)dst - (BYTE*)dststart;
 }
@@ -741,7 +740,7 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr
 #if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1)
     size_t regenSize;
     ZSTD_DCtx* const dctx = ZSTD_createDCtx();
-    if (dctx==NULL) return ERROR(memory_allocation);
+    RETURN_ERROR_IF(dctx==NULL, memory_allocation);
     regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize);
     ZSTD_freeDCtx(dctx);
     return regenSize;
@@ -791,8 +790,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
 {
     DEBUGLOG(5, "ZSTD_decompressContinue (srcSize:%u)", (unsigned)srcSize);
     /* Sanity check */
-    if (srcSize != dctx->expected)
-        return ERROR(srcSize_wrong);  /* not allowed */
+    RETURN_ERROR_IF(srcSize != dctx->expected, srcSize_wrong, "not allowed");
     if (dstCapacity) ZSTD_checkContinuity(dctx, dst);
 
     switch (dctx->stage)
@@ -867,7 +865,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
                 break;
             case bt_reserved :   /* should never happen */
             default:
-                return ERROR(corruption_detected);
+                RETURN_ERROR_IF(1, corruption_detected);
             }
             if (ZSTD_isError(rSize)) return rSize;
             DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize);
@@ -876,10 +874,10 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
 
             if (dctx->stage == ZSTDds_decompressLastBlock) {   /* end of frame */
                 DEBUGLOG(4, "ZSTD_decompressContinue: decoded size from frame : %u", (unsigned)dctx->decodedSize);
-                if (dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) {
-                    if (dctx->decodedSize != dctx->fParams.frameContentSize) {
-                        return ERROR(corruption_detected);
-                }   }
+                RETURN_ERROR_IF(
+                    dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN
+                 && dctx->decodedSize != dctx->fParams.frameContentSize,
+                    corruption_detected);
                 if (dctx->fParams.checksumFlag) {  /* another round for frame checksum */
                     dctx->expected = 4;
                     dctx->stage = ZSTDds_checkChecksum;
@@ -900,7 +898,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
         {   U32 const h32 = (U32)XXH64_digest(&dctx->xxhState);
             U32 const check32 = MEM_readLE32(src);
             DEBUGLOG(4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", (unsigned)h32, (unsigned)check32);
-            if (check32 != h32) return ERROR(checksum_wrong);
+            RETURN_ERROR_IF(check32 != h32, checksum_wrong);
             dctx->expected = 0;
             dctx->stage = ZSTDds_getFrameHeaderSize;
             return 0;
@@ -921,7 +919,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
 
     default:
         assert(0);   /* impossible */
-        return ERROR(GENERIC);   /* some compiler require default to do something */
+        RETURN_ERROR_IF(1, GENERIC);   /* some compiler require default to do something */
     }
 }
 
@@ -945,7 +943,7 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
     const BYTE* dictPtr = (const BYTE*)dict;
     const BYTE* const dictEnd = dictPtr + dictSize;
 
-    if (dictSize <= 8) return ERROR(dictionary_corrupted);
+    RETURN_ERROR_IF(dictSize <= 8, dictionary_corrupted);
     assert(MEM_readLE32(dict) == ZSTD_MAGIC_DICTIONARY);   /* dict must be valid */
     dictPtr += 8;   /* skip header = magic + dictID */
 
@@ -964,16 +962,16 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
                                                 dictPtr, dictEnd - dictPtr,
                                                 workspace, workspaceSize);
 #endif
-        if (HUF_isError(hSize)) return ERROR(dictionary_corrupted);
+        RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted);
         dictPtr += hSize;
     }
 
     {   short offcodeNCount[MaxOff+1];
         unsigned offcodeMaxValue = MaxOff, offcodeLog;
         size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
-        if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
-        if (offcodeMaxValue > MaxOff) return ERROR(dictionary_corrupted);
-        if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted);
+        RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted);
+        RETURN_ERROR_IF(offcodeMaxValue > MaxOff, dictionary_corrupted);
+        RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted);
         ZSTD_buildFSETable( entropy->OFTable,
                             offcodeNCount, offcodeMaxValue,
                             OF_base, OF_bits,
@@ -984,9 +982,9 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
     {   short matchlengthNCount[MaxML+1];
         unsigned matchlengthMaxValue = MaxML, matchlengthLog;
         size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
-        if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
-        if (matchlengthMaxValue > MaxML) return ERROR(dictionary_corrupted);
-        if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted);
+        RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted);
+        RETURN_ERROR_IF(matchlengthMaxValue > MaxML, dictionary_corrupted);
+        RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted);
         ZSTD_buildFSETable( entropy->MLTable,
                             matchlengthNCount, matchlengthMaxValue,
                             ML_base, ML_bits,
@@ -997,9 +995,9 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
     {   short litlengthNCount[MaxLL+1];
         unsigned litlengthMaxValue = MaxLL, litlengthLog;
         size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
-        if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
-        if (litlengthMaxValue > MaxLL) return ERROR(dictionary_corrupted);
-        if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted);
+        RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted);
+        RETURN_ERROR_IF(litlengthMaxValue > MaxLL, dictionary_corrupted);
+        RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted);
         ZSTD_buildFSETable( entropy->LLTable,
                             litlengthNCount, litlengthMaxValue,
                             LL_base, LL_bits,
@@ -1007,12 +1005,13 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
         dictPtr += litlengthHeaderSize;
     }
 
-    if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted);
+    RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted);
     {   int i;
         size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12));
         for (i=0; i<3; i++) {
             U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4;
-            if (rep==0 || rep >= dictContentSize) return ERROR(dictionary_corrupted);
+            RETURN_ERROR_IF(rep==0 || rep >= dictContentSize,
+                            dictionary_corrupted);
             entropy->rep[i] = rep;
     }   }
 
@@ -1030,7 +1029,7 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
 
     /* load entropy tables */
     {   size_t const eSize = ZSTD_loadDEntropy(&dctx->entropy, dict, dictSize);
-        if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted);
+        RETURN_ERROR_IF(ZSTD_isError(eSize), dictionary_corrupted);
         dict = (const char*)dict + eSize;
         dictSize -= eSize;
     }
@@ -1176,11 +1175,11 @@ size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx,
                                          ZSTD_dictLoadMethod_e dictLoadMethod,
                                          ZSTD_dictContentType_e dictContentType)
 {
-    if (dctx->streamStage != zdss_init) return ERROR(stage_wrong);
+    RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong);
     ZSTD_freeDDict(dctx->ddictLocal);
     if (dict && dictSize >= 8) {
         dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem);
-        if (dctx->ddictLocal == NULL) return ERROR(memory_allocation);
+        RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation);
     } else {
         dctx->ddictLocal = NULL;
     }
@@ -1254,7 +1253,7 @@ size_t ZSTD_resetDStream(ZSTD_DStream* dctx)
 
 size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
 {
-    if (dctx->streamStage != zdss_init) return ERROR(stage_wrong);
+    RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong);
     dctx->ddict = ddict;
     return 0;
 }
@@ -1267,9 +1266,9 @@ size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize)
     ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax);
     size_t const min = (size_t)1 << bounds.lowerBound;
     size_t const max = (size_t)1 << bounds.upperBound;
-    if (dctx->streamStage != zdss_init) return ERROR(stage_wrong);
-    if (maxWindowSize < min) return ERROR(parameter_outOfBound);
-    if (maxWindowSize > max) return ERROR(parameter_outOfBound);
+    RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong);
+    RETURN_ERROR_IF(maxWindowSize < min, parameter_outOfBound);
+    RETURN_ERROR_IF(maxWindowSize > max, parameter_outOfBound);
     dctx->maxWindowSize = maxWindowSize;
     return 0;
 }
@@ -1311,13 +1310,12 @@ static int ZSTD_dParam_withinBounds(ZSTD_dParameter dParam, int value)
 }
 
 #define CHECK_DBOUNDS(p,v) {                \
-    if (!ZSTD_dParam_withinBounds(p, v))    \
-        return ERROR(parameter_outOfBound); \
+    RETURN_ERROR_IF(!ZSTD_dParam_withinBounds(p, v), parameter_outOfBound); \
 }
 
 size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value)
 {
-    if (dctx->streamStage != zdss_init) return ERROR(stage_wrong);
+    RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong);
     switch(dParam) {
         case ZSTD_d_windowLogMax:
             CHECK_DBOUNDS(ZSTD_d_windowLogMax, value);
@@ -1329,7 +1327,7 @@ size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value
             return 0;
         default:;
     }
-    return ERROR(parameter_unsupported);
+    RETURN_ERROR_IF(1, parameter_unsupported);
 }
 
 size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset)
@@ -1340,8 +1338,7 @@ size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset)
     }
     if ( (reset == ZSTD_reset_parameters)
       || (reset == ZSTD_reset_session_and_parameters) ) {
-        if (dctx->streamStage != zdss_init)
-            return ERROR(stage_wrong);
+        RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong);
         dctx->format = ZSTD_f_zstd1;
         dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
     }
@@ -1360,7 +1357,8 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
     unsigned long long const neededRBSize = windowSize + blockSize + (WILDCOPY_OVERLENGTH * 2);
     unsigned long long const neededSize = MIN(frameContentSize, neededRBSize);
     size_t const minRBSize = (size_t) neededSize;
-    if ((unsigned long long)minRBSize != neededSize) return ERROR(frameParameter_windowTooLarge);
+    RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize,
+                    frameParameter_windowTooLarge);
     return minRBSize;
 }
 
@@ -1378,9 +1376,9 @@ size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize)
     ZSTD_frameHeader zfh;
     size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize);
     if (ZSTD_isError(err)) return err;
-    if (err>0) return ERROR(srcSize_wrong);
-    if (zfh.windowSize > windowSizeMax)
-        return ERROR(frameParameter_windowTooLarge);
+    RETURN_ERROR_IF(err>0, srcSize_wrong);
+    RETURN_ERROR_IF(zfh.windowSize > windowSizeMax,
+                    frameParameter_windowTooLarge);
     return ZSTD_estimateDStreamSize((size_t)zfh.windowSize);
 }
 
@@ -1406,16 +1404,16 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
     U32 someMoreWork = 1;
 
     DEBUGLOG(5, "ZSTD_decompressStream");
-    if (input->pos > input->size) {  /* forbidden */
-        DEBUGLOG(5, "in: pos: %u   vs size: %u",
-                    (U32)input->pos, (U32)input->size);
-        return ERROR(srcSize_wrong);
-    }
-    if (output->pos > output->size) {  /* forbidden */
-        DEBUGLOG(5, "out: pos: %u   vs size: %u",
-                    (U32)output->pos, (U32)output->size);
-        return ERROR(dstSize_tooSmall);
-    }
+    RETURN_ERROR_IF(
+        input->pos > input->size,
+        srcSize_wrong,
+        "forbidden. in: pos: %u   vs size: %u",
+        (U32)input->pos, (U32)input->size);
+    RETURN_ERROR_IF(
+        output->pos > output->size,
+        dstSize_tooSmall,
+        "forbidden. out: pos: %u   vs size: %u",
+        (U32)output->pos, (U32)output->size);
     DEBUGLOG(5, "input size : %u", (U32)(input->size - input->pos));
 
     while (someMoreWork) {
@@ -1430,8 +1428,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
             DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip));
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
             if (zds->legacyVersion) {
-                /* legacy support is incompatible with static dctx */
-                if (zds->staticSize) return ERROR(memory_allocation);
+                RETURN_ERROR_IF(zds->staticSize, memory_allocation,
+                    "legacy support is incompatible with static dctx");
                 {   size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input);
                     if (hint==0) zds->streamStage = zdss_init;
                     return hint;
@@ -1446,8 +1444,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                         const void* const dict = zds->ddict ? ZSTD_DDict_dictContent(zds->ddict) : NULL;
                         size_t const dictSize = zds->ddict ? ZSTD_DDict_dictSize(zds->ddict) : 0;
                         DEBUGLOG(5, "ZSTD_decompressStream: detected legacy version v0.%u", legacyVersion);
-                        /* legacy support is incompatible with static dctx */
-                        if (zds->staticSize) return ERROR(memory_allocation);
+                        RETURN_ERROR_IF(zds->staticSize, memory_allocation,
+                            "legacy support is incompatible with static dctx");
                         CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext,
                                     zds->previousLegacyVersion, legacyVersion,
                                     dict, dictSize));
@@ -1511,7 +1509,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                         (U32)(zds->fParams.windowSize >>10),
                         (U32)(zds->maxWindowSize >> 10) );
             zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
-            if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge);
+            RETURN_ERROR_IF(zds->fParams.windowSize > zds->maxWindowSize,
+                            frameParameter_windowTooLarge);
 
             /* Adapt buffer sizes to frame header instructions */
             {   size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */);
@@ -1525,14 +1524,15 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                     if (zds->staticSize) {  /* static DCtx */
                         DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize);
                         assert(zds->staticSize >= sizeof(ZSTD_DCtx));  /* controlled at init */
-                        if (bufferSize > zds->staticSize - sizeof(ZSTD_DCtx))
-                            return ERROR(memory_allocation);
+                        RETURN_ERROR_IF(
+                            bufferSize > zds->staticSize - sizeof(ZSTD_DCtx),
+                            memory_allocation);
                     } else {
                         ZSTD_free(zds->inBuff, zds->customMem);
                         zds->inBuffSize = 0;
                         zds->outBuffSize = 0;
                         zds->inBuff = (char*)ZSTD_malloc(bufferSize, zds->customMem);
-                        if (zds->inBuff == NULL) return ERROR(memory_allocation);
+                        RETURN_ERROR_IF(zds->inBuff == NULL, memory_allocation);
                     }
                     zds->inBuffSize = neededInBuffSize;
                     zds->outBuff = zds->inBuff + zds->inBuffSize;
@@ -1574,7 +1574,9 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                 if (isSkipFrame) {
                     loadedSize = MIN(toLoad, (size_t)(iend-ip));
                 } else {
-                    if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected);   /* should never happen */
+                    RETURN_ERROR_IF(toLoad > zds->inBuffSize - zds->inPos,
+                                    corruption_detected,
+                                    "should never happen");
                     loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend-ip);
                 }
                 ip += loadedSize;
@@ -1615,7 +1617,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
 
         default:
             assert(0);    /* impossible */
-            return ERROR(GENERIC);   /* some compiler require default to do something */
+            RETURN_ERROR_IF(1, GENERIC);   /* some compiler require default to do something */
     }   }
 
     /* result */
@@ -1624,8 +1626,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
     if ((ip==istart) && (op==ostart)) {  /* no forward progress */
         zds->noForwardProgress ++;
         if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) {
-            if (op==oend) return ERROR(dstSize_tooSmall);
-            if (ip==iend) return ERROR(srcSize_wrong);
+            RETURN_ERROR_IF(op==oend, dstSize_tooSmall);
+            RETURN_ERROR_IF(ip==iend, srcSize_wrong);
             assert(0);
         }
     } else {

From a3538bbc6f4b8557fe3bf2bfc657f00e3817fd83 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Mon, 28 Jan 2019 12:34:08 -0500
Subject: [PATCH 033/178] Add RETURN_ERROR and FORWARD_ERROR Macros

---
 lib/common/zstd_internal.h | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index 73fc7705b94..bd534d71553 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -56,6 +56,11 @@ extern "C" {
 #define CHECK_F(f) { size_t const errcod = f; if (ERR_isError(errcod)) return errcod; }  /* check and Forward error code */
 #define CHECK_E(f, e) { size_t const errcod = f; if (ERR_isError(errcod)) return ERROR(e); }  /* check and send Error code */
 
+/**
+ * Return the specified error if the condition evaluates to true.
+ *
+ * In debug modes, prints additional information.
+ */
 #define RETURN_ERROR_IF(cond, err, ...) \
   if (cond) { \
     RAWLOG(3, "%s:%d: check %s failed, returning %s", __FILE__, __LINE__, ZSTD_QUOTE(cond), ZSTD_QUOTE(ERROR(err))); \
@@ -64,6 +69,35 @@ extern "C" {
     return ERROR(err); \
   }
 
+/**
+ * Unconditionally return the specified error.
+ *
+ * In debug modes, prints additional information.
+ */
+#define RETURN_ERROR(err, ...) \
+  do { \
+    RAWLOG(3, "%s:%d: unconditional check failed, returning %s", __FILE__, __LINE__, ZSTD_QUOTE(ERROR(err))); \
+    RAWLOG(3, ": " __VA_ARGS__); \
+    RAWLOG(3, "\n"); \
+    return ERROR(err); \
+  } while(0);
+
+/**
+ * If the provided expression evaluates to an error code, returns that error code.
+ *
+ * In debug modes, prints additional information.
+ */
+#define FORWARD_ERROR(err, ...) \
+  do { \
+    size_t const err_code = (err); \
+    if (ERR_isError(err_code)) { \
+      RAWLOG(3, "%s:%d: forwarding error in %s: %s", __FILE__, __LINE__, ZSTD_QUOTE(err), ERR_getErrorName(err_code)); \
+      RAWLOG(3, ": " __VA_ARGS__); \
+      RAWLOG(3, "\n"); \
+      return err_code; \
+    } \
+  } while(0);
+
 
 /*-*************************************
 *  Common constants

From 800c87fed03a1720ca8af74a7a2a71cfcde31e85 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Mon, 28 Jan 2019 12:35:56 -0500
Subject: [PATCH 034/178] Switch Unconditional RETURN_ERROR_IF Calls to
 RETURN_ERROR

---
 lib/decompress/zstd_decompress.c       | 16 ++++++++--------
 lib/decompress/zstd_decompress_block.c |  4 ++--
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index ae976e11451..18c3bc0333e 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -251,7 +251,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s
             zfhPtr->frameType = ZSTD_skippableFrame;
             return 0;
         }
-        RETURN_ERROR_IF(1, prefix_unknown);
+        RETURN_ERROR(prefix_unknown);
     }
 
     /* ensure there is enough `srcSize` to fully read/decode frame header */
@@ -520,7 +520,7 @@ static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity,
     DEBUGLOG(5, "ZSTD_copyRawBlock");
     if (dst == NULL) {
         if (srcSize == 0) return 0;
-        RETURN_ERROR_IF(1, dstBuffer_null);
+        RETURN_ERROR(dstBuffer_null);
     }
     RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall);
     memcpy(dst, src, srcSize);
@@ -533,7 +533,7 @@ static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity,
 {
     if (dst == NULL) {
         if (regenSize == 0) return 0;
-        RETURN_ERROR_IF(1, dstBuffer_null);
+        RETURN_ERROR(dstBuffer_null);
     }
     RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall);
     memset(dst, b, regenSize);
@@ -595,7 +595,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
             break;
         case bt_reserved :
         default:
-            RETURN_ERROR_IF(1, corruption_detected);
+            RETURN_ERROR(corruption_detected);
         }
 
         if (ZSTD_isError(decodedSize)) return decodedSize;
@@ -865,7 +865,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
                 break;
             case bt_reserved :   /* should never happen */
             default:
-                RETURN_ERROR_IF(1, corruption_detected);
+                RETURN_ERROR(corruption_detected);
             }
             if (ZSTD_isError(rSize)) return rSize;
             DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize);
@@ -919,7 +919,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
 
     default:
         assert(0);   /* impossible */
-        RETURN_ERROR_IF(1, GENERIC);   /* some compiler require default to do something */
+        RETURN_ERROR(GENERIC);   /* some compiler require default to do something */
     }
 }
 
@@ -1327,7 +1327,7 @@ size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value
             return 0;
         default:;
     }
-    RETURN_ERROR_IF(1, parameter_unsupported);
+    RETURN_ERROR(parameter_unsupported);
 }
 
 size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset)
@@ -1617,7 +1617,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
 
         default:
             assert(0);    /* impossible */
-            RETURN_ERROR_IF(1, GENERIC);   /* some compiler require default to do something */
+            RETURN_ERROR(GENERIC);   /* some compiler require default to do something */
     }   }
 
     /* result */
diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c
index b1932e5dede..3c592c08d22 100644
--- a/lib/decompress/zstd_decompress_block.c
+++ b/lib/decompress/zstd_decompress_block.c
@@ -227,7 +227,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
                 return lhSize+1;
             }
         default:
-            RETURN_ERROR_IF(1, corruption_detected, "impossible");
+            RETURN_ERROR(corruption_detected, "impossible");
         }
     }
 }
@@ -470,7 +470,7 @@ static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymb
         }
     default :
         assert(0);
-        RETURN_ERROR_IF(1, GENERIC, "impossible");
+        RETURN_ERROR(GENERIC, "impossible");
     }
 }
 

From 32fed9c7beaf3ecdc922d57598f51ec303973ebe Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Mon, 28 Jan 2019 12:42:00 -0500
Subject: [PATCH 035/178] Switch CHECK_F Calls to FORWARD_ERROR

---
 lib/decompress/zstd_decompress.c | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 18c3bc0333e..7ffe2e6e111 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -567,7 +567,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
         if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize;
         RETURN_ERROR_IF(remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize,
                         srcSize_wrong);
-        CHECK_F( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) );
+        FORWARD_ERROR( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) );
         ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize;
     }
 
@@ -684,11 +684,11 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
 
         if (ddict) {
             /* we were called from ZSTD_decompress_usingDDict */
-            CHECK_F(ZSTD_decompressBegin_usingDDict(dctx, ddict));
+            FORWARD_ERROR(ZSTD_decompressBegin_usingDDict(dctx, ddict));
         } else {
             /* this will initialize correctly with no dict if dict == NULL, so
              * use this in all cases but ddict */
-            CHECK_F(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize));
+            FORWARD_ERROR(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize));
         }
         ZSTD_checkContinuity(dctx, dst);
 
@@ -815,7 +815,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
     case ZSTDds_decodeFrameHeader:
         assert(src != NULL);
         memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize);
-        CHECK_F(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize));
+        FORWARD_ERROR(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize));
         dctx->expected = ZSTD_blockHeaderSize;
         dctx->stage = ZSTDds_decodeBlockHeader;
         return 0;
@@ -1063,7 +1063,7 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
 
 size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
 {
-    CHECK_F( ZSTD_decompressBegin(dctx) );
+    FORWARD_ERROR( ZSTD_decompressBegin(dctx) );
     if (dict && dictSize)
         CHECK_E(ZSTD_decompress_insertDictionary(dctx, dict, dictSize), dictionary_corrupted);
     return 0;
@@ -1084,7 +1084,7 @@ size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
         DEBUGLOG(4, "DDict is %s",
                     dctx->ddictIsCold ? "~cold~" : "hot!");
     }
-    CHECK_F( ZSTD_decompressBegin(dctx) );
+    FORWARD_ERROR( ZSTD_decompressBegin(dctx) );
     if (ddict) {   /* NULL ddict is equivalent to no dictionary */
         ZSTD_copyDDictParameters(dctx, ddict);
     }
@@ -1216,7 +1216,7 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
     DEBUGLOG(4, "ZSTD_initDStream_usingDict");
     zds->streamStage = zdss_init;
     zds->noForwardProgress = 0;
-    CHECK_F( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) );
+    FORWARD_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) );
     return ZSTD_FRAMEHEADERSIZE_PREFIX;
 }
 
@@ -1446,7 +1446,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                         DEBUGLOG(5, "ZSTD_decompressStream: detected legacy version v0.%u", legacyVersion);
                         RETURN_ERROR_IF(zds->staticSize, memory_allocation,
                             "legacy support is incompatible with static dctx");
-                        CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext,
+                        FORWARD_ERROR(ZSTD_initLegacyStream(&zds->legacyContext,
                                     zds->previousLegacyVersion, legacyVersion,
                                     dict, dictSize));
                         zds->legacyVersion = zds->previousLegacyVersion = legacyVersion;
@@ -1493,13 +1493,13 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
 
             /* Consume header (see ZSTDds_decodeFrameHeader) */
             DEBUGLOG(4, "Consume header");
-            CHECK_F(ZSTD_decompressBegin_usingDDict(zds, zds->ddict));
+            FORWARD_ERROR(ZSTD_decompressBegin_usingDDict(zds, zds->ddict));
 
             if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {  /* skippable frame */
                 zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE);
                 zds->stage = ZSTDds_skipFrame;
             } else {
-                CHECK_F(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize));
+                FORWARD_ERROR(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize));
                 zds->expected = ZSTD_blockHeaderSize;
                 zds->stage = ZSTDds_decodeBlockHeader;
             }

From 324e9654d366a3340f072c259119a78699d0d008 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Mon, 28 Jan 2019 12:50:36 -0500
Subject: [PATCH 036/178] Add grep-able String to Error Macros

---
 lib/common/zstd_internal.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index bd534d71553..abf35d21af6 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -63,7 +63,7 @@ extern "C" {
  */
 #define RETURN_ERROR_IF(cond, err, ...) \
   if (cond) { \
-    RAWLOG(3, "%s:%d: check %s failed, returning %s", __FILE__, __LINE__, ZSTD_QUOTE(cond), ZSTD_QUOTE(ERROR(err))); \
+    RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", __FILE__, __LINE__, ZSTD_QUOTE(cond), ZSTD_QUOTE(ERROR(err))); \
     RAWLOG(3, ": " __VA_ARGS__); \
     RAWLOG(3, "\n"); \
     return ERROR(err); \
@@ -76,7 +76,7 @@ extern "C" {
  */
 #define RETURN_ERROR(err, ...) \
   do { \
-    RAWLOG(3, "%s:%d: unconditional check failed, returning %s", __FILE__, __LINE__, ZSTD_QUOTE(ERROR(err))); \
+    RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", __FILE__, __LINE__, ZSTD_QUOTE(ERROR(err))); \
     RAWLOG(3, ": " __VA_ARGS__); \
     RAWLOG(3, "\n"); \
     return ERROR(err); \
@@ -91,7 +91,7 @@ extern "C" {
   do { \
     size_t const err_code = (err); \
     if (ERR_isError(err_code)) { \
-      RAWLOG(3, "%s:%d: forwarding error in %s: %s", __FILE__, __LINE__, ZSTD_QUOTE(err), ERR_getErrorName(err_code)); \
+      RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", __FILE__, __LINE__, ZSTD_QUOTE(err), ERR_getErrorName(err_code)); \
       RAWLOG(3, ": " __VA_ARGS__); \
       RAWLOG(3, "\n"); \
       return err_code; \

From cafc3b1bcbba548cff44543ff40fdc4f090c6534 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Mon, 28 Jan 2019 17:05:18 -0500
Subject: [PATCH 037/178] Also Convert zstd_compress.c

---
 lib/compress/zstd_compress.c | 306 +++++++++++++++++------------------
 1 file changed, 153 insertions(+), 153 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index ef55f032388..758a453dbb5 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -117,7 +117,8 @@ static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx)
 size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
 {
     if (cctx==NULL) return 0;   /* support free on NULL */
-    if (cctx->staticSize) return ERROR(memory_allocation);   /* not compatible with static CCtx */
+    RETURN_ERROR_IF(cctx->staticSize, memory_allocation,
+                    "not compatible with static CCtx");
     ZSTD_freeCCtxContent(cctx);
     ZSTD_free(cctx, cctx->customMem);
     return 0;
@@ -195,7 +196,7 @@ size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params)
 }
 
 size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel) {
-    if (!cctxParams) { return ERROR(GENERIC); }
+    RETURN_ERROR_IF(!cctxParams, GENERIC);
     memset(cctxParams, 0, sizeof(*cctxParams));
     cctxParams->compressionLevel = compressionLevel;
     cctxParams->fParams.contentSizeFlag = 1;
@@ -204,8 +205,8 @@ size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel)
 
 size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params)
 {
-    if (!cctxParams) { return ERROR(GENERIC); }
-    CHECK_F( ZSTD_checkCParams(params.cParams) );
+    RETURN_ERROR_IF(!cctxParams, GENERIC);
+    FORWARD_ERROR( ZSTD_checkCParams(params.cParams) );
     memset(cctxParams, 0, sizeof(*cctxParams));
     cctxParams->cParams = params.cParams;
     cctxParams->fParams = params.fParams;
@@ -378,10 +379,10 @@ static int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value)
     return 1;
 }
 
-#define BOUNDCHECK(cParam, val) {                  \
-    if (!ZSTD_cParam_withinBounds(cParam,val)) {   \
-        return ERROR(parameter_outOfBound);        \
-}   }
+#define BOUNDCHECK(cParam, val) { \
+    RETURN_ERROR_IF(!ZSTD_cParam_withinBounds(cParam,val), \
+                    parameter_outOfBound); \
+}
 
 
 static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param)
@@ -425,7 +426,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
         if (ZSTD_isUpdateAuthorized(param)) {
             cctx->cParamsChanged = 1;
         } else {
-            return ERROR(stage_wrong);
+            RETURN_ERROR(stage_wrong);
     }   }
 
     switch(param)
@@ -434,7 +435,8 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
         return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
 
     case ZSTD_c_compressionLevel:
-        if (cctx->cdict) return ERROR(stage_wrong);
+        RETURN_ERROR_IF(cctx->cdict, stage_wrong,
+                        "compression level is configured in cdict");
         return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
 
     case ZSTD_c_windowLog:
@@ -444,7 +446,8 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
     case ZSTD_c_minMatch:
     case ZSTD_c_targetLength:
     case ZSTD_c_strategy:
-        if (cctx->cdict) return ERROR(stage_wrong);
+        RETURN_ERROR_IF(cctx->cdict, stage_wrong,
+                        "cparams are configured in cdict");
         return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
 
     case ZSTD_c_contentSizeFlag:
@@ -461,9 +464,8 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
         return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
 
     case ZSTD_c_nbWorkers:
-        if ((value!=0) && cctx->staticSize) {
-            return ERROR(parameter_unsupported);  /* MT not compatible with static alloc */
-        }
+        RETURN_ERROR_IF((value!=0) && cctx->staticSize, parameter_unsupported,
+                        "MT not compatible with static alloc");
         return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
 
     case ZSTD_c_jobSize:
@@ -476,10 +478,10 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
     case ZSTD_c_ldmMinMatch:
     case ZSTD_c_ldmBucketSizeLog:
     case ZSTD_c_ldmHashRateLog:
-        if (cctx->cdict) return ERROR(stage_wrong);
+        RETURN_ERROR_IF(cctx->cdict, stage_wrong);
         return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
 
-    default: return ERROR(parameter_unsupported);
+    default: RETURN_ERROR(parameter_unsupported);
     }
 }
 
@@ -575,7 +577,7 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams,
 
     case ZSTD_c_nbWorkers :
 #ifndef ZSTD_MULTITHREAD
-        if (value!=0) return ERROR(parameter_unsupported);
+        RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading");
         return 0;
 #else
         return ZSTDMT_CCtxParam_setNbWorkers(CCtxParams, value);
@@ -583,21 +585,21 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams,
 
     case ZSTD_c_jobSize :
 #ifndef ZSTD_MULTITHREAD
-        return ERROR(parameter_unsupported);
+        RETURN_ERROR(parameter_unsupported, "not compiled with multithreading");
 #else
         return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_jobSize, value);
 #endif
 
     case ZSTD_c_overlapLog :
 #ifndef ZSTD_MULTITHREAD
-        return ERROR(parameter_unsupported);
+        RETURN_ERROR(parameter_unsupported, "not compiled with multithreading");
 #else
         return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_overlapLog, value);
 #endif
 
     case ZSTD_c_rsyncable :
 #ifndef ZSTD_MULTITHREAD
-        return ERROR(parameter_unsupported);
+        RETURN_ERROR(parameter_unsupported, "not compiled with multithreading");
 #else
         return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_rsyncable, value);
 #endif
@@ -625,12 +627,12 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams,
         return CCtxParams->ldmParams.bucketSizeLog;
 
     case ZSTD_c_ldmHashRateLog :
-        if (value > ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN)
-            return ERROR(parameter_outOfBound);
+        RETURN_ERROR_IF(value > ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN,
+                        parameter_outOfBound);
         CCtxParams->ldmParams.hashRateLog = value;
         return CCtxParams->ldmParams.hashRateLog;
 
-    default: return ERROR(parameter_unsupported);
+    default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
     }
 }
 
@@ -694,7 +696,7 @@ size_t ZSTD_CCtxParam_getParameter(
         break;
     case ZSTD_c_jobSize :
 #ifndef ZSTD_MULTITHREAD
-        return ERROR(parameter_unsupported);
+        RETURN_ERROR(parameter_unsupported, "not compiled with multithreading");
 #else
         assert(CCtxParams->jobSize <= INT_MAX);
         *value = (int)CCtxParams->jobSize;
@@ -702,14 +704,14 @@ size_t ZSTD_CCtxParam_getParameter(
 #endif
     case ZSTD_c_overlapLog :
 #ifndef ZSTD_MULTITHREAD
-        return ERROR(parameter_unsupported);
+        RETURN_ERROR(parameter_unsupported, "not compiled with multithreading");
 #else
         *value = CCtxParams->overlapLog;
         break;
 #endif
     case ZSTD_c_rsyncable :
 #ifndef ZSTD_MULTITHREAD
-        return ERROR(parameter_unsupported);
+        RETURN_ERROR(parameter_unsupported, "not compiled with multithreading");
 #else
         *value = CCtxParams->rsyncable;
         break;
@@ -729,7 +731,7 @@ size_t ZSTD_CCtxParam_getParameter(
     case ZSTD_c_ldmHashRateLog :
         *value = CCtxParams->ldmParams.hashRateLog;
         break;
-    default: return ERROR(parameter_unsupported);
+    default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
     }
     return 0;
 }
@@ -745,8 +747,8 @@ size_t ZSTD_CCtx_setParametersUsingCCtxParams(
         ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params)
 {
     DEBUGLOG(4, "ZSTD_CCtx_setParametersUsingCCtxParams");
-    if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
-    if (cctx->cdict) return ERROR(stage_wrong);
+    RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong);
+    RETURN_ERROR_IF(cctx->cdict, stage_wrong);
 
     cctx->requestedParams = *params;
     return 0;
@@ -755,7 +757,7 @@ size_t ZSTD_CCtx_setParametersUsingCCtxParams(
 ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize)
 {
     DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %u bytes", (U32)pledgedSrcSize);
-    if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
+    RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong);
     cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1;
     return 0;
 }
@@ -764,8 +766,9 @@ size_t ZSTD_CCtx_loadDictionary_advanced(
         ZSTD_CCtx* cctx, const void* dict, size_t dictSize,
         ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType)
 {
-    if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
-    if (cctx->staticSize) return ERROR(memory_allocation);  /* no malloc for static CCtx */
+    RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong);
+    RETURN_ERROR_IF(cctx->staticSize, memory_allocation,
+                    "no malloc for static CCtx");
     DEBUGLOG(4, "ZSTD_CCtx_loadDictionary_advanced (size: %u)", (U32)dictSize);
     ZSTD_freeCDict(cctx->cdictLocal);  /* in case one already exists */
     if (dict==NULL || dictSize==0) {   /* no dictionary mode */
@@ -779,8 +782,7 @@ size_t ZSTD_CCtx_loadDictionary_advanced(
                                 dictLoadMethod, dictContentType,
                                 cParams, cctx->customMem);
         cctx->cdict = cctx->cdictLocal;
-        if (cctx->cdictLocal == NULL)
-            return ERROR(memory_allocation);
+        RETURN_ERROR_IF(cctx->cdictLocal == NULL, memory_allocation);
     }
     return 0;
 }
@@ -801,7 +803,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s
 
 size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
 {
-    if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
+    RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong);
     cctx->cdict = cdict;
     memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict));  /* exclusive */
     return 0;
@@ -815,7 +817,7 @@ size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSiz
 size_t ZSTD_CCtx_refPrefix_advanced(
         ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType)
 {
-    if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
+    RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong);
     cctx->cdict = NULL;   /* prefix discards any prior cdict */
     cctx->prefixDict.dict = prefix;
     cctx->prefixDict.dictSize = prefixSize;
@@ -834,7 +836,7 @@ size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset)
     }
     if ( (reset == ZSTD_reset_parameters)
       || (reset == ZSTD_reset_session_and_parameters) ) {
-        if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
+        RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong);
         cctx->cdict = NULL;
         return ZSTD_CCtxParams_reset(&cctx->requestedParams);
     }
@@ -974,8 +976,7 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams,
 
 size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params)
 {
-    /* Estimate CCtx size is supported for single-threaded compression only. */
-    if (params->nbWorkers > 0) { return ERROR(GENERIC); }
+    RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only.");
     {   ZSTD_compressionParameters const cParams =
                 ZSTD_getCParamsFromCCtxParams(params, 0, 0);
         size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
@@ -1023,7 +1024,7 @@ size_t ZSTD_estimateCCtxSize(int compressionLevel)
 
 size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
 {
-    if (params->nbWorkers > 0) { return ERROR(GENERIC); }
+    RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only.");
     {   size_t const CCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params);
         size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params->cParams.windowLog);
         size_t const inBuffSize = ((size_t)1 << params->cParams.windowLog) + blockSize;
@@ -1368,13 +1369,13 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
                 DEBUGLOG(4, "Need to resize workSpaceSize from %zuKB to %zuKB",
                             zc->workSpaceSize >> 10,
                             neededSpace >> 10);
-                /* static cctx : no resize, error out */
-                if (zc->staticSize) return ERROR(memory_allocation);
+
+                RETURN_ERROR_IF(zc->staticSize, memory_allocation, "static cctx : no resize");
 
                 zc->workSpaceSize = 0;
                 ZSTD_free(zc->workSpace, zc->customMem);
                 zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem);
-                if (zc->workSpace == NULL) return ERROR(memory_allocation);
+                RETURN_ERROR_IF(zc->workSpace == NULL, memory_allocation);
                 zc->workSpaceSize = neededSpace;
                 zc->workSpaceOversizedDuration = 0;
 
@@ -1645,7 +1646,7 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
                             ZSTD_buffered_policy_e zbuff)
 {
     DEBUGLOG(5, "ZSTD_copyCCtx_internal");
-    if (srcCCtx->stage!=ZSTDcs_init) return ERROR(stage_wrong);
+    RETURN_ERROR_IF(srcCCtx->stage!=ZSTDcs_init, stage_wrong);
 
     memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
     {   ZSTD_CCtx_params params = dstCCtx->requestedParams;
@@ -1778,7 +1779,8 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue)
 static size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastBlock)
 {
     U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(srcSize << 3);
-    if (srcSize + ZSTD_blockHeaderSize > dstCapacity) return ERROR(dstSize_tooSmall);
+    RETURN_ERROR_IF(srcSize + ZSTD_blockHeaderSize > dstCapacity,
+                    dstSize_tooSmall);
     MEM_writeLE24(dst, cBlockHeader24);
     memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize);
     return ZSTD_blockHeaderSize + srcSize;
@@ -1789,7 +1791,7 @@ static size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void
     BYTE* const ostart = (BYTE* const)dst;
     U32   const flSize = 1 + (srcSize>31) + (srcSize>4095);
 
-    if (srcSize + flSize > dstCapacity) return ERROR(dstSize_tooSmall);
+    RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall);
 
     switch(flSize)
     {
@@ -1879,7 +1881,7 @@ static size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
         if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
     }
 
-    if (dstCapacity < lhSize+1) return ERROR(dstSize_tooSmall);   /* not enough space for compression */
+    RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression");
     {   HUF_repeat repeat = prevHuf->repeatMode;
         int const preferRepeat = strategy < ZSTD_lazy ? srcSize <= 1024 : 0;
         if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1;
@@ -2051,21 +2053,17 @@ static size_t ZSTD_fseBitCost(
     unsigned s;
     FSE_CState_t cstate;
     FSE_initCState(&cstate, ctable);
-    if (ZSTD_getFSEMaxSymbolValue(ctable) < max) {
-        DEBUGLOG(5, "Repeat FSE_CTable has maxSymbolValue %u < %u",
+    RETURN_ERROR_IF(ZSTD_getFSEMaxSymbolValue(ctable) < max, GENERIC,
+                    "Repeat FSE_CTable has maxSymbolValue %u < %u",
                     ZSTD_getFSEMaxSymbolValue(ctable), max);
-        return ERROR(GENERIC);
-    }
     for (s = 0; s <= max; ++s) {
         unsigned const tableLog = cstate.stateLog;
         unsigned const badCost = (tableLog + 1) << kAccuracyLog;
         unsigned const bitCost = FSE_bitCost(cstate.symbolTT, tableLog, s, kAccuracyLog);
         if (count[s] == 0)
             continue;
-        if (bitCost >= badCost) {
-            DEBUGLOG(5, "Repeat FSE_CTable has Prob[%u] == 0", s);
-            return ERROR(GENERIC);
-        }
+        RETURN_ERROR_IF(bitCost >= badCost, GENERIC,
+                        "Repeat FSE_CTable has Prob[%u] == 0", s);
         cost += count[s] * bitCost;
     }
     return cost >> kAccuracyLog;
@@ -2081,7 +2079,7 @@ static size_t ZSTD_NCountCost(unsigned const* count, unsigned const max,
     BYTE wksp[FSE_NCOUNTBOUND];
     S16 norm[MaxSeq + 1];
     const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max);
-    CHECK_F(FSE_normalizeCount(norm, tableLog, count, nbSeq, max));
+    FORWARD_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max));
     return FSE_writeNCount(wksp, sizeof(wksp), norm, max, tableLog);
 }
 
@@ -2187,15 +2185,15 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity,
 
     switch (type) {
     case set_rle:
-        CHECK_F(FSE_buildCTable_rle(nextCTable, (BYTE)max));
-        if (dstCapacity==0) return ERROR(dstSize_tooSmall);
+        FORWARD_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max));
+        RETURN_ERROR_IF(dstCapacity==0, dstSize_tooSmall);
         *op = codeTable[0];
         return 1;
     case set_repeat:
         memcpy(nextCTable, prevCTable, prevCTableSize);
         return 0;
     case set_basic:
-        CHECK_F(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, workspace, workspaceSize));  /* note : could be pre-calculated */
+        FORWARD_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, workspace, workspaceSize));  /* note : could be pre-calculated */
         return 0;
     case set_compressed: {
         S16 norm[MaxSeq + 1];
@@ -2206,14 +2204,14 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity,
             nbSeq_1--;
         }
         assert(nbSeq_1 > 1);
-        CHECK_F(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max));
+        FORWARD_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max));
         {   size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog);   /* overflow protected */
-            if (FSE_isError(NCountSize)) return NCountSize;
-            CHECK_F(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, workspace, workspaceSize));
+            FORWARD_ERROR(NCountSize);
+            FORWARD_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, workspace, workspaceSize));
             return NCountSize;
         }
     }
-    default: return assert(0), ERROR(GENERIC);
+    default: assert(0); RETURN_ERROR(GENERIC);
     }
 }
 
@@ -2304,7 +2302,7 @@ ZSTD_encodeSequences_body(
     FSE_flushCState(&blockStream, &stateLitLength);
 
     {   size_t const streamSize = BIT_closeCStream(&blockStream);
-        if (streamSize==0) return ERROR(dstSize_tooSmall);   /* not enough space */
+        RETURN_ERROR_IF(streamSize==0, dstSize_tooSmall, "not enough space");
         return streamSize;
     }
 }
@@ -2412,14 +2410,14 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
                                     literals, litSize,
                                     workspace, wkspSize,
                                     bmi2);
-        if (ZSTD_isError(cSize))
-          return cSize;
+        FORWARD_ERROR(cSize);
         assert(cSize <= dstCapacity);
         op += cSize;
     }
 
     /* Sequences Header */
-    if ((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/) return ERROR(dstSize_tooSmall);
+    RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/,
+                    dstSize_tooSmall);
     if (nbSeq < 0x7F)
         *op++ = (BYTE)nbSeq;
     else if (nbSeq < LONGNBSEQ)
@@ -2453,7 +2451,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
                                                     count, max, llCodeTable, nbSeq, LL_defaultNorm, LL_defaultNormLog, MaxLL,
                                                     prevEntropy->fse.litlengthCTable, sizeof(prevEntropy->fse.litlengthCTable),
                                                     workspace, wkspSize);
-            if (ZSTD_isError(countSize)) return countSize;
+            FORWARD_ERROR(countSize);
             if (LLtype == set_compressed)
                 lastNCount = op;
             op += countSize;
@@ -2475,7 +2473,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
                                                     count, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
                                                     prevEntropy->fse.offcodeCTable, sizeof(prevEntropy->fse.offcodeCTable),
                                                     workspace, wkspSize);
-            if (ZSTD_isError(countSize)) return countSize;
+            FORWARD_ERROR(countSize);
             if (Offtype == set_compressed)
                 lastNCount = op;
             op += countSize;
@@ -2495,7 +2493,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
                                                     count, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML,
                                                     prevEntropy->fse.matchlengthCTable, sizeof(prevEntropy->fse.matchlengthCTable),
                                                     workspace, wkspSize);
-            if (ZSTD_isError(countSize)) return countSize;
+            FORWARD_ERROR(countSize);
             if (MLtype == set_compressed)
                 lastNCount = op;
             op += countSize;
@@ -2510,7 +2508,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
                                         CTable_LitLength, llCodeTable,
                                         sequences, nbSeq,
                                         longOffsets, bmi2);
-        if (ZSTD_isError(bitstreamSize)) return bitstreamSize;
+        FORWARD_ERROR(bitstreamSize);
         op += bitstreamSize;
         /* zstd versions <= 1.3.4 mistakenly report corruption when
          * FSE_readNCount() recieves a buffer < 4 bytes.
@@ -2553,7 +2551,7 @@ ZSTD_compressSequences(seqStore_t* seqStorePtr,
      */
     if ((cSize == ERROR(dstSize_tooSmall)) & (srcSize <= dstCapacity))
         return 0;  /* block not compressed */
-    if (ZSTD_isError(cSize)) return cSize;
+    FORWARD_ERROR(cSize);
 
     /* Check compressibility */
     {   size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, cctxParams->cParams.strategy);
@@ -2680,7 +2678,7 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
             ldmSeqStore.seq = zc->ldmSequences;
             ldmSeqStore.capacity = zc->maxNbLdmSequences;
             /* Updates ldmSeqStore.size */
-            CHECK_F(ZSTD_ldm_generateSequences(&zc->ldmState, &ldmSeqStore,
+            FORWARD_ERROR(ZSTD_ldm_generateSequences(&zc->ldmState, &ldmSeqStore,
                                                &zc->appliedParams.ldmParams,
                                                src, srcSize));
             /* Updates ldmSeqStore.pos */
@@ -2753,8 +2751,9 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
         ZSTD_matchState_t* const ms = &cctx->blockState.matchState;
         U32 const lastBlock = lastFrameChunk & (blockSize >= remaining);
 
-        if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE)
-            return ERROR(dstSize_tooSmall);   /* not enough space to store compressed block */
+        RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE,
+                        dstSize_tooSmall,
+                        "not enough space to store compressed block");
         if (remaining < blockSize) blockSize = remaining;
 
         if (ZSTD_window_needOverflowCorrection(ms->window, ip + blockSize)) {
@@ -2775,11 +2774,11 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
         {   size_t cSize = ZSTD_compressBlock_internal(cctx,
                                 op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize,
                                 ip, blockSize);
-            if (ZSTD_isError(cSize)) return cSize;
+            FORWARD_ERROR(cSize);
 
             if (cSize == 0) {  /* block is not compressible */
                 cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock);
-                if (ZSTD_isError(cSize)) return cSize;
+                FORWARD_ERROR(cSize);
             } else {
                 U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
                 MEM_writeLE24(op, cBlockHeader24);
@@ -2816,7 +2815,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
     size_t pos=0;
 
     assert(!(params.fParams.contentSizeFlag && pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN));
-    if (dstCapacity < ZSTD_FRAMEHEADERSIZE_MAX) return ERROR(dstSize_tooSmall);
+    RETURN_ERROR_IF(dstCapacity < ZSTD_FRAMEHEADERSIZE_MAX, dstSize_tooSmall);
     DEBUGLOG(4, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u",
                 !params.fParams.noDictIDFlag, (unsigned)dictID, (unsigned)dictIDSizeCode);
 
@@ -2852,7 +2851,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
  */
 size_t ZSTD_writeLastEmptyBlock(void* dst, size_t dstCapacity)
 {
-    if (dstCapacity < ZSTD_blockHeaderSize) return ERROR(dstSize_tooSmall);
+    RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall);
     {   U32 const cBlockHeader24 = 1 /*lastBlock*/ + (((U32)bt_raw)<<1);  /* 0 size */
         MEM_writeLE24(dst, cBlockHeader24);
         return ZSTD_blockHeaderSize;
@@ -2861,10 +2860,9 @@ size_t ZSTD_writeLastEmptyBlock(void* dst, size_t dstCapacity)
 
 size_t ZSTD_referenceExternalSequences(ZSTD_CCtx* cctx, rawSeq* seq, size_t nbSeq)
 {
-    if (cctx->stage != ZSTDcs_init)
-        return ERROR(stage_wrong);
-    if (cctx->appliedParams.ldmParams.enableLdm)
-        return ERROR(parameter_unsupported);
+    RETURN_ERROR_IF(cctx->stage != ZSTDcs_init, stage_wrong);
+    RETURN_ERROR_IF(cctx->appliedParams.ldmParams.enableLdm,
+                    parameter_unsupported);
     cctx->externSeqStore.seq = seq;
     cctx->externSeqStore.size = nbSeq;
     cctx->externSeqStore.capacity = nbSeq;
@@ -2883,12 +2881,13 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
 
     DEBUGLOG(5, "ZSTD_compressContinue_internal, stage: %u, srcSize: %u",
                 cctx->stage, (unsigned)srcSize);
-    if (cctx->stage==ZSTDcs_created) return ERROR(stage_wrong);   /* missing init (ZSTD_compressBegin) */
+    RETURN_ERROR_IF(cctx->stage==ZSTDcs_created, stage_wrong,
+                    "missing init (ZSTD_compressBegin)");
 
     if (frame && (cctx->stage==ZSTDcs_init)) {
         fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->appliedParams,
                                        cctx->pledgedSrcSizePlusOne-1, cctx->dictID);
-        if (ZSTD_isError(fhSize)) return fhSize;
+        FORWARD_ERROR(fhSize);
         dstCapacity -= fhSize;
         dst = (char*)dst + fhSize;
         cctx->stage = ZSTDcs_ongoing;
@@ -2923,17 +2922,18 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
     {   size_t const cSize = frame ?
                              ZSTD_compress_frameChunk (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) :
                              ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize);
-        if (ZSTD_isError(cSize)) return cSize;
+        FORWARD_ERROR(cSize);
         cctx->consumedSrcSize += srcSize;
         cctx->producedCSize += (cSize + fhSize);
         assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0));
         if (cctx->pledgedSrcSizePlusOne != 0) {  /* control src size */
             ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1);
-            if (cctx->consumedSrcSize+1 > cctx->pledgedSrcSizePlusOne) {
-                DEBUGLOG(4, "error : pledgedSrcSize = %u, while realSrcSize >= %u",
-                    (unsigned)cctx->pledgedSrcSizePlusOne-1, (unsigned)cctx->consumedSrcSize);
-                return ERROR(srcSize_wrong);
-            }
+            RETURN_ERROR_IF(
+                cctx->consumedSrcSize+1 > cctx->pledgedSrcSizePlusOne,
+                srcSize_wrong,
+                "error : pledgedSrcSize = %u, while realSrcSize >= %u",
+                (unsigned)cctx->pledgedSrcSizePlusOne-1,
+                (unsigned)cctx->consumedSrcSize);
         }
         return cSize + fhSize;
     }
@@ -2958,7 +2958,7 @@ size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx)
 size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
     size_t const blockSizeMax = ZSTD_getBlockSize(cctx);
-    if (srcSize > blockSizeMax) return ERROR(srcSize_wrong);
+    RETURN_ERROR_IF(srcSize > blockSizeMax, srcSize_wrong);
 
     return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */);
 }
@@ -3021,9 +3021,9 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
    NOTE: This behavior is not standard and could be improved in the future. */
 static size_t ZSTD_checkDictNCount(short* normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue) {
     U32 s;
-    if (dictMaxSymbolValue < maxSymbolValue) return ERROR(dictionary_corrupted);
+    RETURN_ERROR_IF(dictMaxSymbolValue < maxSymbolValue, dictionary_corrupted);
     for (s = 0; s <= maxSymbolValue; ++s) {
-        if (normalizedCounter[s] == 0) return ERROR(dictionary_corrupted);
+        RETURN_ERROR_IF(normalizedCounter[s] == 0, dictionary_corrupted);
     }
     return 0;
 }
@@ -3061,15 +3061,15 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
 
     {   unsigned maxSymbolValue = 255;
         size_t const hufHeaderSize = HUF_readCTable((HUF_CElt*)bs->entropy.huf.CTable, &maxSymbolValue, dictPtr, dictEnd-dictPtr);
-        if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted);
-        if (maxSymbolValue < 255) return ERROR(dictionary_corrupted);
+        RETURN_ERROR_IF(HUF_isError(hufHeaderSize), dictionary_corrupted);
+        RETURN_ERROR_IF(maxSymbolValue < 255, dictionary_corrupted);
         dictPtr += hufHeaderSize;
     }
 
     {   unsigned offcodeLog;
         size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
-        if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
-        if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted);
+        RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted);
+        RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted);
         /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */
         /* fill all offset symbols to avoid garbage at end of table */
         CHECK_E( FSE_buildCTable_wksp(bs->entropy.fse.offcodeCTable,
@@ -3082,10 +3082,10 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
     {   short matchlengthNCount[MaxML+1];
         unsigned matchlengthMaxValue = MaxML, matchlengthLog;
         size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
-        if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
-        if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted);
+        RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted);
+        RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted);
         /* Every match length code must have non-zero probability */
-        CHECK_F( ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML));
+        FORWARD_ERROR( ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML));
         CHECK_E( FSE_buildCTable_wksp(bs->entropy.fse.matchlengthCTable,
                                     matchlengthNCount, matchlengthMaxValue, matchlengthLog,
                                     workspace, HUF_WORKSPACE_SIZE),
@@ -3096,10 +3096,10 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
     {   short litlengthNCount[MaxLL+1];
         unsigned litlengthMaxValue = MaxLL, litlengthLog;
         size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
-        if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
-        if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted);
+        RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted);
+        RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted);
         /* Every literal length code must have non-zero probability */
-        CHECK_F( ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL));
+        FORWARD_ERROR( ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL));
         CHECK_E( FSE_buildCTable_wksp(bs->entropy.fse.litlengthCTable,
                                     litlengthNCount, litlengthMaxValue, litlengthLog,
                                     workspace, HUF_WORKSPACE_SIZE),
@@ -3107,7 +3107,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
         dictPtr += litlengthHeaderSize;
     }
 
-    if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted);
+    RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted);
     bs->rep[0] = MEM_readLE32(dictPtr+0);
     bs->rep[1] = MEM_readLE32(dictPtr+4);
     bs->rep[2] = MEM_readLE32(dictPtr+8);
@@ -3120,19 +3120,19 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
             offcodeMax = ZSTD_highbit32(maxOffset); /* Calculate minimum offset code required to represent maxOffset */
         }
         /* All offset values <= dictContentSize + 128 KB must be representable */
-        CHECK_F (ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff)));
+        FORWARD_ERROR(ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff)));
         /* All repCodes must be <= dictContentSize and != 0*/
         {   U32 u;
             for (u=0; u<3; u++) {
-                if (bs->rep[u] == 0) return ERROR(dictionary_corrupted);
-                if (bs->rep[u] > dictContentSize) return ERROR(dictionary_corrupted);
+                RETURN_ERROR_IF(bs->rep[u] == 0, dictionary_corrupted);
+                RETURN_ERROR_IF(bs->rep[u] > dictContentSize, dictionary_corrupted);
         }   }
 
         bs->entropy.huf.repeatMode = HUF_repeat_valid;
         bs->entropy.fse.offcode_repeatMode = FSE_repeat_valid;
         bs->entropy.fse.matchlength_repeatMode = FSE_repeat_valid;
         bs->entropy.fse.litlength_repeatMode = FSE_repeat_valid;
-        CHECK_F(ZSTD_loadDictionaryContent(ms, params, dictPtr, dictContentSize, dtlm));
+        FORWARD_ERROR(ZSTD_loadDictionaryContent(ms, params, dictPtr, dictContentSize, dtlm));
         return dictID;
     }
 }
@@ -3162,8 +3162,7 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs,
             DEBUGLOG(4, "raw content dictionary detected");
             return ZSTD_loadDictionaryContent(ms, params, dict, dictSize, dtlm);
         }
-        if (dictContentType == ZSTD_dct_fullDict)
-            return ERROR(dictionary_wrong);
+        RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong);
         assert(0);   /* impossible */
     }
 
@@ -3190,13 +3189,13 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
         return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff);
     }
 
-    CHECK_F( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
+    FORWARD_ERROR( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
                                      ZSTDcrp_continue, zbuff) );
     {
         size_t const dictID = ZSTD_compress_insertDictionary(
                 cctx->blockState.prevCBlock, &cctx->blockState.matchState,
                 ¶ms, dict, dictSize, dictContentType, dtlm, cctx->entropyWorkspace);
-        if (ZSTD_isError(dictID)) return dictID;
+        FORWARD_ERROR(dictID);
         assert(dictID <= (size_t)(U32)-1);
         cctx->dictID = (U32)dictID;
     }
@@ -3213,7 +3212,7 @@ size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx,
 {
     DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal: wlog=%u", params.cParams.windowLog);
     /* compression parameters verification and optimization */
-    CHECK_F( ZSTD_checkCParams(params.cParams) );
+    FORWARD_ERROR( ZSTD_checkCParams(params.cParams) );
     return ZSTD_compressBegin_internal(cctx,
                                        dict, dictSize, dictContentType, dtlm,
                                        cdict,
@@ -3261,12 +3260,12 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
     size_t fhSize = 0;
 
     DEBUGLOG(4, "ZSTD_writeEpilogue");
-    if (cctx->stage == ZSTDcs_created) return ERROR(stage_wrong);  /* init missing */
+    RETURN_ERROR_IF(cctx->stage == ZSTDcs_created, stage_wrong, "init missing");
 
     /* special case : empty frame */
     if (cctx->stage == ZSTDcs_init) {
         fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->appliedParams, 0, 0);
-        if (ZSTD_isError(fhSize)) return fhSize;
+        FORWARD_ERROR(fhSize);
         dstCapacity -= fhSize;
         op += fhSize;
         cctx->stage = ZSTDcs_ongoing;
@@ -3275,7 +3274,7 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
     if (cctx->stage != ZSTDcs_ending) {
         /* write one last empty block, make it the "last" block */
         U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1) + 0;
-        if (dstCapacity<4) return ERROR(dstSize_tooSmall);
+        RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall);
         MEM_writeLE32(op, cBlockHeader24);
         op += ZSTD_blockHeaderSize;
         dstCapacity -= ZSTD_blockHeaderSize;
@@ -3283,7 +3282,7 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
 
     if (cctx->appliedParams.fParams.checksumFlag) {
         U32 const checksum = (U32) XXH64_digest(&cctx->xxhState);
-        if (dstCapacity<4) return ERROR(dstSize_tooSmall);
+        RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall);
         DEBUGLOG(4, "ZSTD_writeEpilogue: write checksum : %08X", (unsigned)checksum);
         MEM_writeLE32(op, checksum);
         op += 4;
@@ -3301,18 +3300,20 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
     size_t const cSize = ZSTD_compressContinue_internal(cctx,
                                 dst, dstCapacity, src, srcSize,
                                 1 /* frame mode */, 1 /* last chunk */);
-    if (ZSTD_isError(cSize)) return cSize;
+    FORWARD_ERROR(cSize);
     endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize);
-    if (ZSTD_isError(endResult)) return endResult;
+    FORWARD_ERROR(endResult);
     assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0));
     if (cctx->pledgedSrcSizePlusOne != 0) {  /* control src size */
         ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1);
         DEBUGLOG(4, "end of frame : controlling src size");
-        if (cctx->pledgedSrcSizePlusOne != cctx->consumedSrcSize+1) {
-            DEBUGLOG(4, "error : pledgedSrcSize = %u, while realSrcSize = %u",
-                (unsigned)cctx->pledgedSrcSizePlusOne-1, (unsigned)cctx->consumedSrcSize);
-            return ERROR(srcSize_wrong);
-    }   }
+        RETURN_ERROR_IF(
+            cctx->pledgedSrcSizePlusOne != cctx->consumedSrcSize+1,
+            srcSize_wrong,
+             "error : pledgedSrcSize = %u, while realSrcSize = %u",
+            (unsigned)cctx->pledgedSrcSizePlusOne-1,
+            (unsigned)cctx->consumedSrcSize);
+    }
     return cSize + endResult;
 }
 
@@ -3340,7 +3341,7 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx,
                                ZSTD_parameters params)
 {
     DEBUGLOG(4, "ZSTD_compress_advanced");
-    CHECK_F(ZSTD_checkCParams(params.cParams));
+    FORWARD_ERROR(ZSTD_checkCParams(params.cParams));
     return ZSTD_compress_internal(cctx,
                                   dst, dstCapacity,
                                   src, srcSize,
@@ -3357,7 +3358,7 @@ size_t ZSTD_compress_advanced_internal(
         ZSTD_CCtx_params params)
 {
     DEBUGLOG(4, "ZSTD_compress_advanced_internal (srcSize:%u)", (unsigned)srcSize);
-    CHECK_F( ZSTD_compressBegin_internal(cctx,
+    FORWARD_ERROR( ZSTD_compressBegin_internal(cctx,
                          dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL,
                          params, srcSize, ZSTDb_not_buffered) );
     return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
@@ -3441,7 +3442,7 @@ static size_t ZSTD_initCDict_internal(
         void* const internalBuffer = ZSTD_malloc(dictSize, cdict->customMem);
         cdict->dictBuffer = internalBuffer;
         cdict->dictContent = internalBuffer;
-        if (!internalBuffer) return ERROR(memory_allocation);
+        RETURN_ERROR_IF(!internalBuffer, memory_allocation);
         memcpy(internalBuffer, dictBuffer, dictSize);
     }
     cdict->dictContentSize = dictSize;
@@ -3467,7 +3468,7 @@ static size_t ZSTD_initCDict_internal(
                     &cdict->cBlockState, &cdict->matchState, ¶ms,
                     cdict->dictContent, cdict->dictContentSize,
                     dictContentType, ZSTD_dtlm_full, cdict->workspace);
-            if (ZSTD_isError(dictID)) return dictID;
+            FORWARD_ERROR(dictID);
             assert(dictID <= (size_t)(U32)-1);
             cdict->dictID = (U32)dictID;
         }
@@ -3597,7 +3598,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
     ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize)
 {
     DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced");
-    if (cdict==NULL) return ERROR(dictionary_wrong);
+    RETURN_ERROR_IF(cdict==NULL, dictionary_wrong);
     {   ZSTD_CCtx_params params = cctx->requestedParams;
         params.cParams = ZSTD_getCParamsFromCDict(cdict);
         /* Increase window log to fit the entire dictionary and source if the
@@ -3633,7 +3634,7 @@ size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
                                 const void* src, size_t srcSize,
                                 const ZSTD_CDict* cdict, ZSTD_frameParameters fParams)
 {
-    CHECK_F (ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, srcSize));   /* will check if cdict != NULL */
+    FORWARD_ERROR(ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, srcSize));   /* will check if cdict != NULL */
     return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
 }
 
@@ -3701,7 +3702,7 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* cctx,
     assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
     assert(!((dict) && (cdict)));  /* either dict or cdict, not both */
 
-    CHECK_F( ZSTD_compressBegin_internal(cctx,
+    FORWARD_ERROR( ZSTD_compressBegin_internal(cctx,
                                          dict, dictSize, dictContentType, ZSTD_dtlm_fast,
                                          cdict,
                                          params, pledgedSrcSize,
@@ -3743,16 +3744,15 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
 
     if (dict && dictSize >= 8) {
         DEBUGLOG(4, "loading dictionary of size %u", (unsigned)dictSize);
-        if (zcs->staticSize) {   /* static CCtx : never uses malloc */
-            /* incompatible with internal cdict creation */
-            return ERROR(memory_allocation);
-        }
+        RETURN_ERROR_IF(
+            zcs->staticSize, memory_allocation,
+            "static CCtx: incompatible with internal cdict creation");
         ZSTD_freeCDict(zcs->cdictLocal);
         zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
                                             ZSTD_dlm_byCopy, ZSTD_dct_auto,
                                             params.cParams, zcs->customMem);
         zcs->cdict = zcs->cdictLocal;
-        if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
+        RETURN_ERROR_IF(zcs->cdictLocal == NULL, memory_allocation);
     } else {
         if (cdict) {
             params.cParams = ZSTD_getCParamsFromCDict(cdict);  /* cParams are enforced from cdict; it includes windowLog */
@@ -3773,7 +3773,8 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
                                             unsigned long long pledgedSrcSize)
 {
     DEBUGLOG(4, "ZSTD_initCStream_usingCDict_advanced");
-    if (!cdict) return ERROR(dictionary_wrong); /* cannot handle NULL cdict (does not know what to do) */
+    RETURN_ERROR_IF(!cdict, dictionary_wrong,
+                    "cannot handle NULL cdict (does not know what to do)");
     {   ZSTD_CCtx_params params = zcs->requestedParams;
         params.cParams = ZSTD_getCParamsFromCDict(cdict);
         params.fParams = fParams;
@@ -3802,7 +3803,7 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
 {
     DEBUGLOG(4, "ZSTD_initCStream_advanced: pledgedSrcSize=%u, flag=%u",
                 (unsigned)pledgedSrcSize, params.fParams.contentSizeFlag);
-    CHECK_F( ZSTD_checkCParams(params.cParams) );
+    FORWARD_ERROR( ZSTD_checkCParams(params.cParams) );
     if ((pledgedSrcSize==0) && (params.fParams.contentSizeFlag==0)) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;  /* for compatibility with older programs relying on this behavior. Users should now specify ZSTD_CONTENTSIZE_UNKNOWN. This line will be removed in the future. */
     zcs->requestedParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params);
     return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL /*cdict*/, zcs->requestedParams, pledgedSrcSize);
@@ -3874,8 +3875,7 @@ size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
         switch(zcs->streamStage)
         {
         case zcss_init:
-            /* call ZSTD_initCStream() first ! */
-            return ERROR(init_missing);
+            RETURN_ERROR(init_missing, "call ZSTD_initCStream() first!");
 
         case zcss_load:
             if ( (flushMode == ZSTD_e_end)
@@ -3885,7 +3885,7 @@ size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                 size_t const cSize = ZSTD_compressEnd(zcs,
                                                 op, oend-op, ip, iend-ip);
                 DEBUGLOG(4, "ZSTD_compressEnd : cSize=%u", (unsigned)cSize);
-                if (ZSTD_isError(cSize)) return cSize;
+                FORWARD_ERROR(cSize);
                 ip = iend;
                 op += cSize;
                 zcs->frameEnded = 1;
@@ -3926,7 +3926,7 @@ size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                                     zcs->inBuff + zcs->inToCompress, iSize) :
                         ZSTD_compressContinue(zcs, cDst, oSize,
                                     zcs->inBuff + zcs->inToCompress, iSize);
-                if (ZSTD_isError(cSize)) return cSize;
+                FORWARD_ERROR(cSize);
                 zcs->frameEnded = lastBlock;
                 /* prepare next block */
                 zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize;
@@ -4002,7 +4002,7 @@ static size_t ZSTD_nextInputSizeHint_MTorST(const ZSTD_CCtx* cctx)
 
 size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
 {
-    CHECK_F( ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue) );
+    FORWARD_ERROR( ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue) );
     return ZSTD_nextInputSizeHint_MTorST(zcs);
 }
 
@@ -4014,8 +4014,8 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
 {
     DEBUGLOG(5, "ZSTD_compressStream2, endOp=%u ", (unsigned)endOp);
     /* check conditions */
-    if (output->pos > output->size) return ERROR(GENERIC);
-    if (input->pos  > input->size)  return ERROR(GENERIC);
+    RETURN_ERROR_IF(output->pos > output->size, GENERIC);
+    RETURN_ERROR_IF(input->pos  > input->size, GENERIC);
     assert(cctx!=NULL);
 
     /* transparent initialization stage */
@@ -4040,11 +4040,11 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
                 DEBUGLOG(4, "ZSTD_compressStream2: creating new mtctx for nbWorkers=%u",
                             params.nbWorkers);
                 cctx->mtctx = ZSTDMT_createCCtx_advanced(params.nbWorkers, cctx->customMem);
-                if (cctx->mtctx == NULL) return ERROR(memory_allocation);
+                RETURN_ERROR_IF(cctx->mtctx == NULL, memory_allocation);
             }
             /* mt compression */
             DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbWorkers=%u", params.nbWorkers);
-            CHECK_F( ZSTDMT_initCStream_internal(
+            FORWARD_ERROR( ZSTDMT_initCStream_internal(
                         cctx->mtctx,
                         prefixDict.dict, prefixDict.dictSize, ZSTD_dct_rawContent,
                         cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) );
@@ -4052,7 +4052,7 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
             cctx->appliedParams.nbWorkers = params.nbWorkers;
         } else
 #endif
-        {   CHECK_F( ZSTD_resetCStream_internal(cctx,
+        {   FORWARD_ERROR( ZSTD_resetCStream_internal(cctx,
                             prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType,
                             cctx->cdict,
                             params, cctx->pledgedSrcSizePlusOne-1) );
@@ -4077,7 +4077,7 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
             return flushMin;
     }   }
 #endif
-    CHECK_F( ZSTD_compressStream_generic(cctx, output, input, endOp) );
+    FORWARD_ERROR( ZSTD_compressStream_generic(cctx, output, input, endOp) );
     DEBUGLOG(5, "completed ZSTD_compressStream2");
     return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */
 }
@@ -4108,10 +4108,10 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx,
                                         dst, dstCapacity, &oPos,
                                         src, srcSize, &iPos,
                                         ZSTD_e_end);
-        if (ZSTD_isError(result)) return result;
+        FORWARD_ERROR(result);
         if (result != 0) {  /* compression not completed, due to lack of output space */
             assert(oPos == dstCapacity);
-            return ERROR(dstSize_tooSmall);
+            RETURN_ERROR(dstSize_tooSmall);
         }
         assert(iPos == srcSize);   /* all input is expected consumed */
         return oPos;
@@ -4133,7 +4133,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
 {
     ZSTD_inBuffer input = { NULL, 0, 0 };
     size_t const remainingToFlush = ZSTD_compressStream2(zcs, output, &input, ZSTD_e_end);
-    CHECK_F( remainingToFlush );
+    FORWARD_ERROR( remainingToFlush );
     if (zcs->appliedParams.nbWorkers > 0) return remainingToFlush;   /* minimal estimation */
     /* single thread mode : attempt to calculate remaining to flush more precisely */
     {   size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE;

From 64bb6640f21bba1588ad92d46e30ed9177d19b90 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Mon, 28 Jan 2019 17:15:57 -0500
Subject: [PATCH 038/178] Replace CHECK_F Uses in zstdmt_compress.c and
 zstd_ddict.c

---
 lib/compress/zstdmt_compress.c | 12 ++++++------
 lib/decompress/zstd_ddict.c    |  2 +-
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 2cbd6ffade3..958fdb7f5f1 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -1056,7 +1056,7 @@ static ZSTD_CCtx_params ZSTDMT_initJobCCtxParams(ZSTD_CCtx_params const params)
 static size_t ZSTDMT_resize(ZSTDMT_CCtx* mtctx, unsigned nbWorkers)
 {
     if (POOL_resize(mtctx->factory, nbWorkers)) return ERROR(memory_allocation);
-    CHECK_F( ZSTDMT_expandJobsTable(mtctx, nbWorkers) );
+    FORWARD_ERROR( ZSTDMT_expandJobsTable(mtctx, nbWorkers) );
     mtctx->bufPool = ZSTDMT_expandBufferPool(mtctx->bufPool, nbWorkers);
     if (mtctx->bufPool == NULL) return ERROR(memory_allocation);
     mtctx->cctxPool = ZSTDMT_expandCCtxPool(mtctx->cctxPool, nbWorkers);
@@ -1263,7 +1263,7 @@ static size_t ZSTDMT_compress_advanced_internal(
     if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, avgJobSize))
         return ERROR(memory_allocation);
 
-    CHECK_F( ZSTDMT_expandJobsTable(mtctx, nbJobs) );  /* only expands if necessary */
+    FORWARD_ERROR( ZSTDMT_expandJobsTable(mtctx, nbJobs) );  /* only expands if necessary */
 
     {   unsigned u;
         for (u=0; uparams.nbWorkers)
-        CHECK_F( ZSTDMT_resize(mtctx, params.nbWorkers) );
+        FORWARD_ERROR( ZSTDMT_resize(mtctx, params.nbWorkers) );
 
     if (params.jobSize != 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN;
     if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = ZSTDMT_JOBSIZE_MAX;
@@ -2051,7 +2051,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
       || ((endOp == ZSTD_e_end) && (!mtctx->frameEnded)) ) {   /* must finish the frame with a zero-size block */
         size_t const jobSize = mtctx->inBuff.filled;
         assert(mtctx->inBuff.filled <= mtctx->targetSectionSize);
-        CHECK_F( ZSTDMT_createCompressionJob(mtctx, jobSize, endOp) );
+        FORWARD_ERROR( ZSTDMT_createCompressionJob(mtctx, jobSize, endOp) );
     }
 
     /* check for potential compressed data ready to be flushed */
@@ -2065,7 +2065,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
 
 size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
 {
-    CHECK_F( ZSTDMT_compressStream_generic(mtctx, output, input, ZSTD_e_continue) );
+    FORWARD_ERROR( ZSTDMT_compressStream_generic(mtctx, output, input, ZSTD_e_continue) );
 
     /* recommended next input size : fill current input buffer */
     return mtctx->targetSectionSize - mtctx->inBuff.filled;   /* note : could be zero when input buffer is fully filled and no more availability to create new job */
@@ -2082,7 +2082,7 @@ static size_t ZSTDMT_flushStream_internal(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* ou
       || ((endFrame==ZSTD_e_end) && !mtctx->frameEnded)) {  /* need a last 0-size block to end frame */
            DEBUGLOG(5, "ZSTDMT_flushStream_internal : create a new job (%u bytes, end:%u)",
                         (U32)srcSize, (U32)endFrame);
-        CHECK_F( ZSTDMT_createCompressionJob(mtctx, srcSize, endFrame) );
+        FORWARD_ERROR( ZSTDMT_createCompressionJob(mtctx, srcSize, endFrame) );
     }
 
     /* check if there is any data available to flush */
diff --git a/lib/decompress/zstd_ddict.c b/lib/decompress/zstd_ddict.c
index 2ad0440684c..e36c240ddbf 100644
--- a/lib/decompress/zstd_ddict.c
+++ b/lib/decompress/zstd_ddict.c
@@ -133,7 +133,7 @@ static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict,
     ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001);  /* cover both little and big endian */
 
     /* parse dictionary content */
-    CHECK_F( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) );
+    FORWARD_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) );
 
     return 0;
 }

From 7ebd897157540bb16b94208217bb0c909cfd6e39 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Mon, 28 Jan 2019 17:16:32 -0500
Subject: [PATCH 039/178] Remove CHECK_F Macro

---
 lib/common/zstd_internal.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index abf35d21af6..4b265bfa922 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -53,7 +53,7 @@ extern "C" {
 #undef MAX
 #define MIN(a,b) ((a)<(b) ? (a) : (b))
 #define MAX(a,b) ((a)>(b) ? (a) : (b))
-#define CHECK_F(f) { size_t const errcod = f; if (ERR_isError(errcod)) return errcod; }  /* check and Forward error code */
+
 #define CHECK_E(f, e) { size_t const errcod = f; if (ERR_isError(errcod)) return ERROR(e); }  /* check and send Error code */
 
 /**

From 03e040a96641415e63c833c70c9cea488e685082 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Mon, 28 Jan 2019 17:27:29 -0500
Subject: [PATCH 040/178] Replace Uses of CHECK_E with
 RETURN_ERROR_IF(*_isError(...

---
 lib/compress/zstd_compress.c           | 31 +++++++++++++++-----------
 lib/decompress/zstd_ddict.c            |  6 ++---
 lib/decompress/zstd_decompress.c       |  4 +++-
 lib/decompress/zstd_decompress_block.c |  8 +++++--
 tests/decodecorpus.c                   |  4 +++-
 5 files changed, 33 insertions(+), 20 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 758a453dbb5..f70129260da 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2228,7 +2228,9 @@ ZSTD_encodeSequences_body(
     FSE_CState_t  stateOffsetBits;
     FSE_CState_t  stateLitLength;
 
-    CHECK_E(BIT_initCStream(&blockStream, dst, dstCapacity), dstSize_tooSmall); /* not enough space remaining */
+    RETURN_ERROR_IF(
+        ERR_isError(BIT_initCStream(&blockStream, dst, dstCapacity)),
+        dstSize_tooSmall, "not enough space remaining");
     DEBUGLOG(6, "available space for bitstream : %i  (dstCapacity=%u)",
                 (int)(blockStream.endPtr - blockStream.startPtr),
                 (unsigned)dstCapacity);
@@ -3072,10 +3074,11 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
         RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted);
         /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */
         /* fill all offset symbols to avoid garbage at end of table */
-        CHECK_E( FSE_buildCTable_wksp(bs->entropy.fse.offcodeCTable,
-                                    offcodeNCount, MaxOff, offcodeLog,
-                                    workspace, HUF_WORKSPACE_SIZE),
-                 dictionary_corrupted);
+        RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp(
+                bs->entropy.fse.offcodeCTable,
+                offcodeNCount, MaxOff, offcodeLog,
+                workspace, HUF_WORKSPACE_SIZE)),
+            dictionary_corrupted);
         dictPtr += offcodeHeaderSize;
     }
 
@@ -3086,10 +3089,11 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
         RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted);
         /* Every match length code must have non-zero probability */
         FORWARD_ERROR( ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML));
-        CHECK_E( FSE_buildCTable_wksp(bs->entropy.fse.matchlengthCTable,
-                                    matchlengthNCount, matchlengthMaxValue, matchlengthLog,
-                                    workspace, HUF_WORKSPACE_SIZE),
-                 dictionary_corrupted);
+        RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp(
+                bs->entropy.fse.matchlengthCTable,
+                matchlengthNCount, matchlengthMaxValue, matchlengthLog,
+                workspace, HUF_WORKSPACE_SIZE)),
+            dictionary_corrupted);
         dictPtr += matchlengthHeaderSize;
     }
 
@@ -3100,10 +3104,11 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
         RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted);
         /* Every literal length code must have non-zero probability */
         FORWARD_ERROR( ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL));
-        CHECK_E( FSE_buildCTable_wksp(bs->entropy.fse.litlengthCTable,
-                                    litlengthNCount, litlengthMaxValue, litlengthLog,
-                                    workspace, HUF_WORKSPACE_SIZE),
-                 dictionary_corrupted);
+        RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp(
+                bs->entropy.fse.litlengthCTable,
+                litlengthNCount, litlengthMaxValue, litlengthLog,
+                workspace, HUF_WORKSPACE_SIZE)),
+            dictionary_corrupted);
         dictPtr += litlengthHeaderSize;
     }
 
diff --git a/lib/decompress/zstd_ddict.c b/lib/decompress/zstd_ddict.c
index e36c240ddbf..27d8470f732 100644
--- a/lib/decompress/zstd_ddict.c
+++ b/lib/decompress/zstd_ddict.c
@@ -105,9 +105,9 @@ ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict,
     ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE);
 
     /* load entropy tables */
-    CHECK_E( ZSTD_loadDEntropy(&ddict->entropy,
-                                ddict->dictContent, ddict->dictSize),
-             dictionary_corrupted );
+    RETURN_ERROR_IF(ZSTD_isError(ZSTD_loadDEntropy(
+            &ddict->entropy, ddict->dictContent, ddict->dictSize)),
+        dictionary_corrupted);
     ddict->entropyPresent = 1;
     return 0;
 }
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 7ffe2e6e111..e08639d1565 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1065,7 +1065,9 @@ size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t
 {
     FORWARD_ERROR( ZSTD_decompressBegin(dctx) );
     if (dict && dictSize)
-        CHECK_E(ZSTD_decompress_insertDictionary(dctx, dict, dictSize), dictionary_corrupted);
+        RETURN_ERROR_IF(
+            ZSTD_isError(ZSTD_decompress_insertDictionary(dctx, dict, dictSize)),
+            dictionary_corrupted);
     return 0;
 }
 
diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c
index 3c592c08d22..4418c51dbd7 100644
--- a/lib/decompress/zstd_decompress_block.c
+++ b/lib/decompress/zstd_decompress_block.c
@@ -911,7 +911,9 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx,
         seqState_t seqState;
         dctx->fseEntropy = 1;
         { U32 i; for (i=0; ientropy.rep[i]; }
-        CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected);
+        RETURN_ERROR_IF(
+            ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)),
+            corruption_detected);
         ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
         ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
         ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
@@ -1066,7 +1068,9 @@ ZSTD_decompressSequencesLong_body(
         seqState.pos = (size_t)(op-prefixStart);
         seqState.dictEnd = dictEnd;
         assert(iend >= ip);
-        CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected);
+        RETURN_ERROR_IF(
+            ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)),
+            corruption_detected);
         ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
         ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
         ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c
index 17c2c18991b..b03dc55eabf 100644
--- a/tests/decodecorpus.c
+++ b/tests/decodecorpus.c
@@ -938,7 +938,9 @@ static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr,
         FSE_CState_t  stateOffsetBits;
         FSE_CState_t  stateLitLength;
 
-        CHECK_E(BIT_initCStream(&blockStream, op, oend-op), dstSize_tooSmall); /* not enough space remaining */
+        RETURN_ERROR_IF(
+            ERR_isError(BIT_initCStream(&blockStream, op, oend-op)),
+            dstSize_tooSmall, "not enough space remaining");
 
         /* first symbols */
         FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]);

From 2179ce00e15f60ba8c56dd34ffb38023e014663c Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Mon, 28 Jan 2019 17:28:14 -0500
Subject: [PATCH 041/178] Remove CHECK_E Macro

---
 lib/common/zstd_internal.h | 2 --
 1 file changed, 2 deletions(-)

diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index 4b265bfa922..2870bdea218 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -54,8 +54,6 @@ extern "C" {
 #define MIN(a,b) ((a)<(b) ? (a) : (b))
 #define MAX(a,b) ((a)>(b) ? (a) : (b))
 
-#define CHECK_E(f, e) { size_t const errcod = f; if (ERR_isError(errcod)) return ERROR(e); }  /* check and send Error code */
-
 /**
  * Return the specified error if the condition evaluates to true.
  *

From 429987c9a67c6b1f0cac94b4327e73a9b1292ed5 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Mon, 28 Jan 2019 17:35:31 -0500
Subject: [PATCH 042/178] Add Comment

---
 lib/common/zstd_internal.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index 2870bdea218..fc3c992e566 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -57,7 +57,9 @@ extern "C" {
 /**
  * Return the specified error if the condition evaluates to true.
  *
- * In debug modes, prints additional information.
+ * In debug modes, prints additional information. In order to do that
+ * (particularly, printing the conditional that failed), this can't just wrap
+ * RETURN_ERROR().
  */
 #define RETURN_ERROR_IF(cond, err, ...) \
   if (cond) { \

From 501eb25102a8590988b9933ad8c2924dddf31c4f Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Tue, 29 Jan 2019 12:56:07 -0500
Subject: [PATCH 043/178] Rename FORWARD_ERROR -> FORWARD_IF_ERROR

---
 lib/common/zstd_internal.h       |  2 +-
 lib/compress/zstd_compress.c     | 84 ++++++++++++++++----------------
 lib/compress/zstdmt_compress.c   | 12 ++---
 lib/decompress/zstd_ddict.c      |  2 +-
 lib/decompress/zstd_decompress.c | 20 ++++----
 5 files changed, 60 insertions(+), 60 deletions(-)

diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index fc3c992e566..c4e2647a24f 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -87,7 +87,7 @@ extern "C" {
  *
  * In debug modes, prints additional information.
  */
-#define FORWARD_ERROR(err, ...) \
+#define FORWARD_IF_ERROR(err, ...) \
   do { \
     size_t const err_code = (err); \
     if (ERR_isError(err_code)) { \
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index f70129260da..2bf8ee535cc 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -206,7 +206,7 @@ size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel)
 size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params)
 {
     RETURN_ERROR_IF(!cctxParams, GENERIC);
-    FORWARD_ERROR( ZSTD_checkCParams(params.cParams) );
+    FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) );
     memset(cctxParams, 0, sizeof(*cctxParams));
     cctxParams->cParams = params.cParams;
     cctxParams->fParams = params.fParams;
@@ -2079,7 +2079,7 @@ static size_t ZSTD_NCountCost(unsigned const* count, unsigned const max,
     BYTE wksp[FSE_NCOUNTBOUND];
     S16 norm[MaxSeq + 1];
     const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max);
-    FORWARD_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max));
+    FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max));
     return FSE_writeNCount(wksp, sizeof(wksp), norm, max, tableLog);
 }
 
@@ -2185,7 +2185,7 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity,
 
     switch (type) {
     case set_rle:
-        FORWARD_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max));
+        FORWARD_IF_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max));
         RETURN_ERROR_IF(dstCapacity==0, dstSize_tooSmall);
         *op = codeTable[0];
         return 1;
@@ -2193,7 +2193,7 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity,
         memcpy(nextCTable, prevCTable, prevCTableSize);
         return 0;
     case set_basic:
-        FORWARD_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, workspace, workspaceSize));  /* note : could be pre-calculated */
+        FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, workspace, workspaceSize));  /* note : could be pre-calculated */
         return 0;
     case set_compressed: {
         S16 norm[MaxSeq + 1];
@@ -2204,10 +2204,10 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity,
             nbSeq_1--;
         }
         assert(nbSeq_1 > 1);
-        FORWARD_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max));
+        FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max));
         {   size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog);   /* overflow protected */
-            FORWARD_ERROR(NCountSize);
-            FORWARD_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, workspace, workspaceSize));
+            FORWARD_IF_ERROR(NCountSize);
+            FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, workspace, workspaceSize));
             return NCountSize;
         }
     }
@@ -2412,7 +2412,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
                                     literals, litSize,
                                     workspace, wkspSize,
                                     bmi2);
-        FORWARD_ERROR(cSize);
+        FORWARD_IF_ERROR(cSize);
         assert(cSize <= dstCapacity);
         op += cSize;
     }
@@ -2453,7 +2453,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
                                                     count, max, llCodeTable, nbSeq, LL_defaultNorm, LL_defaultNormLog, MaxLL,
                                                     prevEntropy->fse.litlengthCTable, sizeof(prevEntropy->fse.litlengthCTable),
                                                     workspace, wkspSize);
-            FORWARD_ERROR(countSize);
+            FORWARD_IF_ERROR(countSize);
             if (LLtype == set_compressed)
                 lastNCount = op;
             op += countSize;
@@ -2475,7 +2475,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
                                                     count, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
                                                     prevEntropy->fse.offcodeCTable, sizeof(prevEntropy->fse.offcodeCTable),
                                                     workspace, wkspSize);
-            FORWARD_ERROR(countSize);
+            FORWARD_IF_ERROR(countSize);
             if (Offtype == set_compressed)
                 lastNCount = op;
             op += countSize;
@@ -2495,7 +2495,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
                                                     count, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML,
                                                     prevEntropy->fse.matchlengthCTable, sizeof(prevEntropy->fse.matchlengthCTable),
                                                     workspace, wkspSize);
-            FORWARD_ERROR(countSize);
+            FORWARD_IF_ERROR(countSize);
             if (MLtype == set_compressed)
                 lastNCount = op;
             op += countSize;
@@ -2510,7 +2510,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
                                         CTable_LitLength, llCodeTable,
                                         sequences, nbSeq,
                                         longOffsets, bmi2);
-        FORWARD_ERROR(bitstreamSize);
+        FORWARD_IF_ERROR(bitstreamSize);
         op += bitstreamSize;
         /* zstd versions <= 1.3.4 mistakenly report corruption when
          * FSE_readNCount() recieves a buffer < 4 bytes.
@@ -2553,7 +2553,7 @@ ZSTD_compressSequences(seqStore_t* seqStorePtr,
      */
     if ((cSize == ERROR(dstSize_tooSmall)) & (srcSize <= dstCapacity))
         return 0;  /* block not compressed */
-    FORWARD_ERROR(cSize);
+    FORWARD_IF_ERROR(cSize);
 
     /* Check compressibility */
     {   size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, cctxParams->cParams.strategy);
@@ -2680,7 +2680,7 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
             ldmSeqStore.seq = zc->ldmSequences;
             ldmSeqStore.capacity = zc->maxNbLdmSequences;
             /* Updates ldmSeqStore.size */
-            FORWARD_ERROR(ZSTD_ldm_generateSequences(&zc->ldmState, &ldmSeqStore,
+            FORWARD_IF_ERROR(ZSTD_ldm_generateSequences(&zc->ldmState, &ldmSeqStore,
                                                &zc->appliedParams.ldmParams,
                                                src, srcSize));
             /* Updates ldmSeqStore.pos */
@@ -2776,11 +2776,11 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
         {   size_t cSize = ZSTD_compressBlock_internal(cctx,
                                 op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize,
                                 ip, blockSize);
-            FORWARD_ERROR(cSize);
+            FORWARD_IF_ERROR(cSize);
 
             if (cSize == 0) {  /* block is not compressible */
                 cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock);
-                FORWARD_ERROR(cSize);
+                FORWARD_IF_ERROR(cSize);
             } else {
                 U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
                 MEM_writeLE24(op, cBlockHeader24);
@@ -2889,7 +2889,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
     if (frame && (cctx->stage==ZSTDcs_init)) {
         fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->appliedParams,
                                        cctx->pledgedSrcSizePlusOne-1, cctx->dictID);
-        FORWARD_ERROR(fhSize);
+        FORWARD_IF_ERROR(fhSize);
         dstCapacity -= fhSize;
         dst = (char*)dst + fhSize;
         cctx->stage = ZSTDcs_ongoing;
@@ -2924,7 +2924,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
     {   size_t const cSize = frame ?
                              ZSTD_compress_frameChunk (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) :
                              ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize);
-        FORWARD_ERROR(cSize);
+        FORWARD_IF_ERROR(cSize);
         cctx->consumedSrcSize += srcSize;
         cctx->producedCSize += (cSize + fhSize);
         assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0));
@@ -3088,7 +3088,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
         RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted);
         RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted);
         /* Every match length code must have non-zero probability */
-        FORWARD_ERROR( ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML));
+        FORWARD_IF_ERROR( ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML));
         RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp(
                 bs->entropy.fse.matchlengthCTable,
                 matchlengthNCount, matchlengthMaxValue, matchlengthLog,
@@ -3103,7 +3103,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
         RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted);
         RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted);
         /* Every literal length code must have non-zero probability */
-        FORWARD_ERROR( ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL));
+        FORWARD_IF_ERROR( ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL));
         RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp(
                 bs->entropy.fse.litlengthCTable,
                 litlengthNCount, litlengthMaxValue, litlengthLog,
@@ -3125,7 +3125,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
             offcodeMax = ZSTD_highbit32(maxOffset); /* Calculate minimum offset code required to represent maxOffset */
         }
         /* All offset values <= dictContentSize + 128 KB must be representable */
-        FORWARD_ERROR(ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff)));
+        FORWARD_IF_ERROR(ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff)));
         /* All repCodes must be <= dictContentSize and != 0*/
         {   U32 u;
             for (u=0; u<3; u++) {
@@ -3137,7 +3137,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
         bs->entropy.fse.offcode_repeatMode = FSE_repeat_valid;
         bs->entropy.fse.matchlength_repeatMode = FSE_repeat_valid;
         bs->entropy.fse.litlength_repeatMode = FSE_repeat_valid;
-        FORWARD_ERROR(ZSTD_loadDictionaryContent(ms, params, dictPtr, dictContentSize, dtlm));
+        FORWARD_IF_ERROR(ZSTD_loadDictionaryContent(ms, params, dictPtr, dictContentSize, dtlm));
         return dictID;
     }
 }
@@ -3194,13 +3194,13 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
         return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff);
     }
 
-    FORWARD_ERROR( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
+    FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
                                      ZSTDcrp_continue, zbuff) );
     {
         size_t const dictID = ZSTD_compress_insertDictionary(
                 cctx->blockState.prevCBlock, &cctx->blockState.matchState,
                 ¶ms, dict, dictSize, dictContentType, dtlm, cctx->entropyWorkspace);
-        FORWARD_ERROR(dictID);
+        FORWARD_IF_ERROR(dictID);
         assert(dictID <= (size_t)(U32)-1);
         cctx->dictID = (U32)dictID;
     }
@@ -3217,7 +3217,7 @@ size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx,
 {
     DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal: wlog=%u", params.cParams.windowLog);
     /* compression parameters verification and optimization */
-    FORWARD_ERROR( ZSTD_checkCParams(params.cParams) );
+    FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) );
     return ZSTD_compressBegin_internal(cctx,
                                        dict, dictSize, dictContentType, dtlm,
                                        cdict,
@@ -3270,7 +3270,7 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
     /* special case : empty frame */
     if (cctx->stage == ZSTDcs_init) {
         fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->appliedParams, 0, 0);
-        FORWARD_ERROR(fhSize);
+        FORWARD_IF_ERROR(fhSize);
         dstCapacity -= fhSize;
         op += fhSize;
         cctx->stage = ZSTDcs_ongoing;
@@ -3305,9 +3305,9 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
     size_t const cSize = ZSTD_compressContinue_internal(cctx,
                                 dst, dstCapacity, src, srcSize,
                                 1 /* frame mode */, 1 /* last chunk */);
-    FORWARD_ERROR(cSize);
+    FORWARD_IF_ERROR(cSize);
     endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize);
-    FORWARD_ERROR(endResult);
+    FORWARD_IF_ERROR(endResult);
     assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0));
     if (cctx->pledgedSrcSizePlusOne != 0) {  /* control src size */
         ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1);
@@ -3346,7 +3346,7 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx,
                                ZSTD_parameters params)
 {
     DEBUGLOG(4, "ZSTD_compress_advanced");
-    FORWARD_ERROR(ZSTD_checkCParams(params.cParams));
+    FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams));
     return ZSTD_compress_internal(cctx,
                                   dst, dstCapacity,
                                   src, srcSize,
@@ -3363,7 +3363,7 @@ size_t ZSTD_compress_advanced_internal(
         ZSTD_CCtx_params params)
 {
     DEBUGLOG(4, "ZSTD_compress_advanced_internal (srcSize:%u)", (unsigned)srcSize);
-    FORWARD_ERROR( ZSTD_compressBegin_internal(cctx,
+    FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx,
                          dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL,
                          params, srcSize, ZSTDb_not_buffered) );
     return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
@@ -3473,7 +3473,7 @@ static size_t ZSTD_initCDict_internal(
                     &cdict->cBlockState, &cdict->matchState, ¶ms,
                     cdict->dictContent, cdict->dictContentSize,
                     dictContentType, ZSTD_dtlm_full, cdict->workspace);
-            FORWARD_ERROR(dictID);
+            FORWARD_IF_ERROR(dictID);
             assert(dictID <= (size_t)(U32)-1);
             cdict->dictID = (U32)dictID;
         }
@@ -3639,7 +3639,7 @@ size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
                                 const void* src, size_t srcSize,
                                 const ZSTD_CDict* cdict, ZSTD_frameParameters fParams)
 {
-    FORWARD_ERROR(ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, srcSize));   /* will check if cdict != NULL */
+    FORWARD_IF_ERROR(ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, srcSize));   /* will check if cdict != NULL */
     return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
 }
 
@@ -3707,7 +3707,7 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* cctx,
     assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
     assert(!((dict) && (cdict)));  /* either dict or cdict, not both */
 
-    FORWARD_ERROR( ZSTD_compressBegin_internal(cctx,
+    FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx,
                                          dict, dictSize, dictContentType, ZSTD_dtlm_fast,
                                          cdict,
                                          params, pledgedSrcSize,
@@ -3808,7 +3808,7 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
 {
     DEBUGLOG(4, "ZSTD_initCStream_advanced: pledgedSrcSize=%u, flag=%u",
                 (unsigned)pledgedSrcSize, params.fParams.contentSizeFlag);
-    FORWARD_ERROR( ZSTD_checkCParams(params.cParams) );
+    FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) );
     if ((pledgedSrcSize==0) && (params.fParams.contentSizeFlag==0)) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;  /* for compatibility with older programs relying on this behavior. Users should now specify ZSTD_CONTENTSIZE_UNKNOWN. This line will be removed in the future. */
     zcs->requestedParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params);
     return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL /*cdict*/, zcs->requestedParams, pledgedSrcSize);
@@ -3890,7 +3890,7 @@ size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                 size_t const cSize = ZSTD_compressEnd(zcs,
                                                 op, oend-op, ip, iend-ip);
                 DEBUGLOG(4, "ZSTD_compressEnd : cSize=%u", (unsigned)cSize);
-                FORWARD_ERROR(cSize);
+                FORWARD_IF_ERROR(cSize);
                 ip = iend;
                 op += cSize;
                 zcs->frameEnded = 1;
@@ -3931,7 +3931,7 @@ size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                                     zcs->inBuff + zcs->inToCompress, iSize) :
                         ZSTD_compressContinue(zcs, cDst, oSize,
                                     zcs->inBuff + zcs->inToCompress, iSize);
-                FORWARD_ERROR(cSize);
+                FORWARD_IF_ERROR(cSize);
                 zcs->frameEnded = lastBlock;
                 /* prepare next block */
                 zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize;
@@ -4007,7 +4007,7 @@ static size_t ZSTD_nextInputSizeHint_MTorST(const ZSTD_CCtx* cctx)
 
 size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
 {
-    FORWARD_ERROR( ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue) );
+    FORWARD_IF_ERROR( ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue) );
     return ZSTD_nextInputSizeHint_MTorST(zcs);
 }
 
@@ -4049,7 +4049,7 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
             }
             /* mt compression */
             DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbWorkers=%u", params.nbWorkers);
-            FORWARD_ERROR( ZSTDMT_initCStream_internal(
+            FORWARD_IF_ERROR( ZSTDMT_initCStream_internal(
                         cctx->mtctx,
                         prefixDict.dict, prefixDict.dictSize, ZSTD_dct_rawContent,
                         cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) );
@@ -4057,7 +4057,7 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
             cctx->appliedParams.nbWorkers = params.nbWorkers;
         } else
 #endif
-        {   FORWARD_ERROR( ZSTD_resetCStream_internal(cctx,
+        {   FORWARD_IF_ERROR( ZSTD_resetCStream_internal(cctx,
                             prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType,
                             cctx->cdict,
                             params, cctx->pledgedSrcSizePlusOne-1) );
@@ -4082,7 +4082,7 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
             return flushMin;
     }   }
 #endif
-    FORWARD_ERROR( ZSTD_compressStream_generic(cctx, output, input, endOp) );
+    FORWARD_IF_ERROR( ZSTD_compressStream_generic(cctx, output, input, endOp) );
     DEBUGLOG(5, "completed ZSTD_compressStream2");
     return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */
 }
@@ -4113,7 +4113,7 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx,
                                         dst, dstCapacity, &oPos,
                                         src, srcSize, &iPos,
                                         ZSTD_e_end);
-        FORWARD_ERROR(result);
+        FORWARD_IF_ERROR(result);
         if (result != 0) {  /* compression not completed, due to lack of output space */
             assert(oPos == dstCapacity);
             RETURN_ERROR(dstSize_tooSmall);
@@ -4138,7 +4138,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
 {
     ZSTD_inBuffer input = { NULL, 0, 0 };
     size_t const remainingToFlush = ZSTD_compressStream2(zcs, output, &input, ZSTD_e_end);
-    FORWARD_ERROR( remainingToFlush );
+    FORWARD_IF_ERROR( remainingToFlush );
     if (zcs->appliedParams.nbWorkers > 0) return remainingToFlush;   /* minimal estimation */
     /* single thread mode : attempt to calculate remaining to flush more precisely */
     {   size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE;
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 958fdb7f5f1..ffa44bc4ee6 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -1056,7 +1056,7 @@ static ZSTD_CCtx_params ZSTDMT_initJobCCtxParams(ZSTD_CCtx_params const params)
 static size_t ZSTDMT_resize(ZSTDMT_CCtx* mtctx, unsigned nbWorkers)
 {
     if (POOL_resize(mtctx->factory, nbWorkers)) return ERROR(memory_allocation);
-    FORWARD_ERROR( ZSTDMT_expandJobsTable(mtctx, nbWorkers) );
+    FORWARD_IF_ERROR( ZSTDMT_expandJobsTable(mtctx, nbWorkers) );
     mtctx->bufPool = ZSTDMT_expandBufferPool(mtctx->bufPool, nbWorkers);
     if (mtctx->bufPool == NULL) return ERROR(memory_allocation);
     mtctx->cctxPool = ZSTDMT_expandCCtxPool(mtctx->cctxPool, nbWorkers);
@@ -1263,7 +1263,7 @@ static size_t ZSTDMT_compress_advanced_internal(
     if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, avgJobSize))
         return ERROR(memory_allocation);
 
-    FORWARD_ERROR( ZSTDMT_expandJobsTable(mtctx, nbJobs) );  /* only expands if necessary */
+    FORWARD_IF_ERROR( ZSTDMT_expandJobsTable(mtctx, nbJobs) );  /* only expands if necessary */
 
     {   unsigned u;
         for (u=0; uparams.nbWorkers)
-        FORWARD_ERROR( ZSTDMT_resize(mtctx, params.nbWorkers) );
+        FORWARD_IF_ERROR( ZSTDMT_resize(mtctx, params.nbWorkers) );
 
     if (params.jobSize != 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN;
     if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = ZSTDMT_JOBSIZE_MAX;
@@ -2051,7 +2051,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
       || ((endOp == ZSTD_e_end) && (!mtctx->frameEnded)) ) {   /* must finish the frame with a zero-size block */
         size_t const jobSize = mtctx->inBuff.filled;
         assert(mtctx->inBuff.filled <= mtctx->targetSectionSize);
-        FORWARD_ERROR( ZSTDMT_createCompressionJob(mtctx, jobSize, endOp) );
+        FORWARD_IF_ERROR( ZSTDMT_createCompressionJob(mtctx, jobSize, endOp) );
     }
 
     /* check for potential compressed data ready to be flushed */
@@ -2065,7 +2065,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
 
 size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
 {
-    FORWARD_ERROR( ZSTDMT_compressStream_generic(mtctx, output, input, ZSTD_e_continue) );
+    FORWARD_IF_ERROR( ZSTDMT_compressStream_generic(mtctx, output, input, ZSTD_e_continue) );
 
     /* recommended next input size : fill current input buffer */
     return mtctx->targetSectionSize - mtctx->inBuff.filled;   /* note : could be zero when input buffer is fully filled and no more availability to create new job */
@@ -2082,7 +2082,7 @@ static size_t ZSTDMT_flushStream_internal(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* ou
       || ((endFrame==ZSTD_e_end) && !mtctx->frameEnded)) {  /* need a last 0-size block to end frame */
            DEBUGLOG(5, "ZSTDMT_flushStream_internal : create a new job (%u bytes, end:%u)",
                         (U32)srcSize, (U32)endFrame);
-        FORWARD_ERROR( ZSTDMT_createCompressionJob(mtctx, srcSize, endFrame) );
+        FORWARD_IF_ERROR( ZSTDMT_createCompressionJob(mtctx, srcSize, endFrame) );
     }
 
     /* check if there is any data available to flush */
diff --git a/lib/decompress/zstd_ddict.c b/lib/decompress/zstd_ddict.c
index 27d8470f732..0af3d23bfe2 100644
--- a/lib/decompress/zstd_ddict.c
+++ b/lib/decompress/zstd_ddict.c
@@ -133,7 +133,7 @@ static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict,
     ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001);  /* cover both little and big endian */
 
     /* parse dictionary content */
-    FORWARD_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) );
+    FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) );
 
     return 0;
 }
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index e08639d1565..601bfe70434 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -567,7 +567,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
         if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize;
         RETURN_ERROR_IF(remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize,
                         srcSize_wrong);
-        FORWARD_ERROR( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) );
+        FORWARD_IF_ERROR( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) );
         ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize;
     }
 
@@ -684,11 +684,11 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
 
         if (ddict) {
             /* we were called from ZSTD_decompress_usingDDict */
-            FORWARD_ERROR(ZSTD_decompressBegin_usingDDict(dctx, ddict));
+            FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(dctx, ddict));
         } else {
             /* this will initialize correctly with no dict if dict == NULL, so
              * use this in all cases but ddict */
-            FORWARD_ERROR(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize));
+            FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize));
         }
         ZSTD_checkContinuity(dctx, dst);
 
@@ -815,7 +815,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
     case ZSTDds_decodeFrameHeader:
         assert(src != NULL);
         memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize);
-        FORWARD_ERROR(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize));
+        FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize));
         dctx->expected = ZSTD_blockHeaderSize;
         dctx->stage = ZSTDds_decodeBlockHeader;
         return 0;
@@ -1063,7 +1063,7 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
 
 size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
 {
-    FORWARD_ERROR( ZSTD_decompressBegin(dctx) );
+    FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) );
     if (dict && dictSize)
         RETURN_ERROR_IF(
             ZSTD_isError(ZSTD_decompress_insertDictionary(dctx, dict, dictSize)),
@@ -1086,7 +1086,7 @@ size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
         DEBUGLOG(4, "DDict is %s",
                     dctx->ddictIsCold ? "~cold~" : "hot!");
     }
-    FORWARD_ERROR( ZSTD_decompressBegin(dctx) );
+    FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) );
     if (ddict) {   /* NULL ddict is equivalent to no dictionary */
         ZSTD_copyDDictParameters(dctx, ddict);
     }
@@ -1218,7 +1218,7 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
     DEBUGLOG(4, "ZSTD_initDStream_usingDict");
     zds->streamStage = zdss_init;
     zds->noForwardProgress = 0;
-    FORWARD_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) );
+    FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) );
     return ZSTD_FRAMEHEADERSIZE_PREFIX;
 }
 
@@ -1448,7 +1448,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                         DEBUGLOG(5, "ZSTD_decompressStream: detected legacy version v0.%u", legacyVersion);
                         RETURN_ERROR_IF(zds->staticSize, memory_allocation,
                             "legacy support is incompatible with static dctx");
-                        FORWARD_ERROR(ZSTD_initLegacyStream(&zds->legacyContext,
+                        FORWARD_IF_ERROR(ZSTD_initLegacyStream(&zds->legacyContext,
                                     zds->previousLegacyVersion, legacyVersion,
                                     dict, dictSize));
                         zds->legacyVersion = zds->previousLegacyVersion = legacyVersion;
@@ -1495,13 +1495,13 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
 
             /* Consume header (see ZSTDds_decodeFrameHeader) */
             DEBUGLOG(4, "Consume header");
-            FORWARD_ERROR(ZSTD_decompressBegin_usingDDict(zds, zds->ddict));
+            FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, zds->ddict));
 
             if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {  /* skippable frame */
                 zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE);
                 zds->stage = ZSTDds_skipFrame;
             } else {
-                FORWARD_ERROR(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize));
+                FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize));
                 zds->expected = ZSTD_blockHeaderSize;
                 zds->stage = ZSTDds_decodeBlockHeader;
             }

From 1a6f2b4f7f599f069062f95f30c1e12681ed5b5f Mon Sep 17 00:00:00 2001
From: Li-Wen Hsu 
Date: Thu, 31 Jan 2019 05:37:04 +0800
Subject: [PATCH 044/178] Change Cirrus CI's badge to dev branch

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index e3f19c5a352..65dcd18213e 100644
--- a/README.md
+++ b/README.md
@@ -22,7 +22,7 @@ a list of known ports and bindings is provided on [Zstandard homepage](http://ww
 [AppveyorLink]: https://ci.appveyor.com/project/YannCollet/zstd-p0yf0
 [CircleDevBadge]: https://circleci.com/gh/facebook/zstd/tree/dev.svg?style=shield "Short test suite"
 [CircleLink]: https://circleci.com/gh/facebook/zstd
-[CirrusDevBadge]: https://api.cirrus-ci.com/github/facebook/zstd.svg
+[CirrusDevBadge]: https://api.cirrus-ci.com/github/facebook/zstd.svg?branch=dev
 [CirrusLink]: https://cirrus-ci.com/github/facebook/zstd
 
 ## Benchmarks

From fb756156f5e75ad13fc8f43f5b8249651f1ce326 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 25 Jan 2019 21:58:44 -0800
Subject: [PATCH 045/178] fixed static size of benchFnState

added a static assert to ensure condition is respected on target platform
---
 programs/benchfn.c | 5 ++++-
 programs/benchfn.h | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/programs/benchfn.c b/programs/benchfn.c
index a27d91a070c..318f4d63b12 100644
--- a/programs/benchfn.c
+++ b/programs/benchfn.c
@@ -180,9 +180,12 @@ void BMK_freeTimedFnState(BMK_timedFnState_t* state) {
 
 BMK_timedFnState_t* BMK_initStatic_timedFnState(void* buffer, size_t size, unsigned total_ms, unsigned run_ms)
 {
+    enum { timedFnState_staticSize_isLargeEnough=(1/(sizeof(BMK_timedFnState_shell) >= sizeof(struct BMK_timedFnState_s))) };  /* static assert */
+    typedef struct { char c; long long ll; } ll_align;  /* this will force ll to be aligned at its next best position */
+    size_t const ll_alignment = offsetof(ll_align, ll); /* provides the minimal alignment restriction for long long */
     BMK_timedFnState_t* const r = (BMK_timedFnState_t*)buffer;
     if (size < sizeof(struct BMK_timedFnState_s)) return NULL;
-    if ((size_t)buffer % 8) return NULL;  /* must be aligned on 8-bytes boundaries */
+    if ((size_t)buffer % ll_alignment) return NULL;  /* must be aligned to satisfy `long long` alignment requirement */
     BMK_resetTimedFnState(r, total_ms, run_ms);
     return r;
 }
diff --git a/programs/benchfn.h b/programs/benchfn.h
index 737dfa9956e..19e056581a5 100644
--- a/programs/benchfn.h
+++ b/programs/benchfn.h
@@ -168,7 +168,7 @@ void BMK_freeTimedFnState(BMK_timedFnState_t* state);
  * It will check if provided buffer is large enough and is correctly aligned,
  * and will return NULL if conditions are not respected.
  */
-#define BMK_TIMEDFNSTATE_SIZE 56
+#define BMK_TIMEDFNSTATE_SIZE 64
 typedef union {
     char never_access_space[BMK_TIMEDFNSTATE_SIZE];
     long long alignment_enforcer;  /* must be aligned on 8-bytes boundaries */

From 2ff9249810a3afe45bcdec571f093dc45fb84eca Mon Sep 17 00:00:00 2001
From: "Peter (Stig) Edwards" 
Date: Fri, 1 Feb 2019 09:27:22 +0000
Subject: [PATCH 046/178] -Wformat-security not needed with -Wformat=2

---
 doc/educational_decoder/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/educational_decoder/Makefile b/doc/educational_decoder/Makefile
index ace1294f8e7..c1d2c4cc42f 100644
--- a/doc/educational_decoder/Makefile
+++ b/doc/educational_decoder/Makefile
@@ -7,7 +7,7 @@ CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
 CFLAGS   ?= -O3
 CFLAGS   += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow                 \
             -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-            -Wstrict-prototypes -Wundef -Wformat-security                   \
+            -Wstrict-prototypes -Wundef                                     \
             -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings      \
             -Wredundant-decls
 CFLAGS   += $(DEBUGFLAGS)

From 2b7120ec719ed825723681bea69ac245bf9936f4 Mon Sep 17 00:00:00 2001
From: "Peter (Stig) Edwards" 
Date: Fri, 1 Feb 2019 09:28:41 +0000
Subject: [PATCH 047/178] -Wformat-security not needed with -Wformat=2

---
 contrib/largeNbDicts/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/largeNbDicts/Makefile b/contrib/largeNbDicts/Makefile
index 541f3969bb9..6fc38256379 100644
--- a/contrib/largeNbDicts/Makefile
+++ b/contrib/largeNbDicts/Makefile
@@ -18,7 +18,7 @@ CFLAGS  ?= -O3
 CFLAGS  += -std=gnu99
 DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
             -Wstrict-aliasing=1 -Wswitch-enum \
-            -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
+            -Wstrict-prototypes -Wundef -Wpointer-arith \
             -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
             -Wredundant-decls
 CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)

From 4a9e0502e632560367a77ba61e9e12fa1d553196 Mon Sep 17 00:00:00 2001
From: "Peter (Stig) Edwards" 
Date: Fri, 1 Feb 2019 09:29:08 +0000
Subject: [PATCH 048/178] -Wformat-security not needed with -Wformat=2

---
 contrib/adaptive-compression/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/adaptive-compression/Makefile b/contrib/adaptive-compression/Makefile
index c26efcd2815..2c6867f5cb7 100644
--- a/contrib/adaptive-compression/Makefile
+++ b/contrib/adaptive-compression/Makefile
@@ -13,7 +13,7 @@ CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
 CFLAGS   ?= -O3
 CFLAGS   += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow                 \
             -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-            -Wstrict-prototypes -Wundef -Wformat-security                   \
+            -Wstrict-prototypes -Wundef                                     \
             -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings      \
             -Wredundant-decls
 CFLAGS   += $(DEBUGFLAGS)

From 723418f8d853c0a2beeeec928e2040b104de2a2f Mon Sep 17 00:00:00 2001
From: "Peter (Stig) Edwards" 
Date: Fri, 1 Feb 2019 09:29:40 +0000
Subject: [PATCH 049/178] -Wformat-security not needed with -Wformat=2

---
 tests/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/Makefile b/tests/Makefile
index 2daf0970f07..2a9cd3d8f64 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -33,7 +33,7 @@ endif
 CFLAGS     ?= -O3
 CFLAGS     += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow                 \
               -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-              -Wstrict-prototypes -Wundef -Wformat-security                   \
+              -Wstrict-prototypes -Wundef                                     \
               -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings      \
               -Wredundant-decls -Wmissing-prototypes
 CFLAGS     += $(DEBUGFLAGS) $(MOREFLAGS)

From 188c7482080aa8d463dca633cabfaf1cebea6d27 Mon Sep 17 00:00:00 2001
From: "Peter (Stig) Edwards" 
Date: Fri, 1 Feb 2019 09:30:31 +0000
Subject: [PATCH 050/178] -Wformat-security not needed with -Wformat=2

---
 build/meson/lib/meson.build | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/meson/lib/meson.build b/build/meson/lib/meson.build
index a02bd2d7a0e..f8014c62586 100644
--- a/build/meson/lib/meson.build
+++ b/build/meson/lib/meson.build
@@ -98,7 +98,7 @@ if use_debug
   if cc_id == compiler_gcc or cc_id == compiler_clang
     libzstd_debug_cflags = ['-Wstrict-aliasing=1', '-Wswitch-enum',
       '-Wdeclaration-after-statement', '-Wstrict-prototypes',
-      '-Wundef', '-Wpointer-arith', '-Wformat-security', '-Wvla',
+      '-Wundef', '-Wpointer-arith', '-Wvla',
       '-Wformat=2', '-Winit-self', '-Wfloat-equal', '-Wwrite-strings',
       '-Wredundant-decls', '-Wmissing-prototypes', '-Wc++-compat']
   endif

From 894bbda44c9a02a67f2a8e6fcae8e039701d685a Mon Sep 17 00:00:00 2001
From: "Peter (Stig) Edwards" 
Date: Fri, 1 Feb 2019 09:31:02 +0000
Subject: [PATCH 051/178] -Wformat-security not needed with -Wformat=2

---
 lib/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/Makefile b/lib/Makefile
index 386583e728b..6a912a6576d 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -25,7 +25,7 @@ endif
 CFLAGS  ?= -O3
 DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
             -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-            -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
+            -Wstrict-prototypes -Wundef -Wpointer-arith \
             -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
             -Wredundant-decls -Wmissing-prototypes -Wc++-compat
 CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)

From 1951c1b3a0579e16a91a4888ba864a5e2e768675 Mon Sep 17 00:00:00 2001
From: "Peter (Stig) Edwards" 
Date: Fri, 1 Feb 2019 09:31:46 +0000
Subject: [PATCH 052/178] -Wformat-security not needed with -Wformat=2

---
 programs/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/Makefile b/programs/Makefile
index d1910fbb4b7..692980e4fee 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -51,7 +51,7 @@ endif
 CFLAGS  ?= -O3
 DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
             -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-            -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
+            -Wstrict-prototypes -Wundef -Wpointer-arith \
             -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
             -Wredundant-decls -Wmissing-prototypes -Wc++-compat
 CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)

From cdb3e7af2fcb3120476b643dafa7da6179097874 Mon Sep 17 00:00:00 2001
From: "Peter (Stig) Edwards" 
Date: Fri, 1 Feb 2019 09:38:49 +0000
Subject: [PATCH 053/178] -Wformat-security not needed with -Wformat=2

---
 tests/fuzz/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/fuzz/Makefile b/tests/fuzz/Makefile
index 4130f18e330..8a22ad1c543 100644
--- a/tests/fuzz/Makefile
+++ b/tests/fuzz/Makefile
@@ -30,7 +30,7 @@ FUZZ_CPPFLAGS := -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
 	$(CPPFLAGS)
 FUZZ_EXTRA_FLAGS := -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
 	-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-	-Wstrict-prototypes -Wundef -Wformat-security \
+	-Wstrict-prototypes -Wundef \
 	-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
 	-Wredundant-decls \
 	-g -fno-omit-frame-pointer

From 21616d8a77ba5791aba3722bec6d8c01197c2e17 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 1 Feb 2019 15:19:32 -0800
Subject: [PATCH 054/178] [zdict] Improve documentation

---
 lib/dictBuilder/zdict.h | 26 ++++++++++++++++++--------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h
index d57d59f01e7..e22973173cb 100644
--- a/lib/dictBuilder/zdict.h
+++ b/lib/dictBuilder/zdict.h
@@ -46,7 +46,12 @@ extern "C" {
  *  The resulting dictionary will be saved into `dictBuffer`.
  * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
  *          or an error code, which can be tested with ZDICT_isError().
- *  Note: ZDICT_trainFromBuffer() requires about 9 bytes of memory for each input byte.
+ *  Note:  Dictionary training will fail if there are not enough samples to construct a
+ *         dictionary, or if most of the samples are too small (< 8 bytes being the lower limit).
+ *         If dictionary training fails, you should use zstd without a dictionary, as the dictionary
+ *         would've been ineffective anyways. If you believe your samples would benefit from a dictionary
+ *         please open an issue with details, and we can look into it.
+ *  Note: ZDICT_trainFromBuffer()'s memory usage is about 6 MB.
  *  Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
  *        It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
  *        In general, it's recommended to provide a few thousands samples, though this can vary a lot.
@@ -110,6 +115,7 @@ typedef struct {
  *  The resulting dictionary will be saved into `dictBuffer`.
  * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
  *          or an error code, which can be tested with ZDICT_isError().
+ *          See ZDICT_trainFromBuffer() for details on failure modes.
  *  Note: ZDICT_trainFromBuffer_cover() requires about 9 bytes of memory for each input byte.
  *  Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
  *        It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
@@ -133,8 +139,9 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
  * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000].
  *
  * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
- *           or an error code, which can be tested with ZDICT_isError().
- *           On success `*parameters` contains the parameters selected.
+ *          or an error code, which can be tested with ZDICT_isError().
+ *          On success `*parameters` contains the parameters selected.
+ *          See ZDICT_trainFromBuffer() for details on failure modes.
  * Note: ZDICT_optimizeTrainFromBuffer_cover() requires about 8 bytes of memory for each input byte and additionally another 5 bytes of memory for each byte of memory for each thread.
  */
 ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
@@ -151,7 +158,8 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
  *  The resulting dictionary will be saved into `dictBuffer`.
  * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
  *          or an error code, which can be tested with ZDICT_isError().
- *  Note: ZDICT_trainFromBuffer_fastCover() requires about 1 bytes of memory for each input byte and additionally another 6 * 2^f bytes of memory .
+ *          See ZDICT_trainFromBuffer() for details on failure modes.
+ *  Note: ZDICT_trainFromBuffer_fastCover() requires 6 * 2^f bytes of memory.
  *  Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
  *        It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
  *        In general, it's recommended to provide a few thousands samples, though this can vary a lot.
@@ -175,9 +183,10 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer,
  * If accel is zero, default value of 1 is used.
  *
  * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
- *           or an error code, which can be tested with ZDICT_isError().
- *           On success `*parameters` contains the parameters selected.
- * Note: ZDICT_optimizeTrainFromBuffer_fastCover() requires about 1 byte of memory for each input byte and additionally another 6 * 2^f bytes of memory for each thread.
+ *          or an error code, which can be tested with ZDICT_isError().
+ *          On success `*parameters` contains the parameters selected.
+ *          See ZDICT_trainFromBuffer() for details on failure modes.
+ * Note: ZDICT_optimizeTrainFromBuffer_fastCover() requires about 6 * 2^f bytes of memory for each thread.
  */
 ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer,
                     size_t dictBufferCapacity, const void* samplesBuffer,
@@ -195,7 +204,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer,
  * maxDictSize must be >= dictContentSize, and must be >= ZDICT_DICTSIZE_MIN bytes.
  *
  * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`),
- *           or an error code, which can be tested by ZDICT_isError().
+ *          or an error code, which can be tested by ZDICT_isError().
  * Note: ZDICT_finalizeDictionary() will push notifications into stderr if instructed to, using notificationLevel>0.
  * Note 2: dictBuffer and dictContent can overlap
  */
@@ -219,6 +228,7 @@ typedef struct {
  * `parameters` is optional and can be provided with values set to 0 to mean "default".
  * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
  *          or an error code, which can be tested with ZDICT_isError().
+ *          See ZDICT_trainFromBuffer() for details on failure modes.
  *  Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
  *        It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
  *        In general, it's recommended to provide a few thousands samples, though this can vary a lot.

From b26c53b0d0560d58b8222df5e889fe39eaec7ecb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Ketelaars?= 
Date: Mon, 11 Feb 2019 11:49:35 +0100
Subject: [PATCH 055/178] 'head -c BYTES' is non-portable.

Pull request #1499 added a new test, which uses 'head -c'. The '-c'
option is non-portable (not in POSIX). Instead use 'dd'. Similar issue
has been resolved in the past (#1321).
---
 tests/playTests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/playTests.sh b/tests/playTests.sh
index 8342455a1e7..6e5582df747 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -815,7 +815,7 @@ FULL_COMPRESSED_FILE=${TEST_DATA_FILE}.zst
 TRUNCATED_COMPRESSED_FILE=truncated-input.txt.zst
 ./datagen -g50000 > $TEST_DATA_FILE
 $ZSTD -f $TEST_DATA_FILE -o $FULL_COMPRESSED_FILE
-head -c 100 $FULL_COMPRESSED_FILE > $TRUNCATED_COMPRESSED_FILE
+dd bs=1 count=100 if=$FULL_COMPRESSED_FILE of=$TRUNCATED_COMPRESSED_FILE status=none
 $ZSTD --list $TRUNCATED_COMPRESSED_FILE && die "-l must fail on truncated file"
 
 rm $TEST_DATA_FILE

From 1e4dc2e5f1964e453942210fb584135de17d7a8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Ketelaars?= 
Date: Tue, 12 Feb 2019 00:03:11 +0100
Subject: [PATCH 056/178] Detect symbolic links on OpenBSD

In #1520 it is described that FreeBSD doesn't detect symbolic links. The
same is true for OpenBSD. This diff fixes this issue for OpenBSD. I'm
guessing that something similar works for FreeBSD as well. However, I'm
unable to test this.
---
 programs/util.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/programs/util.c b/programs/util.c
index 49eea148ec1..0001cc3d83e 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -92,6 +92,7 @@ U32 UTIL_isLink(const char* infilename)
 /* macro guards, as defined in : https://linux.die.net/man/2/lstat */
 #ifndef __STRICT_ANSI__
 #if defined(_BSD_SOURCE) \
+    || defined(__OpenBSD__) \
     || (defined(_XOPEN_SOURCE) && (_XOPEN_SOURCE >= 500)) \
     || (defined(_XOPEN_SOURCE) && defined(_XOPEN_SOURCE_EXTENDED)) \
     || (defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) \

From 482b84f07bd80a917bc8a10a92b2331e680bec56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Ketelaars?= 
Date: Tue, 12 Feb 2019 00:07:32 +0100
Subject: [PATCH 057/178] Make detection of symbolic links more consistent

While fixing the detection of symbolic links on OpenBSD I noticed
inconsistent behaviour:

$ echo hello > hello
$ ln -s hello world
$ zstd hello world
Warning : world is a symbolic link, ignoring
hello                :316.67%   (     6 =>     19 bytes, hello.zst
$ ls *.zst
hello.zst
$ zstd world
world                :316.67%   (     6 =>     19 bytes, world.zst)
$ ls *.zst
hello.zst world.zst
---
 programs/zstdcli.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index ef2fe2e91c7..794e2a8111a 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -951,6 +951,8 @@ int main(int argCount, const char* argv[])
                 filenameTable[fileNamesNb++] = filenameTable[u];
             }
         }
+        if (fileNamesNb == 0 && filenameIdx > 0)
+            CLEAN_RETURN(1);
         filenameIdx = fileNamesNb;
     }
     if (recursive) {  /* at this stage, filenameTable is a list of paths, which can contain both files and directories */

From 77d9109c27959362ea09464cbd17c540406b2cb6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Ketelaars?= 
Date: Tue, 12 Feb 2019 01:14:58 +0100
Subject: [PATCH 058/178] Add test

While here enable symlink test for OpenBSD.
---
 tests/playTests.sh | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/tests/playTests.sh b/tests/playTests.sh
index 6e5582df747..5be065fa892 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -314,18 +314,28 @@ $ECHO foo | $ZSTD > /dev/full && die "write error not detected!"
 $ECHO "$ECHO foo | $ZSTD | $ZSTD -d > /dev/full"
 $ECHO foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!"
 
+fi
+
+
+if [ "$isWindows" = false ] && [ "$UNAME" != 'SunOS' ] ; then
 
 $ECHO "\n===>  symbolic link test "
 
-rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst
+rm -f hello.tmp world.tmp world2.tmp hello.tmp.zst world.tmp.zst
 $ECHO "hello world" > hello.tmp
 ln -s hello.tmp world.tmp
-$ZSTD world.tmp hello.tmp
+ln -s hello.tmp world2.tmp
+$ZSTD world.tmp hello.tmp || true
 test -f hello.tmp.zst  # regular file should have been compressed!
 test ! -f world.tmp.zst  # symbolic link should not have been compressed!
+$ZSTD world.tmp || true
+test ! -f world.tmp.zst  # symbolic link should not have been compressed!
+$ZSTD world.tmp world2.tmp || true
+test ! -f world.tmp.zst  # symbolic link should not have been compressed!
+test ! -f world2.tmp.zst  # symbolic link should not have been compressed!
 $ZSTD world.tmp hello.tmp -f
 test -f world.tmp.zst  # symbolic link should have been compressed with --force
-rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst
+rm -f hello.tmp world.tmp world2.tmp hello.tmp.zst world.tmp.zst
 
 fi
 

From 9c071664ade65c21f1c1fb42c7c54f06c7857cdf Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Mon, 11 Feb 2019 10:32:30 -0800
Subject: [PATCH 059/178] [cirrus] Run playTests.sh

* Pass `-Werror` through `MOREFLAGS`
* Install `coreutils`
* Run `playTests.sh`
---
 .cirrus.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 777d3eca109..506647a009e 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -10,6 +10,7 @@ task:
   install_script:
     - sed -i.bak -e 's,pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly,pkg+http://pkg.FreeBSD.org/\${ABI}/latest,' /etc/pkg/FreeBSD.conf
     - pkg upgrade -y
-    - pkg install -y gmake
-  script:
-    - CFLAGS="-Werror" gmake -j all
+    - pkg install -y gmake coreutils
+  script: |
+    MOREFLAGS="-Werror" gmake -j all
+    gmake shortest

From e1a799e5de563fe6e0739f0694ee8c0079b9fbdd Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Mon, 11 Feb 2019 17:43:43 -0800
Subject: [PATCH 060/178] [util] Fix UTIL_isLink() for FreeBSD

---
 programs/util.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/programs/util.c b/programs/util.c
index 0001cc3d83e..d6466063594 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -92,11 +92,12 @@ U32 UTIL_isLink(const char* infilename)
 /* macro guards, as defined in : https://linux.die.net/man/2/lstat */
 #ifndef __STRICT_ANSI__
 #if defined(_BSD_SOURCE) \
-    || defined(__OpenBSD__) \
     || (defined(_XOPEN_SOURCE) && (_XOPEN_SOURCE >= 500)) \
     || (defined(_XOPEN_SOURCE) && defined(_XOPEN_SOURCE_EXTENDED)) \
     || (defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) \
-    || (defined(__APPLE__) && defined(__MACH__))
+    || (defined(__APPLE__) && defined(__MACH__)) \
+    || defined(__OpenBSD__) \
+    || defined(__FreeBSD__)
     int r;
     stat_t statbuf;
     r = lstat(infilename, &statbuf);

From f9513115e4c0bfbc79b5932382a8db06a155acb9 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Wed, 13 Feb 2019 14:59:22 -0800
Subject: [PATCH 061/178] [libzstd] Add ZSTD_c_literalCompressionMode flag

It controls the literals compression. It is either
`auto`, `huffman`, or `uncompressed`. It defaults to
`auto`, which is the current behavior.
---
 lib/compress/zstd_compress.c          | 39 +++++++++++++++++++++++++--
 lib/compress/zstd_compress_internal.h |  1 +
 lib/zstd.h                            | 19 ++++++++++++-
 3 files changed, 56 insertions(+), 3 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 2bf8ee535cc..e18051f00ce 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -360,6 +360,12 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
         bounds.upperBound = ZSTD_dictForceCopy;       /* note : how to ensure at compile time that this is the highest value enum ? */
         return bounds;
 
+    case ZSTD_c_literalCompressionMode:
+        ZSTD_STATIC_ASSERT(ZSTD_lcm_auto < ZSTD_lcm_huffman && ZSTD_lcm_huffman < ZSTD_lcm_uncompressed);
+        bounds.lowerBound = ZSTD_lcm_auto;
+        bounds.upperBound = ZSTD_lcm_uncompressed;
+        return bounds;
+
     default:
         {   ZSTD_bounds const boundError = { ERROR(parameter_unsupported), 0, 0 };
             return boundError;
@@ -396,6 +402,7 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param)
     case ZSTD_c_minMatch:
     case ZSTD_c_targetLength:
     case ZSTD_c_strategy:
+    case ZSTD_c_literalCompressionMode:
         return 1;
 
     case ZSTD_c_format:
@@ -463,6 +470,9 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
     case ZSTD_c_forceAttachDict:
         return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
 
+    case ZSTD_c_literalCompressionMode:
+        return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
+
     case ZSTD_c_nbWorkers:
         RETURN_ERROR_IF((value!=0) && cctx->staticSize, parameter_unsupported,
                         "MT not compatible with static alloc");
@@ -575,6 +585,13 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams,
         return CCtxParams->attachDictPref;
     }
 
+    case ZSTD_c_literalCompressionMode : {
+        const ZSTD_literalCompressionMode_e lcm = (ZSTD_literalCompressionMode_e)value;
+        BOUNDCHECK(ZSTD_c_literalCompressionMode, lcm);
+        CCtxParams->literalCompressionMode = lcm;
+        return CCtxParams->literalCompressionMode;
+    }
+
     case ZSTD_c_nbWorkers :
 #ifndef ZSTD_MULTITHREAD
         RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading");
@@ -688,6 +705,9 @@ size_t ZSTD_CCtxParam_getParameter(
     case ZSTD_c_forceAttachDict :
         *value = CCtxParams->attachDictPref;
         break;
+    case ZSTD_c_literalCompressionMode :
+        *value = CCtxParams->literalCompressionMode;
+        break;
     case ZSTD_c_nbWorkers :
 #ifndef ZSTD_MULTITHREAD
         assert(CCtxParams->nbWorkers == 0);
@@ -2369,6 +2389,21 @@ static size_t ZSTD_encodeSequences(
                                         sequences, nbSeq, longOffsets);
 }
 
+static int ZSTD_disableLiteralsCompression(const ZSTD_CCtx_params* cctxParams)
+{
+    switch (cctxParams->literalCompressionMode) {
+    case ZSTD_lcm_huffman:
+        return 0;
+    case ZSTD_lcm_uncompressed:
+        return 1;
+    default:
+        assert(0 /* impossible: pre-validated */);
+        /* fall-through */
+    case ZSTD_lcm_auto:
+        return (cctxParams->cParams.strategy == ZSTD_fast) && (cctxParams->cParams.targetLength > 0);
+    }
+}
+
 /* ZSTD_compressSequences_internal():
  * actually compresses both literals and sequences */
 MEM_STATIC size_t
@@ -2404,10 +2439,10 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
     /* Compress literals */
     {   const BYTE* const literals = seqStorePtr->litStart;
         size_t const litSize = seqStorePtr->lit - literals;
-        int const disableLiteralCompression = (cctxParams->cParams.strategy == ZSTD_fast) && (cctxParams->cParams.targetLength > 0);
         size_t const cSize = ZSTD_compressLiterals(
                                     &prevEntropy->huf, &nextEntropy->huf,
-                                    cctxParams->cParams.strategy, disableLiteralCompression,
+                                    cctxParams->cParams.strategy,
+                                    ZSTD_disableLiteralsCompression(cctxParams),
                                     op, dstCapacity,
                                     literals, litSize,
                                     workspace, wkspSize,
diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index 29bca59859e..e0b54299d6a 100644
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -188,6 +188,7 @@ struct ZSTD_CCtx_params_s {
                                 * 1<
Date: Wed, 13 Feb 2019 15:00:32 -0800
Subject: [PATCH 062/178] [regression] Test ZSTD_c_literalCompressionMode

Test a positive compression level with uncompressed literals,
and a negative compression level with compressed literals.

I double checked the `results.csv` and made sure that the compressed
sizes make sense.
---
 tests/regression/config.c    | 22 ++++++++++++++++++++++
 tests/regression/results.csv | 30 ++++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+)

diff --git a/tests/regression/config.c b/tests/regression/config.c
index 8a6103b2929..f04d32d8381 100644
--- a/tests/regression/config.c
+++ b/tests/regression/config.c
@@ -122,6 +122,26 @@ static config_t small_clog = {
     .param_values = PARAM_VALUES(small_clog_param_values),
 };
 
+static param_value_t const uncompressed_literals_param_values[] = {
+    {.param = ZSTD_c_compressionLevel, .value = 3},
+    {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_lcm_uncompressed},
+};
+
+static config_t uncompressed_literals = {
+    .name = "uncompressed literals",
+    .param_values = PARAM_VALUES(uncompressed_literals_param_values),
+};
+
+static param_value_t const huffman_literals_param_values[] = {
+    {.param = ZSTD_c_compressionLevel, .value = -1},
+    {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_lcm_huffman},
+};
+
+static config_t huffman_literals = {
+    .name = "huffman literals",
+    .param_values = PARAM_VALUES(huffman_literals_param_values),
+};
+
 static param_value_t const explicit_params_param_values[] = {
     {.param = ZSTD_c_checksumFlag, .value = 1},
     {.param = ZSTD_c_contentSizeFlag, .value = 0},
@@ -155,6 +175,8 @@ static config_t const* g_configs[] = {
     &small_hlog,
     &small_clog,
     &explicit_params,
+    &uncompressed_literals,
+    &huffman_literals,
     NULL,
 };
 
diff --git a/tests/regression/results.csv b/tests/regression/results.csv
index a62178d6ed0..d9c5a4a0a5d 100644
--- a/tests/regression/results.csv
+++ b/tests/regression/results.csv
@@ -13,6 +13,8 @@ silesia.tar,                      level 9,                          compress sim
 silesia.tar,                      level 13,                         compress simple,                  4491702
 silesia.tar,                      level 16,                         compress simple,                  4381277
 silesia.tar,                      level 19,                         compress simple,                  4281514
+silesia.tar,                      uncompressed literals,            compress simple,                  4875008
+silesia.tar,                      huffman literals,                 compress simple,                  6195462
 silesia,                          level -5,                         compress cctx,                    7152294
 silesia,                          level -3,                         compress cctx,                    6789969
 silesia,                          level -1,                         compress cctx,                    6191548
@@ -34,6 +36,8 @@ silesia,                          small window log,                 compress cct
 silesia,                          small hash log,                   compress cctx,                    6554898
 silesia,                          small chain log,                  compress cctx,                    4931093
 silesia,                          explicit params,                  compress cctx,                    4813352
+silesia,                          uncompressed literals,            compress cctx,                    4862377
+silesia,                          huffman literals,                 compress cctx,                    6191548
 github,                           level -5,                         compress cctx,                    232744
 github,                           level -5 with dict,               compress cctx,                    47294
 github,                           level -3,                         compress cctx,                    220611
@@ -69,6 +73,8 @@ github,                           small window log,                 compress cct
 github,                           small hash log,                   compress cctx,                    decompression error
 github,                           small chain log,                  compress cctx,                    decompression error
 github,                           explicit params,                  compress cctx,                    decompression error
+github,                           uncompressed literals,            compress cctx,                    136397
+github,                           huffman literals,                 compress cctx,                    176575
 silesia,                          level -5,                         zstdcli,                          7152342
 silesia,                          level -3,                         zstdcli,                          6790021
 silesia,                          level -1,                         zstdcli,                          6191597
@@ -169,6 +175,8 @@ silesia,                          small window log,                 advanced one
 silesia,                          small hash log,                   advanced one pass,                6554898
 silesia,                          small chain log,                  advanced one pass,                4931093
 silesia,                          explicit params,                  advanced one pass,                4815369
+silesia,                          uncompressed literals,            advanced one pass,                5155424
+silesia,                          huffman literals,                 advanced one pass,                5341356
 silesia.tar,                      level -5,                         advanced one pass,                7160438
 silesia.tar,                      level -3,                         advanced one pass,                6789024
 silesia.tar,                      level -1,                         advanced one pass,                6195462
@@ -191,6 +199,8 @@ silesia.tar,                      small window log,                 advanced one
 silesia.tar,                      small hash log,                   advanced one pass,                6587833
 silesia.tar,                      small chain log,                  advanced one pass,                4943255
 silesia.tar,                      explicit params,                  advanced one pass,                4829974
+silesia.tar,                      uncompressed literals,            advanced one pass,                5157992
+silesia.tar,                      huffman literals,                 advanced one pass,                5358079
 github,                           level -5,                         advanced one pass,                232744
 github,                           level -5 with dict,               advanced one pass,                46718
 github,                           level -3,                         advanced one pass,                220611
@@ -227,6 +237,8 @@ github,                           small window log,                 advanced one
 github,                           small hash log,                   advanced one pass,                135467
 github,                           small chain log,                  advanced one pass,                136314
 github,                           explicit params,                  advanced one pass,                137670
+github,                           uncompressed literals,            advanced one pass,                167004
+github,                           huffman literals,                 advanced one pass,                143457
 silesia,                          level -5,                         advanced one pass small out,      7152294
 silesia,                          level -3,                         advanced one pass small out,      6789969
 silesia,                          level -1,                         advanced one pass small out,      6191548
@@ -249,6 +261,8 @@ silesia,                          small window log,                 advanced one
 silesia,                          small hash log,                   advanced one pass small out,      6554898
 silesia,                          small chain log,                  advanced one pass small out,      4931093
 silesia,                          explicit params,                  advanced one pass small out,      4815369
+silesia,                          uncompressed literals,            advanced one pass small out,      5155424
+silesia,                          huffman literals,                 advanced one pass small out,      5341356
 silesia.tar,                      level -5,                         advanced one pass small out,      7160438
 silesia.tar,                      level -3,                         advanced one pass small out,      6789024
 silesia.tar,                      level -1,                         advanced one pass small out,      6195462
@@ -271,6 +285,8 @@ silesia.tar,                      small window log,                 advanced one
 silesia.tar,                      small hash log,                   advanced one pass small out,      6587833
 silesia.tar,                      small chain log,                  advanced one pass small out,      4943255
 silesia.tar,                      explicit params,                  advanced one pass small out,      4829974
+silesia.tar,                      uncompressed literals,            advanced one pass small out,      5157992
+silesia.tar,                      huffman literals,                 advanced one pass small out,      5358079
 github,                           level -5,                         advanced one pass small out,      232744
 github,                           level -5 with dict,               advanced one pass small out,      46718
 github,                           level -3,                         advanced one pass small out,      220611
@@ -307,6 +323,8 @@ github,                           small window log,                 advanced one
 github,                           small hash log,                   advanced one pass small out,      135467
 github,                           small chain log,                  advanced one pass small out,      136314
 github,                           explicit params,                  advanced one pass small out,      137670
+github,                           uncompressed literals,            advanced one pass small out,      167004
+github,                           huffman literals,                 advanced one pass small out,      143457
 silesia,                          level -5,                         advanced streaming,               7152294
 silesia,                          level -3,                         advanced streaming,               6789973
 silesia,                          level -1,                         advanced streaming,               6191549
@@ -329,6 +347,8 @@ silesia,                          small window log,                 advanced str
 silesia,                          small hash log,                   advanced streaming,               6554898
 silesia,                          small chain log,                  advanced streaming,               4931093
 silesia,                          explicit params,                  advanced streaming,               4815380
+silesia,                          uncompressed literals,            advanced streaming,               5155424
+silesia,                          huffman literals,                 advanced streaming,               5341357
 silesia.tar,                      level -5,                         advanced streaming,               7160440
 silesia.tar,                      level -3,                         advanced streaming,               6789026
 silesia.tar,                      level -1,                         advanced streaming,               6195465
@@ -351,6 +371,8 @@ silesia.tar,                      small window log,                 advanced str
 silesia.tar,                      small hash log,                   advanced streaming,               6587834
 silesia.tar,                      small chain log,                  advanced streaming,               4943260
 silesia.tar,                      explicit params,                  advanced streaming,               4830002
+silesia.tar,                      uncompressed literals,            advanced streaming,               5157995
+silesia.tar,                      huffman literals,                 advanced streaming,               5358083
 github,                           level -5,                         advanced streaming,               232744
 github,                           level -5 with dict,               advanced streaming,               46718
 github,                           level -3,                         advanced streaming,               220611
@@ -387,6 +409,8 @@ github,                           small window log,                 advanced str
 github,                           small hash log,                   advanced streaming,               135467
 github,                           small chain log,                  advanced streaming,               136314
 github,                           explicit params,                  advanced streaming,               137670
+github,                           uncompressed literals,            advanced streaming,               167004
+github,                           huffman literals,                 advanced streaming,               143457
 silesia,                          level -5,                         old streaming,                    7152294
 silesia,                          level -3,                         old streaming,                    6789973
 silesia,                          level -1,                         old streaming,                    6191549
@@ -402,6 +426,8 @@ silesia,                          level 13,                         old streamin
 silesia,                          level 16,                         old streaming,                    4377391
 silesia,                          level 19,                         old streaming,                    4293262
 silesia,                          no source size,                   old streaming,                    4862341
+silesia,                          uncompressed literals,            old streaming,                    4862377
+silesia,                          huffman literals,                 old streaming,                    6191549
 silesia.tar,                      level -5,                         old streaming,                    7160440
 silesia.tar,                      level -3,                         old streaming,                    6789026
 silesia.tar,                      level -1,                         old streaming,                    6195465
@@ -417,6 +443,8 @@ silesia.tar,                      level 13,                         old streamin
 silesia.tar,                      level 16,                         old streaming,                    4381277
 silesia.tar,                      level 19,                         old streaming,                    4281514
 silesia.tar,                      no source size,                   old streaming,                    4875006
+silesia.tar,                      uncompressed literals,            old streaming,                    4875010
+silesia.tar,                      huffman literals,                 old streaming,                    6195465
 github,                           level -5,                         old streaming,                    232744
 github,                           level -5 with dict,               old streaming,                    46718
 github,                           level -3,                         old streaming,                    220611
@@ -446,3 +474,5 @@ github,                           level 16 with dict,               old streamin
 github,                           level 19,                         old streaming,                    133717
 github,                           level 19 with dict,               old streaming,                    37576
 github,                           no source size,                   old streaming,                    141003
+github,                           uncompressed literals,            old streaming,                    136397
+github,                           huffman literals,                 old streaming,                    176575

From a96e67af6cab104728ff28b50a8121306bdfdf53 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 15 Feb 2019 10:28:44 -0800
Subject: [PATCH 063/178] [regression] Test level 19 with uncompressed literals

---
 tests/regression/config.c    | 11 +++++++++++
 tests/regression/results.csv | 15 +++++++++++++++
 2 files changed, 26 insertions(+)

diff --git a/tests/regression/config.c b/tests/regression/config.c
index f04d32d8381..4d5d191059a 100644
--- a/tests/regression/config.c
+++ b/tests/regression/config.c
@@ -132,6 +132,16 @@ static config_t uncompressed_literals = {
     .param_values = PARAM_VALUES(uncompressed_literals_param_values),
 };
 
+static param_value_t const uncompressed_literals_opt_param_values[] = {
+    {.param = ZSTD_c_compressionLevel, .value = 19},
+    {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_lcm_uncompressed},
+};
+
+static config_t uncompressed_literals_opt = {
+    .name = "uncompressed literals optimal",
+    .param_values = PARAM_VALUES(uncompressed_literals_opt_param_values),
+};
+
 static param_value_t const huffman_literals_param_values[] = {
     {.param = ZSTD_c_compressionLevel, .value = -1},
     {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_lcm_huffman},
@@ -176,6 +186,7 @@ static config_t const* g_configs[] = {
     &small_clog,
     &explicit_params,
     &uncompressed_literals,
+    &uncompressed_literals_opt,
     &huffman_literals,
     NULL,
 };
diff --git a/tests/regression/results.csv b/tests/regression/results.csv
index d9c5a4a0a5d..076e3b454c0 100644
--- a/tests/regression/results.csv
+++ b/tests/regression/results.csv
@@ -14,6 +14,7 @@ silesia.tar,                      level 13,                         compress sim
 silesia.tar,                      level 16,                         compress simple,                  4381277
 silesia.tar,                      level 19,                         compress simple,                  4281514
 silesia.tar,                      uncompressed literals,            compress simple,                  4875008
+silesia.tar,                      uncompressed literals optimal,    compress simple,                  4281514
 silesia.tar,                      huffman literals,                 compress simple,                  6195462
 silesia,                          level -5,                         compress cctx,                    7152294
 silesia,                          level -3,                         compress cctx,                    6789969
@@ -37,6 +38,7 @@ silesia,                          small hash log,                   compress cct
 silesia,                          small chain log,                  compress cctx,                    4931093
 silesia,                          explicit params,                  compress cctx,                    4813352
 silesia,                          uncompressed literals,            compress cctx,                    4862377
+silesia,                          uncompressed literals optimal,    compress cctx,                    4293262
 silesia,                          huffman literals,                 compress cctx,                    6191548
 github,                           level -5,                         compress cctx,                    232744
 github,                           level -5 with dict,               compress cctx,                    47294
@@ -74,6 +76,7 @@ github,                           small hash log,                   compress cct
 github,                           small chain log,                  compress cctx,                    decompression error
 github,                           explicit params,                  compress cctx,                    decompression error
 github,                           uncompressed literals,            compress cctx,                    136397
+github,                           uncompressed literals optimal,    compress cctx,                    133717
 github,                           huffman literals,                 compress cctx,                    176575
 silesia,                          level -5,                         zstdcli,                          7152342
 silesia,                          level -3,                         zstdcli,                          6790021
@@ -176,6 +179,7 @@ silesia,                          small hash log,                   advanced one
 silesia,                          small chain log,                  advanced one pass,                4931093
 silesia,                          explicit params,                  advanced one pass,                4815369
 silesia,                          uncompressed literals,            advanced one pass,                5155424
+silesia,                          uncompressed literals optimal,    advanced one pass,                4426654
 silesia,                          huffman literals,                 advanced one pass,                5341356
 silesia.tar,                      level -5,                         advanced one pass,                7160438
 silesia.tar,                      level -3,                         advanced one pass,                6789024
@@ -200,6 +204,7 @@ silesia.tar,                      small hash log,                   advanced one
 silesia.tar,                      small chain log,                  advanced one pass,                4943255
 silesia.tar,                      explicit params,                  advanced one pass,                4829974
 silesia.tar,                      uncompressed literals,            advanced one pass,                5157992
+silesia.tar,                      uncompressed literals optimal,    advanced one pass,                4372744
 silesia.tar,                      huffman literals,                 advanced one pass,                5358079
 github,                           level -5,                         advanced one pass,                232744
 github,                           level -5 with dict,               advanced one pass,                46718
@@ -238,6 +243,7 @@ github,                           small hash log,                   advanced one
 github,                           small chain log,                  advanced one pass,                136314
 github,                           explicit params,                  advanced one pass,                137670
 github,                           uncompressed literals,            advanced one pass,                167004
+github,                           uncompressed literals optimal,    advanced one pass,                164600
 github,                           huffman literals,                 advanced one pass,                143457
 silesia,                          level -5,                         advanced one pass small out,      7152294
 silesia,                          level -3,                         advanced one pass small out,      6789969
@@ -262,6 +268,7 @@ silesia,                          small hash log,                   advanced one
 silesia,                          small chain log,                  advanced one pass small out,      4931093
 silesia,                          explicit params,                  advanced one pass small out,      4815369
 silesia,                          uncompressed literals,            advanced one pass small out,      5155424
+silesia,                          uncompressed literals optimal,    advanced one pass small out,      4426654
 silesia,                          huffman literals,                 advanced one pass small out,      5341356
 silesia.tar,                      level -5,                         advanced one pass small out,      7160438
 silesia.tar,                      level -3,                         advanced one pass small out,      6789024
@@ -286,6 +293,7 @@ silesia.tar,                      small hash log,                   advanced one
 silesia.tar,                      small chain log,                  advanced one pass small out,      4943255
 silesia.tar,                      explicit params,                  advanced one pass small out,      4829974
 silesia.tar,                      uncompressed literals,            advanced one pass small out,      5157992
+silesia.tar,                      uncompressed literals optimal,    advanced one pass small out,      4372744
 silesia.tar,                      huffman literals,                 advanced one pass small out,      5358079
 github,                           level -5,                         advanced one pass small out,      232744
 github,                           level -5 with dict,               advanced one pass small out,      46718
@@ -324,6 +332,7 @@ github,                           small hash log,                   advanced one
 github,                           small chain log,                  advanced one pass small out,      136314
 github,                           explicit params,                  advanced one pass small out,      137670
 github,                           uncompressed literals,            advanced one pass small out,      167004
+github,                           uncompressed literals optimal,    advanced one pass small out,      164600
 github,                           huffman literals,                 advanced one pass small out,      143457
 silesia,                          level -5,                         advanced streaming,               7152294
 silesia,                          level -3,                         advanced streaming,               6789973
@@ -348,6 +357,7 @@ silesia,                          small hash log,                   advanced str
 silesia,                          small chain log,                  advanced streaming,               4931093
 silesia,                          explicit params,                  advanced streaming,               4815380
 silesia,                          uncompressed literals,            advanced streaming,               5155424
+silesia,                          uncompressed literals optimal,    advanced streaming,               4426654
 silesia,                          huffman literals,                 advanced streaming,               5341357
 silesia.tar,                      level -5,                         advanced streaming,               7160440
 silesia.tar,                      level -3,                         advanced streaming,               6789026
@@ -372,6 +382,7 @@ silesia.tar,                      small hash log,                   advanced str
 silesia.tar,                      small chain log,                  advanced streaming,               4943260
 silesia.tar,                      explicit params,                  advanced streaming,               4830002
 silesia.tar,                      uncompressed literals,            advanced streaming,               5157995
+silesia.tar,                      uncompressed literals optimal,    advanced streaming,               4372744
 silesia.tar,                      huffman literals,                 advanced streaming,               5358083
 github,                           level -5,                         advanced streaming,               232744
 github,                           level -5 with dict,               advanced streaming,               46718
@@ -410,6 +421,7 @@ github,                           small hash log,                   advanced str
 github,                           small chain log,                  advanced streaming,               136314
 github,                           explicit params,                  advanced streaming,               137670
 github,                           uncompressed literals,            advanced streaming,               167004
+github,                           uncompressed literals optimal,    advanced streaming,               164600
 github,                           huffman literals,                 advanced streaming,               143457
 silesia,                          level -5,                         old streaming,                    7152294
 silesia,                          level -3,                         old streaming,                    6789973
@@ -427,6 +439,7 @@ silesia,                          level 16,                         old streamin
 silesia,                          level 19,                         old streaming,                    4293262
 silesia,                          no source size,                   old streaming,                    4862341
 silesia,                          uncompressed literals,            old streaming,                    4862377
+silesia,                          uncompressed literals optimal,    old streaming,                    4293262
 silesia,                          huffman literals,                 old streaming,                    6191549
 silesia.tar,                      level -5,                         old streaming,                    7160440
 silesia.tar,                      level -3,                         old streaming,                    6789026
@@ -444,6 +457,7 @@ silesia.tar,                      level 16,                         old streamin
 silesia.tar,                      level 19,                         old streaming,                    4281514
 silesia.tar,                      no source size,                   old streaming,                    4875006
 silesia.tar,                      uncompressed literals,            old streaming,                    4875010
+silesia.tar,                      uncompressed literals optimal,    old streaming,                    4281514
 silesia.tar,                      huffman literals,                 old streaming,                    6195465
 github,                           level -5,                         old streaming,                    232744
 github,                           level -5 with dict,               old streaming,                    46718
@@ -475,4 +489,5 @@ github,                           level 19,                         old streamin
 github,                           level 19 with dict,               old streaming,                    37576
 github,                           no source size,                   old streaming,                    141003
 github,                           uncompressed literals,            old streaming,                    136397
+github,                           uncompressed literals optimal,    old streaming,                    133717
 github,                           huffman literals,                 old streaming,                    176575

From 3d7377b87424877c59e068e97f3d9b930aca630a Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 15 Feb 2019 10:29:03 -0800
Subject: [PATCH 064/178] [libzstd] Handle uncompressed literals

---
 lib/compress/zstd_compress.c          |  7 ++++--
 lib/compress/zstd_compress_internal.h |  1 +
 lib/compress/zstd_opt.c               | 34 ++++++++++++++++++++-------
 tests/regression/results.csv          | 18 +++++++-------
 4 files changed, 40 insertions(+), 20 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index e18051f00ce..9ea7f04efb4 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -402,7 +402,6 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param)
     case ZSTD_c_minMatch:
     case ZSTD_c_targetLength:
     case ZSTD_c_strategy:
-    case ZSTD_c_literalCompressionMode:
         return 1;
 
     case ZSTD_c_format:
@@ -421,6 +420,7 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param)
     case ZSTD_c_ldmBucketSizeLog:
     case ZSTD_c_ldmHashRateLog:
     case ZSTD_c_forceAttachDict:
+    case ZSTD_c_literalCompressionMode:
     default:
         return 0;
     }
@@ -2677,7 +2677,10 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
         goto out;  /* don't even attempt compression below a certain srcSize */
     }
     ZSTD_resetSeqStore(&(zc->seqStore));
-    ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy;   /* required for optimal parser to read stats from dictionary */
+    /* required for optimal parser to read stats from dictionary */
+    ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy;
+    /* tell the optimal parser how we expect to compress literals */
+    ms->opt.literalCompressionMode = zc->appliedParams.literalCompressionMode;
 
     /* a gap between an attached dict and the current window is not safe,
      * they must remain adjacent,
diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index e0b54299d6a..a828de3b4a6 100644
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -107,6 +107,7 @@ typedef struct {
     U32  offCodeSumBasePrice;    /* to compare to log2(offreq)  */
     ZSTD_OptPrice_e priceType;   /* prices can be determined dynamically, or follow a pre-defined cost structure */
     const ZSTD_entropyCTables_t* symbolCosts;  /* pre-calculated dictionary statistics */
+    ZSTD_literalCompressionMode_e literalCompressionMode;
 } optState_t;
 
 typedef struct {
diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 44de6e97fd2..cf2f70b11f4 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -64,9 +64,15 @@ MEM_STATIC double ZSTD_fCost(U32 price)
 }
 #endif
 
+static int ZSTD_compressedLiterals(optState_t const* const optPtr)
+{
+    return optPtr->literalCompressionMode != ZSTD_lcm_uncompressed;
+}
+
 static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel)
 {
-    optPtr->litSumBasePrice = WEIGHT(optPtr->litSum, optLevel);
+    if (ZSTD_compressedLiterals(optPtr))
+        optPtr->litSumBasePrice = WEIGHT(optPtr->litSum, optLevel);
     optPtr->litLengthSumBasePrice = WEIGHT(optPtr->litLengthSum, optLevel);
     optPtr->matchLengthSumBasePrice = WEIGHT(optPtr->matchLengthSum, optLevel);
     optPtr->offCodeSumBasePrice = WEIGHT(optPtr->offCodeSum, optLevel);
@@ -99,6 +105,7 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
             const BYTE* const src, size_t const srcSize,
                   int const optLevel)
 {
+    int const compressedLiterals = ZSTD_compressedLiterals(optPtr);
     DEBUGLOG(5, "ZSTD_rescaleFreqs (srcSize=%u)", (unsigned)srcSize);
     optPtr->priceType = zop_dynamic;
 
@@ -113,9 +120,10 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
             /* huffman table presumed generated by dictionary */
             optPtr->priceType = zop_dynamic;
 
-            assert(optPtr->litFreq != NULL);
-            optPtr->litSum = 0;
-            {   unsigned lit;
+            if (compressedLiterals) {
+                unsigned lit;
+                assert(optPtr->litFreq != NULL);
+                optPtr->litSum = 0;
                 for (lit=0; lit<=MaxLit; lit++) {
                     U32 const scaleLog = 11;   /* scale to 2K */
                     U32 const bitCost = HUF_getNbBits(optPtr->symbolCosts->huf.CTable, lit);
@@ -163,10 +171,11 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
         } else {  /* not a dictionary */
 
             assert(optPtr->litFreq != NULL);
-            {   unsigned lit = MaxLit;
+            if (compressedLiterals) {
+                unsigned lit = MaxLit;
                 HIST_count_simple(optPtr->litFreq, &lit, src, srcSize);   /* use raw first block to init statistics */
+                optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1);
             }
-            optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1);
 
             {   unsigned ll;
                 for (ll=0; ll<=MaxLL; ll++)
@@ -190,7 +199,8 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
 
     } else {   /* new block : re-use previous statistics, scaled down */
 
-        optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1);
+        if (compressedLiterals)
+            optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1);
         optPtr->litLengthSum = ZSTD_downscaleStat(optPtr->litLengthFreq, MaxLL, 0);
         optPtr->matchLengthSum = ZSTD_downscaleStat(optPtr->matchLengthFreq, MaxML, 0);
         optPtr->offCodeSum = ZSTD_downscaleStat(optPtr->offCodeFreq, MaxOff, 0);
@@ -207,6 +217,10 @@ static U32 ZSTD_rawLiteralsCost(const BYTE* const literals, U32 const litLength,
                                 int optLevel)
 {
     if (litLength == 0) return 0;
+
+    if (!ZSTD_compressedLiterals(optPtr))
+        return (litLength << 3) * BITCOST_MULTIPLIER;  /* Uncompressed - 8 bytes per literal. */
+
     if (optPtr->priceType == zop_predef)
         return (litLength*6) * BITCOST_MULTIPLIER;  /* 6 bit per literal - no statistic used */
 
@@ -310,7 +324,8 @@ static void ZSTD_updateStats(optState_t* const optPtr,
                              U32 offsetCode, U32 matchLength)
 {
     /* literals */
-    {   U32 u;
+    if (ZSTD_compressedLiterals(optPtr)) {
+        U32 u;
         for (u=0; u < litLength; u++)
             optPtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD;
         optPtr->litSum += litLength*ZSTD_LITFREQ_ADD;
@@ -1108,7 +1123,8 @@ static U32 ZSTD_upscaleStat(unsigned* table, U32 lastEltIndex, int bonus)
 /* used in 2-pass strategy */
 MEM_STATIC void ZSTD_upscaleStats(optState_t* optPtr)
 {
-    optPtr->litSum = ZSTD_upscaleStat(optPtr->litFreq, MaxLit, 0);
+    if (ZSTD_compressedLiterals(optPtr))
+        optPtr->litSum = ZSTD_upscaleStat(optPtr->litFreq, MaxLit, 0);
     optPtr->litLengthSum = ZSTD_upscaleStat(optPtr->litLengthFreq, MaxLL, 0);
     optPtr->matchLengthSum = ZSTD_upscaleStat(optPtr->matchLengthFreq, MaxML, 0);
     optPtr->offCodeSum = ZSTD_upscaleStat(optPtr->offCodeFreq, MaxOff, 0);
diff --git a/tests/regression/results.csv b/tests/regression/results.csv
index 076e3b454c0..6d0d1b51580 100644
--- a/tests/regression/results.csv
+++ b/tests/regression/results.csv
@@ -179,7 +179,7 @@ silesia,                          small hash log,                   advanced one
 silesia,                          small chain log,                  advanced one pass,                4931093
 silesia,                          explicit params,                  advanced one pass,                4815369
 silesia,                          uncompressed literals,            advanced one pass,                5155424
-silesia,                          uncompressed literals optimal,    advanced one pass,                4426654
+silesia,                          uncompressed literals optimal,    advanced one pass,                4325427
 silesia,                          huffman literals,                 advanced one pass,                5341356
 silesia.tar,                      level -5,                         advanced one pass,                7160438
 silesia.tar,                      level -3,                         advanced one pass,                6789024
@@ -204,7 +204,7 @@ silesia.tar,                      small hash log,                   advanced one
 silesia.tar,                      small chain log,                  advanced one pass,                4943255
 silesia.tar,                      explicit params,                  advanced one pass,                4829974
 silesia.tar,                      uncompressed literals,            advanced one pass,                5157992
-silesia.tar,                      uncompressed literals optimal,    advanced one pass,                4372744
+silesia.tar,                      uncompressed literals optimal,    advanced one pass,                4321094
 silesia.tar,                      huffman literals,                 advanced one pass,                5358079
 github,                           level -5,                         advanced one pass,                232744
 github,                           level -5 with dict,               advanced one pass,                46718
@@ -243,7 +243,7 @@ github,                           small hash log,                   advanced one
 github,                           small chain log,                  advanced one pass,                136314
 github,                           explicit params,                  advanced one pass,                137670
 github,                           uncompressed literals,            advanced one pass,                167004
-github,                           uncompressed literals optimal,    advanced one pass,                164600
+github,                           uncompressed literals optimal,    advanced one pass,                156824
 github,                           huffman literals,                 advanced one pass,                143457
 silesia,                          level -5,                         advanced one pass small out,      7152294
 silesia,                          level -3,                         advanced one pass small out,      6789969
@@ -268,7 +268,7 @@ silesia,                          small hash log,                   advanced one
 silesia,                          small chain log,                  advanced one pass small out,      4931093
 silesia,                          explicit params,                  advanced one pass small out,      4815369
 silesia,                          uncompressed literals,            advanced one pass small out,      5155424
-silesia,                          uncompressed literals optimal,    advanced one pass small out,      4426654
+silesia,                          uncompressed literals optimal,    advanced one pass small out,      4325427
 silesia,                          huffman literals,                 advanced one pass small out,      5341356
 silesia.tar,                      level -5,                         advanced one pass small out,      7160438
 silesia.tar,                      level -3,                         advanced one pass small out,      6789024
@@ -293,7 +293,7 @@ silesia.tar,                      small hash log,                   advanced one
 silesia.tar,                      small chain log,                  advanced one pass small out,      4943255
 silesia.tar,                      explicit params,                  advanced one pass small out,      4829974
 silesia.tar,                      uncompressed literals,            advanced one pass small out,      5157992
-silesia.tar,                      uncompressed literals optimal,    advanced one pass small out,      4372744
+silesia.tar,                      uncompressed literals optimal,    advanced one pass small out,      4321094
 silesia.tar,                      huffman literals,                 advanced one pass small out,      5358079
 github,                           level -5,                         advanced one pass small out,      232744
 github,                           level -5 with dict,               advanced one pass small out,      46718
@@ -332,7 +332,7 @@ github,                           small hash log,                   advanced one
 github,                           small chain log,                  advanced one pass small out,      136314
 github,                           explicit params,                  advanced one pass small out,      137670
 github,                           uncompressed literals,            advanced one pass small out,      167004
-github,                           uncompressed literals optimal,    advanced one pass small out,      164600
+github,                           uncompressed literals optimal,    advanced one pass small out,      156824
 github,                           huffman literals,                 advanced one pass small out,      143457
 silesia,                          level -5,                         advanced streaming,               7152294
 silesia,                          level -3,                         advanced streaming,               6789973
@@ -357,7 +357,7 @@ silesia,                          small hash log,                   advanced str
 silesia,                          small chain log,                  advanced streaming,               4931093
 silesia,                          explicit params,                  advanced streaming,               4815380
 silesia,                          uncompressed literals,            advanced streaming,               5155424
-silesia,                          uncompressed literals optimal,    advanced streaming,               4426654
+silesia,                          uncompressed literals optimal,    advanced streaming,               4325427
 silesia,                          huffman literals,                 advanced streaming,               5341357
 silesia.tar,                      level -5,                         advanced streaming,               7160440
 silesia.tar,                      level -3,                         advanced streaming,               6789026
@@ -382,7 +382,7 @@ silesia.tar,                      small hash log,                   advanced str
 silesia.tar,                      small chain log,                  advanced streaming,               4943260
 silesia.tar,                      explicit params,                  advanced streaming,               4830002
 silesia.tar,                      uncompressed literals,            advanced streaming,               5157995
-silesia.tar,                      uncompressed literals optimal,    advanced streaming,               4372744
+silesia.tar,                      uncompressed literals optimal,    advanced streaming,               4321094
 silesia.tar,                      huffman literals,                 advanced streaming,               5358083
 github,                           level -5,                         advanced streaming,               232744
 github,                           level -5 with dict,               advanced streaming,               46718
@@ -421,7 +421,7 @@ github,                           small hash log,                   advanced str
 github,                           small chain log,                  advanced streaming,               136314
 github,                           explicit params,                  advanced streaming,               137670
 github,                           uncompressed literals,            advanced streaming,               167004
-github,                           uncompressed literals optimal,    advanced streaming,               164600
+github,                           uncompressed literals optimal,    advanced streaming,               156824
 github,                           huffman literals,                 advanced streaming,               143457
 silesia,                          level -5,                         old streaming,                    7152294
 silesia,                          level -3,                         old streaming,                    6789973

From 5261a288d1fcecdbae988468d8d3191572bb7f49 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 15 Feb 2019 14:14:32 -0800
Subject: [PATCH 065/178] Fix a bug in the compress cctx method

---
 tests/regression/method.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/regression/method.c b/tests/regression/method.c
index 51a0fb9d14f..16701c0b440 100644
--- a/tests/regression/method.c
+++ b/tests/regression/method.c
@@ -175,8 +175,8 @@ static result_t compress_cctx_compress(
                 state->compressed.capacity,
                 input.data,
                 input.size,
-                state->dictionary.data,
-                state->dictionary.size,
+                config->use_dictionary ? state->dictionary.data : NULL,
+                config->use_dictionary ? state->dictionary.size : 0,
                 params);
         else if (config->use_dictionary)
             state->compressed.size = ZSTD_compress_usingDict(

From 0c53c5ad4aa419d1f3aab0022ad181d1efec6f30 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 15 Feb 2019 14:15:36 -0800
Subject: [PATCH 066/178] [zstdcli] Add a flag to control literals compression

---
 programs/fileio.c            |  9 +++++++++
 programs/fileio.h            |  3 +++
 programs/zstdcli.c           |  5 ++++-
 tests/playTests.sh           |  7 +++++++
 tests/regression/config.c    |  3 +++
 tests/regression/results.csv | 23 ++++++++++++++++-------
 6 files changed, 42 insertions(+), 8 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index dd47a1e32ed..8a5715113c8 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -296,6 +296,7 @@ struct FIO_prefs_s {
     int ldmMinMatch;
     int ldmBucketSizeLog;
     int ldmHashRateLog;
+    ZSTD_literalCompressionMode_e literalCompressionMode;
 
     /* IO preferences */
     U32 removeSrcFile;
@@ -339,6 +340,7 @@ FIO_prefs_t* FIO_createPreferences(void)
     ret->ldmMinMatch = 0;
     ret->ldmBucketSizeLog = FIO_LDM_PARAM_NOTSET;
     ret->ldmHashRateLog = FIO_LDM_PARAM_NOTSET;
+    ret->literalCompressionMode = ZSTD_lcm_auto;
     return ret;
 }
 
@@ -406,6 +408,12 @@ void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable) {
     prefs->rsyncable = rsyncable;
 }
 
+void FIO_setLiteralCompressionMode(
+        FIO_prefs_t* const prefs,
+        ZSTD_literalCompressionMode_e mode) {
+    prefs->literalCompressionMode = mode;
+}
+
 void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel)
 {
 #ifndef ZSTD_NOCOMPRESS
@@ -674,6 +682,7 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
         CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_minMatch, (int)comprParams.minMatch) );
         CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetLength, (int)comprParams.targetLength) );
         CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_strategy, comprParams.strategy) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_literalCompressionMode, (int)prefs->literalCompressionMode) );
         /* multi-threading */
 #ifdef ZSTD_MULTITHREAD
         DISPLAYLEVEL(5,"set nb workers = %u \n", prefs->nbWorkers);
diff --git a/programs/fileio.h b/programs/fileio.h
index b20570bcbc8..e46633752f4 100644
--- a/programs/fileio.h
+++ b/programs/fileio.h
@@ -71,6 +71,9 @@ void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog);
 void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, unsigned flag);
 void FIO_setSparseWrite(FIO_prefs_t* const prefs, unsigned sparse);  /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
 void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable);
+void FIO_setLiteralCompressionMode(
+        FIO_prefs_t* const prefs,
+        ZSTD_literalCompressionMode_e mode);
 
 void FIO_setNoProgress(unsigned noProgress);
 void FIO_setNotificationLevel(int level);
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 794e2a8111a..60035e5668f 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -148,6 +148,7 @@ static int usage_advanced(const char* programName)
 #endif
     DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n");
     DISPLAY( "--[no-]check : integrity check (default: enabled) \n");
+    DISPLAY( "--[no-]compress-literals : force (un)compressed literals \n");
 #endif
 #ifdef UTIL_HAS_CREATEFILELIST
     DISPLAY( " -r     : operate recursively on directories \n");
@@ -483,7 +484,7 @@ static int init_cLevel(void) {
 
         if ((*ptr>='0') && (*ptr<='9')) {
             unsigned absLevel;
-            if (readU32FromCharChecked(&ptr, &absLevel)) { 
+            if (readU32FromCharChecked(&ptr, &absLevel)) {
                 DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large\n", ENV_CLEVEL, env);
                 return ZSTDCLI_CLEVEL_DEFAULT;
             } else if (*ptr == 0) {
@@ -659,6 +660,8 @@ int main(int argCount, const char* argv[])
                     if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; FIO_setCompressionType(prefs, FIO_lz4Compression);  continue; }
 #endif
                     if (!strcmp(argument, "--rsyncable")) { rsyncable = 1; continue; }
+                    if (!strcmp(argument, "--compress-literals")) { FIO_setLiteralCompressionMode(prefs, ZSTD_lcm_huffman); continue; }
+                    if (!strcmp(argument, "--no-compress-literals")) { FIO_setLiteralCompressionMode(prefs, ZSTD_lcm_uncompressed); continue; }
                     if (!strcmp(argument, "--no-progress")) { FIO_setNoProgress(1); continue; }
 
                     /* long commands with arguments */
diff --git a/tests/playTests.sh b/tests/playTests.sh
index 5be065fa892..62b3057e982 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -200,6 +200,13 @@ $ZSTD tmp -fo tmp && die "zstd compression overwrote the input file"
 $ZSTD tmp.zst -dfo tmp.zst && die "zstd decompression overwrote the input file"
 $ECHO "test: detect that input file does not exist"
 $ZSTD nothere && die "zstd hasn't detected that input file does not exist"
+$ECHO "test: --[no-]compress-literals"
+$ZSTD tmp -c --no-compress-literals -1       | $ZSTD -t
+$ZSTD tmp -c --no-compress-literals --fast=1 | $ZSTD -t
+$ZSTD tmp -c --no-compress-literals -19      | $ZSTD -t
+$ZSTD tmp -c --compress-literals    -1       | $ZSTD -t
+$ZSTD tmp -c --compress-literals    --fast=1 | $ZSTD -t
+$ZSTD tmp -c --compress-literals    -19      | $ZSTD -t
 
 $ECHO "test : file removal"
 $ZSTD -f --rm tmp
diff --git a/tests/regression/config.c b/tests/regression/config.c
index 4d5d191059a..bd364009941 100644
--- a/tests/regression/config.c
+++ b/tests/regression/config.c
@@ -129,6 +129,7 @@ static param_value_t const uncompressed_literals_param_values[] = {
 
 static config_t uncompressed_literals = {
     .name = "uncompressed literals",
+    .cli_args = "-3 --no-compress-literals",
     .param_values = PARAM_VALUES(uncompressed_literals_param_values),
 };
 
@@ -139,6 +140,7 @@ static param_value_t const uncompressed_literals_opt_param_values[] = {
 
 static config_t uncompressed_literals_opt = {
     .name = "uncompressed literals optimal",
+    .cli_args = "-19 --no-compress-literals",
     .param_values = PARAM_VALUES(uncompressed_literals_opt_param_values),
 };
 
@@ -149,6 +151,7 @@ static param_value_t const huffman_literals_param_values[] = {
 
 static config_t huffman_literals = {
     .name = "huffman literals",
+    .cli_args = "--fast=1 --compress-literals",
     .param_values = PARAM_VALUES(huffman_literals_param_values),
 };
 
diff --git a/tests/regression/results.csv b/tests/regression/results.csv
index 6d0d1b51580..23afae5ee41 100644
--- a/tests/regression/results.csv
+++ b/tests/regression/results.csv
@@ -68,13 +68,13 @@ github,                           level 16,                         compress cct
 github,                           level 16 with dict,               compress cctx,                    37568
 github,                           level 19,                         compress cctx,                    133717
 github,                           level 19 with dict,               compress cctx,                    37567
-github,                           long distance mode,               compress cctx,                    decompression error
-github,                           multithreaded,                    compress cctx,                    decompression error
-github,                           multithreaded long distance mode, compress cctx,                    decompression error
-github,                           small window log,                 compress cctx,                    decompression error
-github,                           small hash log,                   compress cctx,                    decompression error
-github,                           small chain log,                  compress cctx,                    decompression error
-github,                           explicit params,                  compress cctx,                    decompression error
+github,                           long distance mode,               compress cctx,                    141473
+github,                           multithreaded,                    compress cctx,                    141473
+github,                           multithreaded long distance mode, compress cctx,                    141473
+github,                           small window log,                 compress cctx,                    141473
+github,                           small hash log,                   compress cctx,                    138943
+github,                           small chain log,                  compress cctx,                    139239
+github,                           explicit params,                  compress cctx,                    140924
 github,                           uncompressed literals,            compress cctx,                    136397
 github,                           uncompressed literals optimal,    compress cctx,                    133717
 github,                           huffman literals,                 compress cctx,                    176575
@@ -99,6 +99,9 @@ silesia,                          small window log,                 zstdcli,
 silesia,                          small hash log,                   zstdcli,                          6554946
 silesia,                          small chain log,                  zstdcli,                          4931141
 silesia,                          explicit params,                  zstdcli,                          4815380
+silesia,                          uncompressed literals,            zstdcli,                          5155472
+silesia,                          uncompressed literals optimal,    zstdcli,                          4325475
+silesia,                          huffman literals,                 zstdcli,                          5341405
 silesia.tar,                      level -5,                         zstdcli,                          7161160
 silesia.tar,                      level -3,                         zstdcli,                          6789865
 silesia.tar,                      level -1,                         zstdcli,                          6196433
@@ -121,6 +124,9 @@ silesia.tar,                      small window log,                 zstdcli,
 silesia.tar,                      small hash log,                   zstdcli,                          6587841
 silesia.tar,                      small chain log,                  zstdcli,                          4943259
 silesia.tar,                      explicit params,                  zstdcli,                          4839202
+silesia.tar,                      uncompressed literals,            zstdcli,                          5158134
+silesia.tar,                      uncompressed literals optimal,    zstdcli,                          4321098
+silesia.tar,                      huffman literals,                 zstdcli,                          5358479
 github,                           level -5,                         zstdcli,                          234744
 github,                           level -5 with dict,               zstdcli,                          48718
 github,                           level -3,                         zstdcli,                          222611
@@ -156,6 +162,9 @@ github,                           small window log,                 zstdcli,
 github,                           small hash log,                   zstdcli,                          137467
 github,                           small chain log,                  zstdcli,                          138314
 github,                           explicit params,                  zstdcli,                          136140
+github,                           uncompressed literals,            zstdcli,                          169004
+github,                           uncompressed literals optimal,    zstdcli,                          158824
+github,                           huffman literals,                 zstdcli,                          145457
 silesia,                          level -5,                         advanced one pass,                7152294
 silesia,                          level -3,                         advanced one pass,                6789969
 silesia,                          level -1,                         advanced one pass,                6191548

From 19ca3fbc037e2708b8f604c13a6ed176480f3642 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 15 Feb 2019 15:24:55 -0800
Subject: [PATCH 067/178] [zstdcli] Respect --[no-]compress-literals in
 benchmark mode

---
 programs/benchzstd.c | 4 +++-
 programs/benchzstd.h | 1 +
 programs/zstdcli.c   | 9 ++++++---
 tests/playTests.sh   | 2 ++
 4 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/programs/benchzstd.c b/programs/benchzstd.c
index 0be74a96504..4bd42cfe739 100644
--- a/programs/benchzstd.c
+++ b/programs/benchzstd.c
@@ -135,7 +135,8 @@ BMK_advancedParams_t BMK_initAdvancedParams(void) {
         0, /* ldmMinMatch */
         0, /* ldmHashLog */
         0, /* ldmBuckSizeLog */
-        0  /* ldmHashRateLog */
+        0,  /* ldmHashRateLog */
+        ZSTD_lcm_auto /* literalCompressionMode */
     };
     return res;
 }
@@ -174,6 +175,7 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx,
     CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmHashLog, adv->ldmHashLog));
     CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmBucketSizeLog, adv->ldmBucketSizeLog));
     CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmHashRateLog, adv->ldmHashRateLog));
+    CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_literalCompressionMode, (int)adv->literalCompressionMode));
     CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_windowLog, comprParams->windowLog));
     CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_hashLog, comprParams->hashLog));
     CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_chainLog, comprParams->chainLog));
diff --git a/programs/benchzstd.h b/programs/benchzstd.h
index 3a8b893e7ee..d6f5486ab1f 100644
--- a/programs/benchzstd.h
+++ b/programs/benchzstd.h
@@ -116,6 +116,7 @@ typedef struct {
     unsigned ldmHashLog;
     unsigned ldmBucketSizeLog;
     unsigned ldmHashRateLog;
+    ZSTD_literalCompressionMode_e literalCompressionMode;
 } BMK_advancedParams_t;
 
 /* returns default parameters used by nonAdvanced functions */
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 60035e5668f..f5782174284 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -568,6 +568,7 @@ int main(int argCount, const char* argv[])
 #ifndef ZSTD_NOBENCH
     BMK_advancedParams_t benchParams = BMK_initAdvancedParams();
 #endif
+    ZSTD_literalCompressionMode_e literalCompressionMode = ZSTD_lcm_auto;
 
 
     /* init */
@@ -660,8 +661,8 @@ int main(int argCount, const char* argv[])
                     if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; FIO_setCompressionType(prefs, FIO_lz4Compression);  continue; }
 #endif
                     if (!strcmp(argument, "--rsyncable")) { rsyncable = 1; continue; }
-                    if (!strcmp(argument, "--compress-literals")) { FIO_setLiteralCompressionMode(prefs, ZSTD_lcm_huffman); continue; }
-                    if (!strcmp(argument, "--no-compress-literals")) { FIO_setLiteralCompressionMode(prefs, ZSTD_lcm_uncompressed); continue; }
+                    if (!strcmp(argument, "--compress-literals")) { literalCompressionMode = ZSTD_lcm_huffman; continue; }
+                    if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_lcm_uncompressed; continue; }
                     if (!strcmp(argument, "--no-progress")) { FIO_setNoProgress(1); continue; }
 
                     /* long commands with arguments */
@@ -998,6 +999,7 @@ int main(int argCount, const char* argv[])
         if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) {
             benchParams.ldmHashRateLog = g_ldmHashRateLog;
         }
+        benchParams.literalCompressionMode = literalCompressionMode;
 
         if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
         if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel();
@@ -1111,6 +1113,7 @@ int main(int argCount, const char* argv[])
         FIO_setAdaptMin(prefs, adaptMin);
         FIO_setAdaptMax(prefs, adaptMax);
         FIO_setRsyncable(prefs, rsyncable);
+        FIO_setLiteralCompressionMode(prefs, literalCompressionMode);
         if (adaptMin > cLevel) cLevel = adaptMin;
         if (adaptMax < cLevel) cLevel = adaptMax;
 
@@ -1119,7 +1122,7 @@ int main(int argCount, const char* argv[])
         else
           operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams);
 #else
-        (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; /* not used when ZSTD_NOCOMPRESS set */
+        (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; /* not used when ZSTD_NOCOMPRESS set */
         DISPLAY("Compression not supported \n");
 #endif
     } else {  /* decompression or test */
diff --git a/tests/playTests.sh b/tests/playTests.sh
index 62b3057e982..493542960ed 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -207,6 +207,8 @@ $ZSTD tmp -c --no-compress-literals -19      | $ZSTD -t
 $ZSTD tmp -c --compress-literals    -1       | $ZSTD -t
 $ZSTD tmp -c --compress-literals    --fast=1 | $ZSTD -t
 $ZSTD tmp -c --compress-literals    -19      | $ZSTD -t
+$ZSTD -b --fast=1 -i1e1 tmp --compress-literals
+$ZSTD -b --fast=1 -i1e1 tmp --no-compress-literals
 
 $ECHO "test : file removal"
 $ZSTD -f --rm tmp

From f4abba02ba34113f53d05cae504ae832f098ec1c Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 15 Feb 2019 16:15:20 -0800
Subject: [PATCH 068/178] [libzstd] Clean up parameter code

* Move all ZSTDMT parameter setting code to ZSTD_CCtxParams_*Parameter().
  ZSTDMT now calls these functions, so we can keep all the logic in the
  same place.
* Clean up `ZSTD_CCtx_setParameter()` to only add extra checks where needed.
* Clean up `ZSTDMT_initJobCCtxParams()` by copying all parameters by default,
  and then zeroing the ones that need to be zeroed. We've missed adding several
  parameters here, and it makes more sense to only have to update it if you
  change something in ZSTDMT.
* Add `ZSTDMT_cParam_clampBounds()` to clamp a parameter into its valid
  range. Use this to keep backwards compatibility when setting ZSTDMT parameters,
  which clamp into the valid range.
---
 lib/compress/zstd_compress.c   |   91 +--
 lib/compress/zstdmt_compress.c |   54 +-
 tests/regression/config.c      |   12 +
 tests/regression/results.csv   | 1018 ++++++++++++++++----------------
 4 files changed, 598 insertions(+), 577 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 9ea7f04efb4..1429b84f97b 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -385,6 +385,18 @@ static int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value)
     return 1;
 }
 
+/* ZSTD_cParam_clampBounds:
+ * Clamps the value into the bounded range.
+ */
+static size_t ZSTD_cParam_clampBounds(ZSTD_cParameter cParam, int* value)
+{
+    ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam);
+    if (ZSTD_isError(bounds.error)) return bounds.error;
+    if (*value < bounds.lowerBound) *value = bounds.lowerBound;
+    if (*value > bounds.upperBound) *value = bounds.upperBound;
+    return 0;
+}
+
 #define BOUNDCHECK(cParam, val) { \
     RETURN_ERROR_IF(!ZSTD_cParam_withinBounds(cParam,val), \
                     parameter_outOfBound); \
@@ -438,13 +450,10 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
 
     switch(param)
     {
-    case ZSTD_c_format :
-        return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
-
     case ZSTD_c_compressionLevel:
         RETURN_ERROR_IF(cctx->cdict, stage_wrong,
                         "compression level is configured in cdict");
-        return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
+        break;
 
     case ZSTD_c_windowLog:
     case ZSTD_c_hashLog:
@@ -455,44 +464,37 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
     case ZSTD_c_strategy:
         RETURN_ERROR_IF(cctx->cdict, stage_wrong,
                         "cparams are configured in cdict");
-        return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
+        break;
 
+    case ZSTD_c_nbWorkers:
+        RETURN_ERROR_IF((value!=0) && cctx->staticSize, parameter_unsupported,
+                        "MT not compatible with static alloc");
+        break;
+
+    case ZSTD_c_ldmHashRateLog:
+        RETURN_ERROR_IF(cctx->cdict, stage_wrong,
+                        "LDM hash rate log is configured in cdict");
+        break;
+
+    case ZSTD_c_format:
     case ZSTD_c_contentSizeFlag:
     case ZSTD_c_checksumFlag:
     case ZSTD_c_dictIDFlag:
-        return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
-
-    case ZSTD_c_forceMaxWindow :  /* Force back-references to remain < windowSize,
-                                   * even when referencing into Dictionary content.
-                                   * default : 0 when using a CDict, 1 when using a Prefix */
-        return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
-
+    case ZSTD_c_forceMaxWindow:
     case ZSTD_c_forceAttachDict:
-        return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
-
     case ZSTD_c_literalCompressionMode:
-        return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
-
-    case ZSTD_c_nbWorkers:
-        RETURN_ERROR_IF((value!=0) && cctx->staticSize, parameter_unsupported,
-                        "MT not compatible with static alloc");
-        return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
-
     case ZSTD_c_jobSize:
     case ZSTD_c_overlapLog:
     case ZSTD_c_rsyncable:
-        return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
-
     case ZSTD_c_enableLongDistanceMatching:
     case ZSTD_c_ldmHashLog:
     case ZSTD_c_ldmMinMatch:
     case ZSTD_c_ldmBucketSizeLog:
-    case ZSTD_c_ldmHashRateLog:
-        RETURN_ERROR_IF(cctx->cdict, stage_wrong);
-        return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
+        break;
 
     default: RETURN_ERROR(parameter_unsupported);
     }
+    return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
 }
 
 size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams,
@@ -507,11 +509,9 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams,
         return (size_t)CCtxParams->format;
 
     case ZSTD_c_compressionLevel : {
-        int cLevel = value;
-        if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
-        if (cLevel < ZSTD_minCLevel()) cLevel = ZSTD_minCLevel();
-        if (cLevel) {  /* 0 : does not change current level */
-            CCtxParams->compressionLevel = cLevel;
+        FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value));
+        if (value) {  /* 0 : does not change current level */
+            CCtxParams->compressionLevel = value;
         }
         if (CCtxParams->compressionLevel >= 0) return CCtxParams->compressionLevel;
         return 0;  /* return type (size_t) cannot represent negative values */
@@ -597,28 +597,43 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams,
         RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading");
         return 0;
 #else
-        return ZSTDMT_CCtxParam_setNbWorkers(CCtxParams, value);
+        FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value));
+        CCtxParams->nbWorkers = value;
+        return CCtxParams->nbWorkers;
 #endif
 
     case ZSTD_c_jobSize :
 #ifndef ZSTD_MULTITHREAD
-        RETURN_ERROR(parameter_unsupported, "not compiled with multithreading");
+        RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading");
+        return 0;
 #else
-        return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_jobSize, value);
+        /* Adjust to the minimum non-default value. */
+        if (value != 0 && value < ZSTDMT_JOBSIZE_MIN)
+            value = ZSTDMT_JOBSIZE_MIN;
+        FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value));
+        assert(value >= 0);
+        CCtxParams->jobSize = value;
+        return CCtxParams->jobSize;
 #endif
 
     case ZSTD_c_overlapLog :
 #ifndef ZSTD_MULTITHREAD
-        RETURN_ERROR(parameter_unsupported, "not compiled with multithreading");
+        RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading");
+        return 0;
 #else
-        return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_overlapLog, value);
+        FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value));
+        CCtxParams->overlapLog = value;
+        return CCtxParams->overlapLog;
 #endif
 
     case ZSTD_c_rsyncable :
 #ifndef ZSTD_MULTITHREAD
-        RETURN_ERROR(parameter_unsupported, "not compiled with multithreading");
+        RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading");
+        return 0;
 #else
-        return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_rsyncable, value);
+        FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value));
+        CCtxParams->rsyncable = value;
+        return CCtxParams->rsyncable;
 #endif
 
     case ZSTD_c_enableLongDistanceMatching :
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index ffa44bc4ee6..419352e7031 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -864,11 +864,7 @@ static size_t ZSTDMT_expandJobsTable (ZSTDMT_CCtx* mtctx, U32 nbWorkers) {
  * Internal use only */
 size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorkers)
 {
-    if (nbWorkers > ZSTDMT_NBWORKERS_MAX) nbWorkers = ZSTDMT_NBWORKERS_MAX;
-    params->nbWorkers = nbWorkers;
-    params->overlapLog = ZSTDMT_OVERLAPLOG_DEFAULT;
-    params->jobSize = 0;
-    return nbWorkers;
+    return ZSTD_CCtxParam_setParameter(params, ZSTD_c_nbWorkers, (int)nbWorkers);
 }
 
 ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, ZSTD_customMem cMem)
@@ -986,26 +982,13 @@ ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params,
     {
     case ZSTDMT_p_jobSize :
         DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter : set jobSize to %i", value);
-        if ( value != 0  /* default */
-          && value < ZSTDMT_JOBSIZE_MIN)
-            value = ZSTDMT_JOBSIZE_MIN;
-        assert(value >= 0);
-        if (value > ZSTDMT_JOBSIZE_MAX) value = ZSTDMT_JOBSIZE_MAX;
-        params->jobSize = value;
-        return value;
-
+        return ZSTD_CCtxParam_setParameter(params, ZSTD_c_jobSize, value);
     case ZSTDMT_p_overlapLog :
         DEBUGLOG(4, "ZSTDMT_p_overlapLog : %i", value);
-        if (value < ZSTD_OVERLAPLOG_MIN) value = ZSTD_OVERLAPLOG_MIN;
-        if (value > ZSTD_OVERLAPLOG_MAX) value = ZSTD_OVERLAPLOG_MAX;
-        params->overlapLog = value;
-        return value;
-
+        return ZSTD_CCtxParam_setParameter(params, ZSTD_c_overlapLog, value);
     case ZSTDMT_p_rsyncable :
-        value = (value != 0);
-        params->rsyncable = value;
-        return value;
-
+        DEBUGLOG(4, "ZSTD_p_rsyncable : %i", value);
+        return ZSTD_CCtxParam_setParameter(params, ZSTD_c_rsyncable, value);
     default :
         return ERROR(parameter_unsupported);
     }
@@ -1021,32 +1004,29 @@ size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter,
 {
     switch (parameter) {
     case ZSTDMT_p_jobSize:
-        assert(mtctx->params.jobSize <= INT_MAX);
-        *value = (int)(mtctx->params.jobSize);
-        break;
+        return ZSTD_CCtxParam_getParameter(&mtctx->params, ZSTD_c_jobSize, value);
     case ZSTDMT_p_overlapLog:
-        *value = mtctx->params.overlapLog;
-        break;
+        return ZSTD_CCtxParam_getParameter(&mtctx->params, ZSTD_c_overlapLog, value);
     case ZSTDMT_p_rsyncable:
-        *value = mtctx->params.rsyncable;
-        break;
+        return ZSTD_CCtxParam_getParameter(&mtctx->params, ZSTD_c_rsyncable, value);
     default:
         return ERROR(parameter_unsupported);
     }
-    return 0;
 }
 
 /* Sets parameters relevant to the compression job,
  * initializing others to default values. */
 static ZSTD_CCtx_params ZSTDMT_initJobCCtxParams(ZSTD_CCtx_params const params)
 {
-    ZSTD_CCtx_params jobParams;
-    memset(&jobParams, 0, sizeof(jobParams));
-
-    jobParams.cParams = params.cParams;
-    jobParams.fParams = params.fParams;
-    jobParams.compressionLevel = params.compressionLevel;
-
+    ZSTD_CCtx_params jobParams = params;
+    /* Clear parameters related to multithreading */
+    jobParams.forceWindow = 0;
+    jobParams.nbWorkers = 0;
+    jobParams.jobSize = 0;
+    jobParams.overlapLog = 0;
+    jobParams.rsyncable = 0;
+    memset(&jobParams.ldmParams, 0, sizeof(ldmParams_t));
+    memset(&jobParams.customMem, 0, sizeof(ZSTD_customMem));
     return jobParams;
 }
 
diff --git a/tests/regression/config.c b/tests/regression/config.c
index bd364009941..262cb605dc8 100644
--- a/tests/regression/config.c
+++ b/tests/regression/config.c
@@ -90,6 +90,17 @@ static config_t mt_ldm = {
     .param_values = PARAM_VALUES(mt_ldm_param_values),
 };
 
+static param_value_t mt_advanced_param_values[] = {
+    {.param = ZSTD_c_nbWorkers, .value = 2},
+    {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_lcm_uncompressed},
+};
+
+static config_t mt_advanced = {
+    .name = "multithreaded with advanced params",
+    .cli_args = "-T2 --no-compressed-literals",
+    .param_values = PARAM_VALUES(mt_advanced_param_values),
+};
+
 static param_value_t const small_wlog_param_values[] = {
     {.param = ZSTD_c_windowLog, .value = 10},
 };
@@ -191,6 +202,7 @@ static config_t const* g_configs[] = {
     &uncompressed_literals,
     &uncompressed_literals_opt,
     &huffman_literals,
+    &mt_advanced,
     NULL,
 };
 
diff --git a/tests/regression/results.csv b/tests/regression/results.csv
index 23afae5ee41..a24553b018f 100644
--- a/tests/regression/results.csv
+++ b/tests/regression/results.csv
@@ -1,502 +1,516 @@
-Data,                             Config,                           Method,                           Total compressed size
-silesia.tar,                      level -5,                         compress simple,                  7160438
-silesia.tar,                      level -3,                         compress simple,                  6789024
-silesia.tar,                      level -1,                         compress simple,                  6195462
-silesia.tar,                      level 0,                          compress simple,                  4875008
-silesia.tar,                      level 1,                          compress simple,                  5339697
-silesia.tar,                      level 3,                          compress simple,                  4875008
-silesia.tar,                      level 4,                          compress simple,                  4813507
-silesia.tar,                      level 5,                          compress simple,                  4722235
-silesia.tar,                      level 6,                          compress simple,                  4672194
-silesia.tar,                      level 7,                          compress simple,                  4606658
-silesia.tar,                      level 9,                          compress simple,                  4554098
-silesia.tar,                      level 13,                         compress simple,                  4491702
-silesia.tar,                      level 16,                         compress simple,                  4381277
-silesia.tar,                      level 19,                         compress simple,                  4281514
-silesia.tar,                      uncompressed literals,            compress simple,                  4875008
-silesia.tar,                      uncompressed literals optimal,    compress simple,                  4281514
-silesia.tar,                      huffman literals,                 compress simple,                  6195462
-silesia,                          level -5,                         compress cctx,                    7152294
-silesia,                          level -3,                         compress cctx,                    6789969
-silesia,                          level -1,                         compress cctx,                    6191548
-silesia,                          level 0,                          compress cctx,                    4862377
-silesia,                          level 1,                          compress cctx,                    5318036
-silesia,                          level 3,                          compress cctx,                    4862377
-silesia,                          level 4,                          compress cctx,                    4800629
-silesia,                          level 5,                          compress cctx,                    4710178
-silesia,                          level 6,                          compress cctx,                    4659996
-silesia,                          level 7,                          compress cctx,                    4596234
-silesia,                          level 9,                          compress cctx,                    4543862
-silesia,                          level 13,                         compress cctx,                    4482073
-silesia,                          level 16,                         compress cctx,                    4377391
-silesia,                          level 19,                         compress cctx,                    4293262
-silesia,                          long distance mode,               compress cctx,                    4862377
-silesia,                          multithreaded,                    compress cctx,                    4862377
-silesia,                          multithreaded long distance mode, compress cctx,                    4862377
-silesia,                          small window log,                 compress cctx,                    7115734
-silesia,                          small hash log,                   compress cctx,                    6554898
-silesia,                          small chain log,                  compress cctx,                    4931093
-silesia,                          explicit params,                  compress cctx,                    4813352
-silesia,                          uncompressed literals,            compress cctx,                    4862377
-silesia,                          uncompressed literals optimal,    compress cctx,                    4293262
-silesia,                          huffman literals,                 compress cctx,                    6191548
-github,                           level -5,                         compress cctx,                    232744
-github,                           level -5 with dict,               compress cctx,                    47294
-github,                           level -3,                         compress cctx,                    220611
-github,                           level -3 with dict,               compress cctx,                    48047
-github,                           level -1,                         compress cctx,                    176575
-github,                           level -1 with dict,               compress cctx,                    43527
-github,                           level 0,                          compress cctx,                    136397
-github,                           level 0 with dict,                compress cctx,                    41536
-github,                           level 1,                          compress cctx,                    143457
-github,                           level 1 with dict,                compress cctx,                    42157
-github,                           level 3,                          compress cctx,                    136397
-github,                           level 3 with dict,                compress cctx,                    41536
-github,                           level 4,                          compress cctx,                    136144
-github,                           level 4 with dict,                compress cctx,                    41721
-github,                           level 5,                          compress cctx,                    135106
-github,                           level 5 with dict,                compress cctx,                    38934
-github,                           level 6,                          compress cctx,                    135108
-github,                           level 6 with dict,                compress cctx,                    38628
-github,                           level 7,                          compress cctx,                    135108
-github,                           level 7 with dict,                compress cctx,                    38741
-github,                           level 9,                          compress cctx,                    135108
-github,                           level 9 with dict,                compress cctx,                    39335
-github,                           level 13,                         compress cctx,                    133717
-github,                           level 13 with dict,               compress cctx,                    39923
-github,                           level 16,                         compress cctx,                    133717
-github,                           level 16 with dict,               compress cctx,                    37568
-github,                           level 19,                         compress cctx,                    133717
-github,                           level 19 with dict,               compress cctx,                    37567
-github,                           long distance mode,               compress cctx,                    141473
-github,                           multithreaded,                    compress cctx,                    141473
-github,                           multithreaded long distance mode, compress cctx,                    141473
-github,                           small window log,                 compress cctx,                    141473
-github,                           small hash log,                   compress cctx,                    138943
-github,                           small chain log,                  compress cctx,                    139239
-github,                           explicit params,                  compress cctx,                    140924
-github,                           uncompressed literals,            compress cctx,                    136397
-github,                           uncompressed literals optimal,    compress cctx,                    133717
-github,                           huffman literals,                 compress cctx,                    176575
-silesia,                          level -5,                         zstdcli,                          7152342
-silesia,                          level -3,                         zstdcli,                          6790021
-silesia,                          level -1,                         zstdcli,                          6191597
-silesia,                          level 0,                          zstdcli,                          4862425
-silesia,                          level 1,                          zstdcli,                          5318084
-silesia,                          level 3,                          zstdcli,                          4862425
-silesia,                          level 4,                          zstdcli,                          4800677
-silesia,                          level 5,                          zstdcli,                          4710226
-silesia,                          level 6,                          zstdcli,                          4660044
-silesia,                          level 7,                          zstdcli,                          4596282
-silesia,                          level 9,                          zstdcli,                          4543910
-silesia,                          level 13,                         zstdcli,                          4482121
-silesia,                          level 16,                         zstdcli,                          4377439
-silesia,                          level 19,                         zstdcli,                          4293310
-silesia,                          long distance mode,               zstdcli,                          4853437
-silesia,                          multithreaded,                    zstdcli,                          4862425
-silesia,                          multithreaded long distance mode, zstdcli,                          4853437
-silesia,                          small window log,                 zstdcli,                          7126434
-silesia,                          small hash log,                   zstdcli,                          6554946
-silesia,                          small chain log,                  zstdcli,                          4931141
-silesia,                          explicit params,                  zstdcli,                          4815380
-silesia,                          uncompressed literals,            zstdcli,                          5155472
-silesia,                          uncompressed literals optimal,    zstdcli,                          4325475
-silesia,                          huffman literals,                 zstdcli,                          5341405
-silesia.tar,                      level -5,                         zstdcli,                          7161160
-silesia.tar,                      level -3,                         zstdcli,                          6789865
-silesia.tar,                      level -1,                         zstdcli,                          6196433
-silesia.tar,                      level 0,                          zstdcli,                          4875136
-silesia.tar,                      level 1,                          zstdcli,                          5340573
-silesia.tar,                      level 3,                          zstdcli,                          4875136
-silesia.tar,                      level 4,                          zstdcli,                          4814531
-silesia.tar,                      level 5,                          zstdcli,                          4723284
-silesia.tar,                      level 6,                          zstdcli,                          4673591
-silesia.tar,                      level 7,                          zstdcli,                          4608342
-silesia.tar,                      level 9,                          zstdcli,                          4554700
-silesia.tar,                      level 13,                         zstdcli,                          4491706
-silesia.tar,                      level 16,                         zstdcli,                          4381281
-silesia.tar,                      level 19,                         zstdcli,                          4281518
-silesia.tar,                      no source size,                   zstdcli,                          4875132
-silesia.tar,                      long distance mode,               zstdcli,                          4866975
-silesia.tar,                      multithreaded,                    zstdcli,                          4875136
-silesia.tar,                      multithreaded long distance mode, zstdcli,                          4866975
-silesia.tar,                      small window log,                 zstdcli,                          7130434
-silesia.tar,                      small hash log,                   zstdcli,                          6587841
-silesia.tar,                      small chain log,                  zstdcli,                          4943259
-silesia.tar,                      explicit params,                  zstdcli,                          4839202
-silesia.tar,                      uncompressed literals,            zstdcli,                          5158134
-silesia.tar,                      uncompressed literals optimal,    zstdcli,                          4321098
-silesia.tar,                      huffman literals,                 zstdcli,                          5358479
-github,                           level -5,                         zstdcli,                          234744
-github,                           level -5 with dict,               zstdcli,                          48718
-github,                           level -3,                         zstdcli,                          222611
-github,                           level -3 with dict,               zstdcli,                          47395
-github,                           level -1,                         zstdcli,                          178575
-github,                           level -1 with dict,               zstdcli,                          45170
-github,                           level 0,                          zstdcli,                          138397
-github,                           level 0 with dict,                zstdcli,                          43170
-github,                           level 1,                          zstdcli,                          145457
-github,                           level 1 with dict,                zstdcli,                          43682
-github,                           level 3,                          zstdcli,                          138397
-github,                           level 3 with dict,                zstdcli,                          43170
-github,                           level 4,                          zstdcli,                          138144
-github,                           level 4 with dict,                zstdcli,                          43306
-github,                           level 5,                          zstdcli,                          137106
-github,                           level 5 with dict,                zstdcli,                          40938
-github,                           level 6,                          zstdcli,                          137108
-github,                           level 6 with dict,                zstdcli,                          40632
-github,                           level 7,                          zstdcli,                          137108
-github,                           level 7 with dict,                zstdcli,                          40766
-github,                           level 9,                          zstdcli,                          137108
-github,                           level 9 with dict,                zstdcli,                          41326
-github,                           level 13,                         zstdcli,                          135717
-github,                           level 13 with dict,               zstdcli,                          41716
-github,                           level 16,                         zstdcli,                          135717
-github,                           level 16 with dict,               zstdcli,                          39577
-github,                           level 19,                         zstdcli,                          135717
-github,                           level 19 with dict,               zstdcli,                          39576
-github,                           long distance mode,               zstdcli,                          138397
-github,                           multithreaded,                    zstdcli,                          138397
-github,                           multithreaded long distance mode, zstdcli,                          138397
-github,                           small window log,                 zstdcli,                          138397
-github,                           small hash log,                   zstdcli,                          137467
-github,                           small chain log,                  zstdcli,                          138314
-github,                           explicit params,                  zstdcli,                          136140
-github,                           uncompressed literals,            zstdcli,                          169004
-github,                           uncompressed literals optimal,    zstdcli,                          158824
-github,                           huffman literals,                 zstdcli,                          145457
-silesia,                          level -5,                         advanced one pass,                7152294
-silesia,                          level -3,                         advanced one pass,                6789969
-silesia,                          level -1,                         advanced one pass,                6191548
-silesia,                          level 0,                          advanced one pass,                4862377
-silesia,                          level 1,                          advanced one pass,                5318036
-silesia,                          level 3,                          advanced one pass,                4862377
-silesia,                          level 4,                          advanced one pass,                4800629
-silesia,                          level 5,                          advanced one pass,                4710178
-silesia,                          level 6,                          advanced one pass,                4659996
-silesia,                          level 7,                          advanced one pass,                4596234
-silesia,                          level 9,                          advanced one pass,                4543862
-silesia,                          level 13,                         advanced one pass,                4482073
-silesia,                          level 16,                         advanced one pass,                4377391
-silesia,                          level 19,                         advanced one pass,                4293262
-silesia,                          no source size,                   advanced one pass,                4862377
-silesia,                          long distance mode,               advanced one pass,                4853389
-silesia,                          multithreaded,                    advanced one pass,                4862377
-silesia,                          multithreaded long distance mode, advanced one pass,                4853389
-silesia,                          small window log,                 advanced one pass,                7126386
-silesia,                          small hash log,                   advanced one pass,                6554898
-silesia,                          small chain log,                  advanced one pass,                4931093
-silesia,                          explicit params,                  advanced one pass,                4815369
-silesia,                          uncompressed literals,            advanced one pass,                5155424
-silesia,                          uncompressed literals optimal,    advanced one pass,                4325427
-silesia,                          huffman literals,                 advanced one pass,                5341356
-silesia.tar,                      level -5,                         advanced one pass,                7160438
-silesia.tar,                      level -3,                         advanced one pass,                6789024
-silesia.tar,                      level -1,                         advanced one pass,                6195462
-silesia.tar,                      level 0,                          advanced one pass,                4875008
-silesia.tar,                      level 1,                          advanced one pass,                5339697
-silesia.tar,                      level 3,                          advanced one pass,                4875008
-silesia.tar,                      level 4,                          advanced one pass,                4813507
-silesia.tar,                      level 5,                          advanced one pass,                4722235
-silesia.tar,                      level 6,                          advanced one pass,                4672194
-silesia.tar,                      level 7,                          advanced one pass,                4606658
-silesia.tar,                      level 9,                          advanced one pass,                4554098
-silesia.tar,                      level 13,                         advanced one pass,                4491702
-silesia.tar,                      level 16,                         advanced one pass,                4381277
-silesia.tar,                      level 19,                         advanced one pass,                4281514
-silesia.tar,                      no source size,                   advanced one pass,                4875008
-silesia.tar,                      long distance mode,               advanced one pass,                4861218
-silesia.tar,                      multithreaded,                    advanced one pass,                4874631
-silesia.tar,                      multithreaded long distance mode, advanced one pass,                4860683
-silesia.tar,                      small window log,                 advanced one pass,                7130394
-silesia.tar,                      small hash log,                   advanced one pass,                6587833
-silesia.tar,                      small chain log,                  advanced one pass,                4943255
-silesia.tar,                      explicit params,                  advanced one pass,                4829974
-silesia.tar,                      uncompressed literals,            advanced one pass,                5157992
-silesia.tar,                      uncompressed literals optimal,    advanced one pass,                4321094
-silesia.tar,                      huffman literals,                 advanced one pass,                5358079
-github,                           level -5,                         advanced one pass,                232744
-github,                           level -5 with dict,               advanced one pass,                46718
-github,                           level -3,                         advanced one pass,                220611
-github,                           level -3 with dict,               advanced one pass,                45395
-github,                           level -1,                         advanced one pass,                176575
-github,                           level -1 with dict,               advanced one pass,                43170
-github,                           level 0,                          advanced one pass,                136397
-github,                           level 0 with dict,                advanced one pass,                41170
-github,                           level 1,                          advanced one pass,                143457
-github,                           level 1 with dict,                advanced one pass,                41682
-github,                           level 3,                          advanced one pass,                136397
-github,                           level 3 with dict,                advanced one pass,                41170
-github,                           level 4,                          advanced one pass,                136144
-github,                           level 4 with dict,                advanced one pass,                41306
-github,                           level 5,                          advanced one pass,                135106
-github,                           level 5 with dict,                advanced one pass,                38938
-github,                           level 6,                          advanced one pass,                135108
-github,                           level 6 with dict,                advanced one pass,                38632
-github,                           level 7,                          advanced one pass,                135108
-github,                           level 7 with dict,                advanced one pass,                38766
-github,                           level 9,                          advanced one pass,                135108
-github,                           level 9 with dict,                advanced one pass,                39326
-github,                           level 13,                         advanced one pass,                133717
-github,                           level 13 with dict,               advanced one pass,                39716
-github,                           level 16,                         advanced one pass,                133717
-github,                           level 16 with dict,               advanced one pass,                37577
-github,                           level 19,                         advanced one pass,                133717
-github,                           level 19 with dict,               advanced one pass,                37576
-github,                           no source size,                   advanced one pass,                136397
-github,                           long distance mode,               advanced one pass,                136397
-github,                           multithreaded,                    advanced one pass,                136397
-github,                           multithreaded long distance mode, advanced one pass,                136397
-github,                           small window log,                 advanced one pass,                136397
-github,                           small hash log,                   advanced one pass,                135467
-github,                           small chain log,                  advanced one pass,                136314
-github,                           explicit params,                  advanced one pass,                137670
-github,                           uncompressed literals,            advanced one pass,                167004
-github,                           uncompressed literals optimal,    advanced one pass,                156824
-github,                           huffman literals,                 advanced one pass,                143457
-silesia,                          level -5,                         advanced one pass small out,      7152294
-silesia,                          level -3,                         advanced one pass small out,      6789969
-silesia,                          level -1,                         advanced one pass small out,      6191548
-silesia,                          level 0,                          advanced one pass small out,      4862377
-silesia,                          level 1,                          advanced one pass small out,      5318036
-silesia,                          level 3,                          advanced one pass small out,      4862377
-silesia,                          level 4,                          advanced one pass small out,      4800629
-silesia,                          level 5,                          advanced one pass small out,      4710178
-silesia,                          level 6,                          advanced one pass small out,      4659996
-silesia,                          level 7,                          advanced one pass small out,      4596234
-silesia,                          level 9,                          advanced one pass small out,      4543862
-silesia,                          level 13,                         advanced one pass small out,      4482073
-silesia,                          level 16,                         advanced one pass small out,      4377391
-silesia,                          level 19,                         advanced one pass small out,      4293262
-silesia,                          no source size,                   advanced one pass small out,      4862377
-silesia,                          long distance mode,               advanced one pass small out,      4853389
-silesia,                          multithreaded,                    advanced one pass small out,      4862377
-silesia,                          multithreaded long distance mode, advanced one pass small out,      4853389
-silesia,                          small window log,                 advanced one pass small out,      7126386
-silesia,                          small hash log,                   advanced one pass small out,      6554898
-silesia,                          small chain log,                  advanced one pass small out,      4931093
-silesia,                          explicit params,                  advanced one pass small out,      4815369
-silesia,                          uncompressed literals,            advanced one pass small out,      5155424
-silesia,                          uncompressed literals optimal,    advanced one pass small out,      4325427
-silesia,                          huffman literals,                 advanced one pass small out,      5341356
-silesia.tar,                      level -5,                         advanced one pass small out,      7160438
-silesia.tar,                      level -3,                         advanced one pass small out,      6789024
-silesia.tar,                      level -1,                         advanced one pass small out,      6195462
-silesia.tar,                      level 0,                          advanced one pass small out,      4875008
-silesia.tar,                      level 1,                          advanced one pass small out,      5339697
-silesia.tar,                      level 3,                          advanced one pass small out,      4875008
-silesia.tar,                      level 4,                          advanced one pass small out,      4813507
-silesia.tar,                      level 5,                          advanced one pass small out,      4722235
-silesia.tar,                      level 6,                          advanced one pass small out,      4672194
-silesia.tar,                      level 7,                          advanced one pass small out,      4606658
-silesia.tar,                      level 9,                          advanced one pass small out,      4554098
-silesia.tar,                      level 13,                         advanced one pass small out,      4491702
-silesia.tar,                      level 16,                         advanced one pass small out,      4381277
-silesia.tar,                      level 19,                         advanced one pass small out,      4281514
-silesia.tar,                      no source size,                   advanced one pass small out,      4875008
-silesia.tar,                      long distance mode,               advanced one pass small out,      4861218
-silesia.tar,                      multithreaded,                    advanced one pass small out,      4874631
-silesia.tar,                      multithreaded long distance mode, advanced one pass small out,      4860683
-silesia.tar,                      small window log,                 advanced one pass small out,      7130394
-silesia.tar,                      small hash log,                   advanced one pass small out,      6587833
-silesia.tar,                      small chain log,                  advanced one pass small out,      4943255
-silesia.tar,                      explicit params,                  advanced one pass small out,      4829974
-silesia.tar,                      uncompressed literals,            advanced one pass small out,      5157992
-silesia.tar,                      uncompressed literals optimal,    advanced one pass small out,      4321094
-silesia.tar,                      huffman literals,                 advanced one pass small out,      5358079
-github,                           level -5,                         advanced one pass small out,      232744
-github,                           level -5 with dict,               advanced one pass small out,      46718
-github,                           level -3,                         advanced one pass small out,      220611
-github,                           level -3 with dict,               advanced one pass small out,      45395
-github,                           level -1,                         advanced one pass small out,      176575
-github,                           level -1 with dict,               advanced one pass small out,      43170
-github,                           level 0,                          advanced one pass small out,      136397
-github,                           level 0 with dict,                advanced one pass small out,      41170
-github,                           level 1,                          advanced one pass small out,      143457
-github,                           level 1 with dict,                advanced one pass small out,      41682
-github,                           level 3,                          advanced one pass small out,      136397
-github,                           level 3 with dict,                advanced one pass small out,      41170
-github,                           level 4,                          advanced one pass small out,      136144
-github,                           level 4 with dict,                advanced one pass small out,      41306
-github,                           level 5,                          advanced one pass small out,      135106
-github,                           level 5 with dict,                advanced one pass small out,      38938
-github,                           level 6,                          advanced one pass small out,      135108
-github,                           level 6 with dict,                advanced one pass small out,      38632
-github,                           level 7,                          advanced one pass small out,      135108
-github,                           level 7 with dict,                advanced one pass small out,      38766
-github,                           level 9,                          advanced one pass small out,      135108
-github,                           level 9 with dict,                advanced one pass small out,      39326
-github,                           level 13,                         advanced one pass small out,      133717
-github,                           level 13 with dict,               advanced one pass small out,      39716
-github,                           level 16,                         advanced one pass small out,      133717
-github,                           level 16 with dict,               advanced one pass small out,      37577
-github,                           level 19,                         advanced one pass small out,      133717
-github,                           level 19 with dict,               advanced one pass small out,      37576
-github,                           no source size,                   advanced one pass small out,      136397
-github,                           long distance mode,               advanced one pass small out,      136397
-github,                           multithreaded,                    advanced one pass small out,      136397
-github,                           multithreaded long distance mode, advanced one pass small out,      136397
-github,                           small window log,                 advanced one pass small out,      136397
-github,                           small hash log,                   advanced one pass small out,      135467
-github,                           small chain log,                  advanced one pass small out,      136314
-github,                           explicit params,                  advanced one pass small out,      137670
-github,                           uncompressed literals,            advanced one pass small out,      167004
-github,                           uncompressed literals optimal,    advanced one pass small out,      156824
-github,                           huffman literals,                 advanced one pass small out,      143457
-silesia,                          level -5,                         advanced streaming,               7152294
-silesia,                          level -3,                         advanced streaming,               6789973
-silesia,                          level -1,                         advanced streaming,               6191549
-silesia,                          level 0,                          advanced streaming,               4862377
-silesia,                          level 1,                          advanced streaming,               5318036
-silesia,                          level 3,                          advanced streaming,               4862377
-silesia,                          level 4,                          advanced streaming,               4800629
-silesia,                          level 5,                          advanced streaming,               4710178
-silesia,                          level 6,                          advanced streaming,               4659996
-silesia,                          level 7,                          advanced streaming,               4596234
-silesia,                          level 9,                          advanced streaming,               4543862
-silesia,                          level 13,                         advanced streaming,               4482073
-silesia,                          level 16,                         advanced streaming,               4377391
-silesia,                          level 19,                         advanced streaming,               4293262
-silesia,                          no source size,                   advanced streaming,               4862341
-silesia,                          long distance mode,               advanced streaming,               4853389
-silesia,                          multithreaded,                    advanced streaming,               4862377
-silesia,                          multithreaded long distance mode, advanced streaming,               4853389
-silesia,                          small window log,                 advanced streaming,               7126389
-silesia,                          small hash log,                   advanced streaming,               6554898
-silesia,                          small chain log,                  advanced streaming,               4931093
-silesia,                          explicit params,                  advanced streaming,               4815380
-silesia,                          uncompressed literals,            advanced streaming,               5155424
-silesia,                          uncompressed literals optimal,    advanced streaming,               4325427
-silesia,                          huffman literals,                 advanced streaming,               5341357
-silesia.tar,                      level -5,                         advanced streaming,               7160440
-silesia.tar,                      level -3,                         advanced streaming,               6789026
-silesia.tar,                      level -1,                         advanced streaming,               6195465
-silesia.tar,                      level 0,                          advanced streaming,               4875010
-silesia.tar,                      level 1,                          advanced streaming,               5339701
-silesia.tar,                      level 3,                          advanced streaming,               4875010
-silesia.tar,                      level 4,                          advanced streaming,               4813507
-silesia.tar,                      level 5,                          advanced streaming,               4722240
-silesia.tar,                      level 6,                          advanced streaming,               4672203
-silesia.tar,                      level 7,                          advanced streaming,               4606658
-silesia.tar,                      level 9,                          advanced streaming,               4554105
-silesia.tar,                      level 13,                         advanced streaming,               4491703
-silesia.tar,                      level 16,                         advanced streaming,               4381277
-silesia.tar,                      level 19,                         advanced streaming,               4281514
-silesia.tar,                      no source size,                   advanced streaming,               4875006
-silesia.tar,                      long distance mode,               advanced streaming,               4861218
-silesia.tar,                      multithreaded,                    advanced streaming,               4875132
-silesia.tar,                      multithreaded long distance mode, advanced streaming,               4866971
-silesia.tar,                      small window log,                 advanced streaming,               7130394
-silesia.tar,                      small hash log,                   advanced streaming,               6587834
-silesia.tar,                      small chain log,                  advanced streaming,               4943260
-silesia.tar,                      explicit params,                  advanced streaming,               4830002
-silesia.tar,                      uncompressed literals,            advanced streaming,               5157995
-silesia.tar,                      uncompressed literals optimal,    advanced streaming,               4321094
-silesia.tar,                      huffman literals,                 advanced streaming,               5358083
-github,                           level -5,                         advanced streaming,               232744
-github,                           level -5 with dict,               advanced streaming,               46718
-github,                           level -3,                         advanced streaming,               220611
-github,                           level -3 with dict,               advanced streaming,               45395
-github,                           level -1,                         advanced streaming,               176575
-github,                           level -1 with dict,               advanced streaming,               43170
-github,                           level 0,                          advanced streaming,               136397
-github,                           level 0 with dict,                advanced streaming,               41170
-github,                           level 1,                          advanced streaming,               143457
-github,                           level 1 with dict,                advanced streaming,               41682
-github,                           level 3,                          advanced streaming,               136397
-github,                           level 3 with dict,                advanced streaming,               41170
-github,                           level 4,                          advanced streaming,               136144
-github,                           level 4 with dict,                advanced streaming,               41306
-github,                           level 5,                          advanced streaming,               135106
-github,                           level 5 with dict,                advanced streaming,               38938
-github,                           level 6,                          advanced streaming,               135108
-github,                           level 6 with dict,                advanced streaming,               38632
-github,                           level 7,                          advanced streaming,               135108
-github,                           level 7 with dict,                advanced streaming,               38766
-github,                           level 9,                          advanced streaming,               135108
-github,                           level 9 with dict,                advanced streaming,               39326
-github,                           level 13,                         advanced streaming,               133717
-github,                           level 13 with dict,               advanced streaming,               39716
-github,                           level 16,                         advanced streaming,               133717
-github,                           level 16 with dict,               advanced streaming,               37577
-github,                           level 19,                         advanced streaming,               133717
-github,                           level 19 with dict,               advanced streaming,               37576
-github,                           no source size,                   advanced streaming,               136397
-github,                           long distance mode,               advanced streaming,               136397
-github,                           multithreaded,                    advanced streaming,               136397
-github,                           multithreaded long distance mode, advanced streaming,               136397
-github,                           small window log,                 advanced streaming,               136397
-github,                           small hash log,                   advanced streaming,               135467
-github,                           small chain log,                  advanced streaming,               136314
-github,                           explicit params,                  advanced streaming,               137670
-github,                           uncompressed literals,            advanced streaming,               167004
-github,                           uncompressed literals optimal,    advanced streaming,               156824
-github,                           huffman literals,                 advanced streaming,               143457
-silesia,                          level -5,                         old streaming,                    7152294
-silesia,                          level -3,                         old streaming,                    6789973
-silesia,                          level -1,                         old streaming,                    6191549
-silesia,                          level 0,                          old streaming,                    4862377
-silesia,                          level 1,                          old streaming,                    5318036
-silesia,                          level 3,                          old streaming,                    4862377
-silesia,                          level 4,                          old streaming,                    4800629
-silesia,                          level 5,                          old streaming,                    4710178
-silesia,                          level 6,                          old streaming,                    4659996
-silesia,                          level 7,                          old streaming,                    4596234
-silesia,                          level 9,                          old streaming,                    4543862
-silesia,                          level 13,                         old streaming,                    4482073
-silesia,                          level 16,                         old streaming,                    4377391
-silesia,                          level 19,                         old streaming,                    4293262
-silesia,                          no source size,                   old streaming,                    4862341
-silesia,                          uncompressed literals,            old streaming,                    4862377
-silesia,                          uncompressed literals optimal,    old streaming,                    4293262
-silesia,                          huffman literals,                 old streaming,                    6191549
-silesia.tar,                      level -5,                         old streaming,                    7160440
-silesia.tar,                      level -3,                         old streaming,                    6789026
-silesia.tar,                      level -1,                         old streaming,                    6195465
-silesia.tar,                      level 0,                          old streaming,                    4875010
-silesia.tar,                      level 1,                          old streaming,                    5339701
-silesia.tar,                      level 3,                          old streaming,                    4875010
-silesia.tar,                      level 4,                          old streaming,                    4813507
-silesia.tar,                      level 5,                          old streaming,                    4722240
-silesia.tar,                      level 6,                          old streaming,                    4672203
-silesia.tar,                      level 7,                          old streaming,                    4606658
-silesia.tar,                      level 9,                          old streaming,                    4554105
-silesia.tar,                      level 13,                         old streaming,                    4491703
-silesia.tar,                      level 16,                         old streaming,                    4381277
-silesia.tar,                      level 19,                         old streaming,                    4281514
-silesia.tar,                      no source size,                   old streaming,                    4875006
-silesia.tar,                      uncompressed literals,            old streaming,                    4875010
-silesia.tar,                      uncompressed literals optimal,    old streaming,                    4281514
-silesia.tar,                      huffman literals,                 old streaming,                    6195465
-github,                           level -5,                         old streaming,                    232744
-github,                           level -5 with dict,               old streaming,                    46718
-github,                           level -3,                         old streaming,                    220611
-github,                           level -3 with dict,               old streaming,                    45395
-github,                           level -1,                         old streaming,                    176575
-github,                           level -1 with dict,               old streaming,                    43170
-github,                           level 0,                          old streaming,                    136397
-github,                           level 0 with dict,                old streaming,                    41170
-github,                           level 1,                          old streaming,                    143457
-github,                           level 1 with dict,                old streaming,                    41682
-github,                           level 3,                          old streaming,                    136397
-github,                           level 3 with dict,                old streaming,                    41170
-github,                           level 4,                          old streaming,                    136144
-github,                           level 4 with dict,                old streaming,                    41306
-github,                           level 5,                          old streaming,                    135106
-github,                           level 5 with dict,                old streaming,                    38938
-github,                           level 6,                          old streaming,                    135108
-github,                           level 6 with dict,                old streaming,                    38632
-github,                           level 7,                          old streaming,                    135108
-github,                           level 7 with dict,                old streaming,                    38766
-github,                           level 9,                          old streaming,                    135108
-github,                           level 9 with dict,                old streaming,                    39326
-github,                           level 13,                         old streaming,                    133717
-github,                           level 13 with dict,               old streaming,                    39716
-github,                           level 16,                         old streaming,                    133717
-github,                           level 16 with dict,               old streaming,                    37577
-github,                           level 19,                         old streaming,                    133717
-github,                           level 19 with dict,               old streaming,                    37576
-github,                           no source size,                   old streaming,                    141003
-github,                           uncompressed literals,            old streaming,                    136397
-github,                           uncompressed literals optimal,    old streaming,                    133717
-github,                           huffman literals,                 old streaming,                    176575
+Data,                               Config,                             Method,                             Total compressed size
+silesia.tar,                        level -5,                           compress simple,                    7160438
+silesia.tar,                        level -3,                           compress simple,                    6789024
+silesia.tar,                        level -1,                           compress simple,                    6195462
+silesia.tar,                        level 0,                            compress simple,                    4875008
+silesia.tar,                        level 1,                            compress simple,                    5339697
+silesia.tar,                        level 3,                            compress simple,                    4875008
+silesia.tar,                        level 4,                            compress simple,                    4813507
+silesia.tar,                        level 5,                            compress simple,                    4722235
+silesia.tar,                        level 6,                            compress simple,                    4672194
+silesia.tar,                        level 7,                            compress simple,                    4606658
+silesia.tar,                        level 9,                            compress simple,                    4554098
+silesia.tar,                        level 13,                           compress simple,                    4491702
+silesia.tar,                        level 16,                           compress simple,                    4381277
+silesia.tar,                        level 19,                           compress simple,                    4281514
+silesia.tar,                        uncompressed literals,              compress simple,                    4875008
+silesia.tar,                        uncompressed literals optimal,      compress simple,                    4281514
+silesia.tar,                        huffman literals,                   compress simple,                    6195462
+silesia,                            level -5,                           compress cctx,                      7152294
+silesia,                            level -3,                           compress cctx,                      6789969
+silesia,                            level -1,                           compress cctx,                      6191548
+silesia,                            level 0,                            compress cctx,                      4862377
+silesia,                            level 1,                            compress cctx,                      5318036
+silesia,                            level 3,                            compress cctx,                      4862377
+silesia,                            level 4,                            compress cctx,                      4800629
+silesia,                            level 5,                            compress cctx,                      4710178
+silesia,                            level 6,                            compress cctx,                      4659996
+silesia,                            level 7,                            compress cctx,                      4596234
+silesia,                            level 9,                            compress cctx,                      4543862
+silesia,                            level 13,                           compress cctx,                      4482073
+silesia,                            level 16,                           compress cctx,                      4377391
+silesia,                            level 19,                           compress cctx,                      4293262
+silesia,                            long distance mode,                 compress cctx,                      4862377
+silesia,                            multithreaded,                      compress cctx,                      4862377
+silesia,                            multithreaded long distance mode,   compress cctx,                      4862377
+silesia,                            small window log,                   compress cctx,                      7115734
+silesia,                            small hash log,                     compress cctx,                      6554898
+silesia,                            small chain log,                    compress cctx,                      4931093
+silesia,                            explicit params,                    compress cctx,                      4813352
+silesia,                            uncompressed literals,              compress cctx,                      4862377
+silesia,                            uncompressed literals optimal,      compress cctx,                      4293262
+silesia,                            huffman literals,                   compress cctx,                      6191548
+silesia,                            multithreaded with advanced params, compress cctx,                      4862377
+github,                             level -5,                           compress cctx,                      232744
+github,                             level -5 with dict,                 compress cctx,                      47294
+github,                             level -3,                           compress cctx,                      220611
+github,                             level -3 with dict,                 compress cctx,                      48047
+github,                             level -1,                           compress cctx,                      176575
+github,                             level -1 with dict,                 compress cctx,                      43527
+github,                             level 0,                            compress cctx,                      136397
+github,                             level 0 with dict,                  compress cctx,                      41536
+github,                             level 1,                            compress cctx,                      143457
+github,                             level 1 with dict,                  compress cctx,                      42157
+github,                             level 3,                            compress cctx,                      136397
+github,                             level 3 with dict,                  compress cctx,                      41536
+github,                             level 4,                            compress cctx,                      136144
+github,                             level 4 with dict,                  compress cctx,                      41721
+github,                             level 5,                            compress cctx,                      135106
+github,                             level 5 with dict,                  compress cctx,                      38934
+github,                             level 6,                            compress cctx,                      135108
+github,                             level 6 with dict,                  compress cctx,                      38628
+github,                             level 7,                            compress cctx,                      135108
+github,                             level 7 with dict,                  compress cctx,                      38741
+github,                             level 9,                            compress cctx,                      135108
+github,                             level 9 with dict,                  compress cctx,                      39335
+github,                             level 13,                           compress cctx,                      133717
+github,                             level 13 with dict,                 compress cctx,                      39923
+github,                             level 16,                           compress cctx,                      133717
+github,                             level 16 with dict,                 compress cctx,                      37568
+github,                             level 19,                           compress cctx,                      133717
+github,                             level 19 with dict,                 compress cctx,                      37567
+github,                             long distance mode,                 compress cctx,                      141473
+github,                             multithreaded,                      compress cctx,                      141473
+github,                             multithreaded long distance mode,   compress cctx,                      141473
+github,                             small window log,                   compress cctx,                      141473
+github,                             small hash log,                     compress cctx,                      138943
+github,                             small chain log,                    compress cctx,                      139239
+github,                             explicit params,                    compress cctx,                      140924
+github,                             uncompressed literals,              compress cctx,                      136397
+github,                             uncompressed literals optimal,      compress cctx,                      133717
+github,                             huffman literals,                   compress cctx,                      176575
+github,                             multithreaded with advanced params, compress cctx,                      141473
+silesia,                            level -5,                           zstdcli,                            7152342
+silesia,                            level -3,                           zstdcli,                            6790021
+silesia,                            level -1,                           zstdcli,                            6191597
+silesia,                            level 0,                            zstdcli,                            4862425
+silesia,                            level 1,                            zstdcli,                            5318084
+silesia,                            level 3,                            zstdcli,                            4862425
+silesia,                            level 4,                            zstdcli,                            4800677
+silesia,                            level 5,                            zstdcli,                            4710226
+silesia,                            level 6,                            zstdcli,                            4660044
+silesia,                            level 7,                            zstdcli,                            4596282
+silesia,                            level 9,                            zstdcli,                            4543910
+silesia,                            level 13,                           zstdcli,                            4482121
+silesia,                            level 16,                           zstdcli,                            4377439
+silesia,                            level 19,                           zstdcli,                            4293310
+silesia,                            long distance mode,                 zstdcli,                            4853437
+silesia,                            multithreaded,                      zstdcli,                            4862425
+silesia,                            multithreaded long distance mode,   zstdcli,                            4853437
+silesia,                            small window log,                   zstdcli,                            7126434
+silesia,                            small hash log,                     zstdcli,                            6554946
+silesia,                            small chain log,                    zstdcli,                            4931141
+silesia,                            explicit params,                    zstdcli,                            4815380
+silesia,                            uncompressed literals,              zstdcli,                            5155472
+silesia,                            uncompressed literals optimal,      zstdcli,                            4325475
+silesia,                            huffman literals,                   zstdcli,                            5341405
+silesia,                            multithreaded with advanced params, zstdcli,                            compression error
+silesia.tar,                        level -5,                           zstdcli,                            7161160
+silesia.tar,                        level -3,                           zstdcli,                            6789865
+silesia.tar,                        level -1,                           zstdcli,                            6196433
+silesia.tar,                        level 0,                            zstdcli,                            4875136
+silesia.tar,                        level 1,                            zstdcli,                            5340573
+silesia.tar,                        level 3,                            zstdcli,                            4875136
+silesia.tar,                        level 4,                            zstdcli,                            4814531
+silesia.tar,                        level 5,                            zstdcli,                            4723284
+silesia.tar,                        level 6,                            zstdcli,                            4673591
+silesia.tar,                        level 7,                            zstdcli,                            4608342
+silesia.tar,                        level 9,                            zstdcli,                            4554700
+silesia.tar,                        level 13,                           zstdcli,                            4491706
+silesia.tar,                        level 16,                           zstdcli,                            4381281
+silesia.tar,                        level 19,                           zstdcli,                            4281518
+silesia.tar,                        no source size,                     zstdcli,                            4875132
+silesia.tar,                        long distance mode,                 zstdcli,                            4866975
+silesia.tar,                        multithreaded,                      zstdcli,                            4875136
+silesia.tar,                        multithreaded long distance mode,   zstdcli,                            4866975
+silesia.tar,                        small window log,                   zstdcli,                            7130434
+silesia.tar,                        small hash log,                     zstdcli,                            6587841
+silesia.tar,                        small chain log,                    zstdcli,                            4943259
+silesia.tar,                        explicit params,                    zstdcli,                            4839202
+silesia.tar,                        uncompressed literals,              zstdcli,                            5158134
+silesia.tar,                        uncompressed literals optimal,      zstdcli,                            4321098
+silesia.tar,                        huffman literals,                   zstdcli,                            5358479
+silesia.tar,                        multithreaded with advanced params, zstdcli,                            compression error
+github,                             level -5,                           zstdcli,                            234744
+github,                             level -5 with dict,                 zstdcli,                            48718
+github,                             level -3,                           zstdcli,                            222611
+github,                             level -3 with dict,                 zstdcli,                            47395
+github,                             level -1,                           zstdcli,                            178575
+github,                             level -1 with dict,                 zstdcli,                            45170
+github,                             level 0,                            zstdcli,                            138397
+github,                             level 0 with dict,                  zstdcli,                            43170
+github,                             level 1,                            zstdcli,                            145457
+github,                             level 1 with dict,                  zstdcli,                            43682
+github,                             level 3,                            zstdcli,                            138397
+github,                             level 3 with dict,                  zstdcli,                            43170
+github,                             level 4,                            zstdcli,                            138144
+github,                             level 4 with dict,                  zstdcli,                            43306
+github,                             level 5,                            zstdcli,                            137106
+github,                             level 5 with dict,                  zstdcli,                            40938
+github,                             level 6,                            zstdcli,                            137108
+github,                             level 6 with dict,                  zstdcli,                            40632
+github,                             level 7,                            zstdcli,                            137108
+github,                             level 7 with dict,                  zstdcli,                            40766
+github,                             level 9,                            zstdcli,                            137108
+github,                             level 9 with dict,                  zstdcli,                            41326
+github,                             level 13,                           zstdcli,                            135717
+github,                             level 13 with dict,                 zstdcli,                            41716
+github,                             level 16,                           zstdcli,                            135717
+github,                             level 16 with dict,                 zstdcli,                            39577
+github,                             level 19,                           zstdcli,                            135717
+github,                             level 19 with dict,                 zstdcli,                            39576
+github,                             long distance mode,                 zstdcli,                            138397
+github,                             multithreaded,                      zstdcli,                            138397
+github,                             multithreaded long distance mode,   zstdcli,                            138397
+github,                             small window log,                   zstdcli,                            138397
+github,                             small hash log,                     zstdcli,                            137467
+github,                             small chain log,                    zstdcli,                            138314
+github,                             explicit params,                    zstdcli,                            136140
+github,                             uncompressed literals,              zstdcli,                            169004
+github,                             uncompressed literals optimal,      zstdcli,                            158824
+github,                             huffman literals,                   zstdcli,                            145457
+github,                             multithreaded with advanced params, zstdcli,                            compression error
+silesia,                            level -5,                           advanced one pass,                  7152294
+silesia,                            level -3,                           advanced one pass,                  6789969
+silesia,                            level -1,                           advanced one pass,                  6191548
+silesia,                            level 0,                            advanced one pass,                  4862377
+silesia,                            level 1,                            advanced one pass,                  5318036
+silesia,                            level 3,                            advanced one pass,                  4862377
+silesia,                            level 4,                            advanced one pass,                  4800629
+silesia,                            level 5,                            advanced one pass,                  4710178
+silesia,                            level 6,                            advanced one pass,                  4659996
+silesia,                            level 7,                            advanced one pass,                  4596234
+silesia,                            level 9,                            advanced one pass,                  4543862
+silesia,                            level 13,                           advanced one pass,                  4482073
+silesia,                            level 16,                           advanced one pass,                  4377391
+silesia,                            level 19,                           advanced one pass,                  4293262
+silesia,                            no source size,                     advanced one pass,                  4862377
+silesia,                            long distance mode,                 advanced one pass,                  4853389
+silesia,                            multithreaded,                      advanced one pass,                  4862377
+silesia,                            multithreaded long distance mode,   advanced one pass,                  4853389
+silesia,                            small window log,                   advanced one pass,                  7126386
+silesia,                            small hash log,                     advanced one pass,                  6554898
+silesia,                            small chain log,                    advanced one pass,                  4931093
+silesia,                            explicit params,                    advanced one pass,                  4815369
+silesia,                            uncompressed literals,              advanced one pass,                  5155424
+silesia,                            uncompressed literals optimal,      advanced one pass,                  4325427
+silesia,                            huffman literals,                   advanced one pass,                  5341356
+silesia,                            multithreaded with advanced params, advanced one pass,                  5155424
+silesia.tar,                        level -5,                           advanced one pass,                  7160438
+silesia.tar,                        level -3,                           advanced one pass,                  6789024
+silesia.tar,                        level -1,                           advanced one pass,                  6195462
+silesia.tar,                        level 0,                            advanced one pass,                  4875008
+silesia.tar,                        level 1,                            advanced one pass,                  5339697
+silesia.tar,                        level 3,                            advanced one pass,                  4875008
+silesia.tar,                        level 4,                            advanced one pass,                  4813507
+silesia.tar,                        level 5,                            advanced one pass,                  4722235
+silesia.tar,                        level 6,                            advanced one pass,                  4672194
+silesia.tar,                        level 7,                            advanced one pass,                  4606658
+silesia.tar,                        level 9,                            advanced one pass,                  4554098
+silesia.tar,                        level 13,                           advanced one pass,                  4491702
+silesia.tar,                        level 16,                           advanced one pass,                  4381277
+silesia.tar,                        level 19,                           advanced one pass,                  4281514
+silesia.tar,                        no source size,                     advanced one pass,                  4875008
+silesia.tar,                        long distance mode,                 advanced one pass,                  4861218
+silesia.tar,                        multithreaded,                      advanced one pass,                  4874631
+silesia.tar,                        multithreaded long distance mode,   advanced one pass,                  4860683
+silesia.tar,                        small window log,                   advanced one pass,                  7130394
+silesia.tar,                        small hash log,                     advanced one pass,                  6587833
+silesia.tar,                        small chain log,                    advanced one pass,                  4943255
+silesia.tar,                        explicit params,                    advanced one pass,                  4829974
+silesia.tar,                        uncompressed literals,              advanced one pass,                  5157992
+silesia.tar,                        uncompressed literals optimal,      advanced one pass,                  4321094
+silesia.tar,                        huffman literals,                   advanced one pass,                  5358079
+silesia.tar,                        multithreaded with advanced params, advanced one pass,                  5158545
+github,                             level -5,                           advanced one pass,                  232744
+github,                             level -5 with dict,                 advanced one pass,                  46718
+github,                             level -3,                           advanced one pass,                  220611
+github,                             level -3 with dict,                 advanced one pass,                  45395
+github,                             level -1,                           advanced one pass,                  176575
+github,                             level -1 with dict,                 advanced one pass,                  43170
+github,                             level 0,                            advanced one pass,                  136397
+github,                             level 0 with dict,                  advanced one pass,                  41170
+github,                             level 1,                            advanced one pass,                  143457
+github,                             level 1 with dict,                  advanced one pass,                  41682
+github,                             level 3,                            advanced one pass,                  136397
+github,                             level 3 with dict,                  advanced one pass,                  41170
+github,                             level 4,                            advanced one pass,                  136144
+github,                             level 4 with dict,                  advanced one pass,                  41306
+github,                             level 5,                            advanced one pass,                  135106
+github,                             level 5 with dict,                  advanced one pass,                  38938
+github,                             level 6,                            advanced one pass,                  135108
+github,                             level 6 with dict,                  advanced one pass,                  38632
+github,                             level 7,                            advanced one pass,                  135108
+github,                             level 7 with dict,                  advanced one pass,                  38766
+github,                             level 9,                            advanced one pass,                  135108
+github,                             level 9 with dict,                  advanced one pass,                  39326
+github,                             level 13,                           advanced one pass,                  133717
+github,                             level 13 with dict,                 advanced one pass,                  39716
+github,                             level 16,                           advanced one pass,                  133717
+github,                             level 16 with dict,                 advanced one pass,                  37577
+github,                             level 19,                           advanced one pass,                  133717
+github,                             level 19 with dict,                 advanced one pass,                  37576
+github,                             no source size,                     advanced one pass,                  136397
+github,                             long distance mode,                 advanced one pass,                  136397
+github,                             multithreaded,                      advanced one pass,                  136397
+github,                             multithreaded long distance mode,   advanced one pass,                  136397
+github,                             small window log,                   advanced one pass,                  136397
+github,                             small hash log,                     advanced one pass,                  135467
+github,                             small chain log,                    advanced one pass,                  136314
+github,                             explicit params,                    advanced one pass,                  137670
+github,                             uncompressed literals,              advanced one pass,                  167004
+github,                             uncompressed literals optimal,      advanced one pass,                  156824
+github,                             huffman literals,                   advanced one pass,                  143457
+github,                             multithreaded with advanced params, advanced one pass,                  167004
+silesia,                            level -5,                           advanced one pass small out,        7152294
+silesia,                            level -3,                           advanced one pass small out,        6789969
+silesia,                            level -1,                           advanced one pass small out,        6191548
+silesia,                            level 0,                            advanced one pass small out,        4862377
+silesia,                            level 1,                            advanced one pass small out,        5318036
+silesia,                            level 3,                            advanced one pass small out,        4862377
+silesia,                            level 4,                            advanced one pass small out,        4800629
+silesia,                            level 5,                            advanced one pass small out,        4710178
+silesia,                            level 6,                            advanced one pass small out,        4659996
+silesia,                            level 7,                            advanced one pass small out,        4596234
+silesia,                            level 9,                            advanced one pass small out,        4543862
+silesia,                            level 13,                           advanced one pass small out,        4482073
+silesia,                            level 16,                           advanced one pass small out,        4377391
+silesia,                            level 19,                           advanced one pass small out,        4293262
+silesia,                            no source size,                     advanced one pass small out,        4862377
+silesia,                            long distance mode,                 advanced one pass small out,        4853389
+silesia,                            multithreaded,                      advanced one pass small out,        4862377
+silesia,                            multithreaded long distance mode,   advanced one pass small out,        4853389
+silesia,                            small window log,                   advanced one pass small out,        7126386
+silesia,                            small hash log,                     advanced one pass small out,        6554898
+silesia,                            small chain log,                    advanced one pass small out,        4931093
+silesia,                            explicit params,                    advanced one pass small out,        4815369
+silesia,                            uncompressed literals,              advanced one pass small out,        5155424
+silesia,                            uncompressed literals optimal,      advanced one pass small out,        4325427
+silesia,                            huffman literals,                   advanced one pass small out,        5341356
+silesia,                            multithreaded with advanced params, advanced one pass small out,        5155424
+silesia.tar,                        level -5,                           advanced one pass small out,        7160438
+silesia.tar,                        level -3,                           advanced one pass small out,        6789024
+silesia.tar,                        level -1,                           advanced one pass small out,        6195462
+silesia.tar,                        level 0,                            advanced one pass small out,        4875008
+silesia.tar,                        level 1,                            advanced one pass small out,        5339697
+silesia.tar,                        level 3,                            advanced one pass small out,        4875008
+silesia.tar,                        level 4,                            advanced one pass small out,        4813507
+silesia.tar,                        level 5,                            advanced one pass small out,        4722235
+silesia.tar,                        level 6,                            advanced one pass small out,        4672194
+silesia.tar,                        level 7,                            advanced one pass small out,        4606658
+silesia.tar,                        level 9,                            advanced one pass small out,        4554098
+silesia.tar,                        level 13,                           advanced one pass small out,        4491702
+silesia.tar,                        level 16,                           advanced one pass small out,        4381277
+silesia.tar,                        level 19,                           advanced one pass small out,        4281514
+silesia.tar,                        no source size,                     advanced one pass small out,        4875008
+silesia.tar,                        long distance mode,                 advanced one pass small out,        4861218
+silesia.tar,                        multithreaded,                      advanced one pass small out,        4874631
+silesia.tar,                        multithreaded long distance mode,   advanced one pass small out,        4860683
+silesia.tar,                        small window log,                   advanced one pass small out,        7130394
+silesia.tar,                        small hash log,                     advanced one pass small out,        6587833
+silesia.tar,                        small chain log,                    advanced one pass small out,        4943255
+silesia.tar,                        explicit params,                    advanced one pass small out,        4829974
+silesia.tar,                        uncompressed literals,              advanced one pass small out,        5157992
+silesia.tar,                        uncompressed literals optimal,      advanced one pass small out,        4321094
+silesia.tar,                        huffman literals,                   advanced one pass small out,        5358079
+silesia.tar,                        multithreaded with advanced params, advanced one pass small out,        5158545
+github,                             level -5,                           advanced one pass small out,        232744
+github,                             level -5 with dict,                 advanced one pass small out,        46718
+github,                             level -3,                           advanced one pass small out,        220611
+github,                             level -3 with dict,                 advanced one pass small out,        45395
+github,                             level -1,                           advanced one pass small out,        176575
+github,                             level -1 with dict,                 advanced one pass small out,        43170
+github,                             level 0,                            advanced one pass small out,        136397
+github,                             level 0 with dict,                  advanced one pass small out,        41170
+github,                             level 1,                            advanced one pass small out,        143457
+github,                             level 1 with dict,                  advanced one pass small out,        41682
+github,                             level 3,                            advanced one pass small out,        136397
+github,                             level 3 with dict,                  advanced one pass small out,        41170
+github,                             level 4,                            advanced one pass small out,        136144
+github,                             level 4 with dict,                  advanced one pass small out,        41306
+github,                             level 5,                            advanced one pass small out,        135106
+github,                             level 5 with dict,                  advanced one pass small out,        38938
+github,                             level 6,                            advanced one pass small out,        135108
+github,                             level 6 with dict,                  advanced one pass small out,        38632
+github,                             level 7,                            advanced one pass small out,        135108
+github,                             level 7 with dict,                  advanced one pass small out,        38766
+github,                             level 9,                            advanced one pass small out,        135108
+github,                             level 9 with dict,                  advanced one pass small out,        39326
+github,                             level 13,                           advanced one pass small out,        133717
+github,                             level 13 with dict,                 advanced one pass small out,        39716
+github,                             level 16,                           advanced one pass small out,        133717
+github,                             level 16 with dict,                 advanced one pass small out,        37577
+github,                             level 19,                           advanced one pass small out,        133717
+github,                             level 19 with dict,                 advanced one pass small out,        37576
+github,                             no source size,                     advanced one pass small out,        136397
+github,                             long distance mode,                 advanced one pass small out,        136397
+github,                             multithreaded,                      advanced one pass small out,        136397
+github,                             multithreaded long distance mode,   advanced one pass small out,        136397
+github,                             small window log,                   advanced one pass small out,        136397
+github,                             small hash log,                     advanced one pass small out,        135467
+github,                             small chain log,                    advanced one pass small out,        136314
+github,                             explicit params,                    advanced one pass small out,        137670
+github,                             uncompressed literals,              advanced one pass small out,        167004
+github,                             uncompressed literals optimal,      advanced one pass small out,        156824
+github,                             huffman literals,                   advanced one pass small out,        143457
+github,                             multithreaded with advanced params, advanced one pass small out,        167004
+silesia,                            level -5,                           advanced streaming,                 7152294
+silesia,                            level -3,                           advanced streaming,                 6789973
+silesia,                            level -1,                           advanced streaming,                 6191549
+silesia,                            level 0,                            advanced streaming,                 4862377
+silesia,                            level 1,                            advanced streaming,                 5318036
+silesia,                            level 3,                            advanced streaming,                 4862377
+silesia,                            level 4,                            advanced streaming,                 4800629
+silesia,                            level 5,                            advanced streaming,                 4710178
+silesia,                            level 6,                            advanced streaming,                 4659996
+silesia,                            level 7,                            advanced streaming,                 4596234
+silesia,                            level 9,                            advanced streaming,                 4543862
+silesia,                            level 13,                           advanced streaming,                 4482073
+silesia,                            level 16,                           advanced streaming,                 4377391
+silesia,                            level 19,                           advanced streaming,                 4293262
+silesia,                            no source size,                     advanced streaming,                 4862341
+silesia,                            long distance mode,                 advanced streaming,                 4853389
+silesia,                            multithreaded,                      advanced streaming,                 4862377
+silesia,                            multithreaded long distance mode,   advanced streaming,                 4853389
+silesia,                            small window log,                   advanced streaming,                 7126389
+silesia,                            small hash log,                     advanced streaming,                 6554898
+silesia,                            small chain log,                    advanced streaming,                 4931093
+silesia,                            explicit params,                    advanced streaming,                 4815380
+silesia,                            uncompressed literals,              advanced streaming,                 5155424
+silesia,                            uncompressed literals optimal,      advanced streaming,                 4325427
+silesia,                            huffman literals,                   advanced streaming,                 5341357
+silesia,                            multithreaded with advanced params, advanced streaming,                 5155424
+silesia.tar,                        level -5,                           advanced streaming,                 7160440
+silesia.tar,                        level -3,                           advanced streaming,                 6789026
+silesia.tar,                        level -1,                           advanced streaming,                 6195465
+silesia.tar,                        level 0,                            advanced streaming,                 4875010
+silesia.tar,                        level 1,                            advanced streaming,                 5339701
+silesia.tar,                        level 3,                            advanced streaming,                 4875010
+silesia.tar,                        level 4,                            advanced streaming,                 4813507
+silesia.tar,                        level 5,                            advanced streaming,                 4722240
+silesia.tar,                        level 6,                            advanced streaming,                 4672203
+silesia.tar,                        level 7,                            advanced streaming,                 4606658
+silesia.tar,                        level 9,                            advanced streaming,                 4554105
+silesia.tar,                        level 13,                           advanced streaming,                 4491703
+silesia.tar,                        level 16,                           advanced streaming,                 4381277
+silesia.tar,                        level 19,                           advanced streaming,                 4281514
+silesia.tar,                        no source size,                     advanced streaming,                 4875006
+silesia.tar,                        long distance mode,                 advanced streaming,                 4861218
+silesia.tar,                        multithreaded,                      advanced streaming,                 4875132
+silesia.tar,                        multithreaded long distance mode,   advanced streaming,                 4866971
+silesia.tar,                        small window log,                   advanced streaming,                 7130394
+silesia.tar,                        small hash log,                     advanced streaming,                 6587834
+silesia.tar,                        small chain log,                    advanced streaming,                 4943260
+silesia.tar,                        explicit params,                    advanced streaming,                 4830002
+silesia.tar,                        uncompressed literals,              advanced streaming,                 5157995
+silesia.tar,                        uncompressed literals optimal,      advanced streaming,                 4321094
+silesia.tar,                        huffman literals,                   advanced streaming,                 5358083
+silesia.tar,                        multithreaded with advanced params, advanced streaming,                 5158130
+github,                             level -5,                           advanced streaming,                 232744
+github,                             level -5 with dict,                 advanced streaming,                 46718
+github,                             level -3,                           advanced streaming,                 220611
+github,                             level -3 with dict,                 advanced streaming,                 45395
+github,                             level -1,                           advanced streaming,                 176575
+github,                             level -1 with dict,                 advanced streaming,                 43170
+github,                             level 0,                            advanced streaming,                 136397
+github,                             level 0 with dict,                  advanced streaming,                 41170
+github,                             level 1,                            advanced streaming,                 143457
+github,                             level 1 with dict,                  advanced streaming,                 41682
+github,                             level 3,                            advanced streaming,                 136397
+github,                             level 3 with dict,                  advanced streaming,                 41170
+github,                             level 4,                            advanced streaming,                 136144
+github,                             level 4 with dict,                  advanced streaming,                 41306
+github,                             level 5,                            advanced streaming,                 135106
+github,                             level 5 with dict,                  advanced streaming,                 38938
+github,                             level 6,                            advanced streaming,                 135108
+github,                             level 6 with dict,                  advanced streaming,                 38632
+github,                             level 7,                            advanced streaming,                 135108
+github,                             level 7 with dict,                  advanced streaming,                 38766
+github,                             level 9,                            advanced streaming,                 135108
+github,                             level 9 with dict,                  advanced streaming,                 39326
+github,                             level 13,                           advanced streaming,                 133717
+github,                             level 13 with dict,                 advanced streaming,                 39716
+github,                             level 16,                           advanced streaming,                 133717
+github,                             level 16 with dict,                 advanced streaming,                 37577
+github,                             level 19,                           advanced streaming,                 133717
+github,                             level 19 with dict,                 advanced streaming,                 37576
+github,                             no source size,                     advanced streaming,                 136397
+github,                             long distance mode,                 advanced streaming,                 136397
+github,                             multithreaded,                      advanced streaming,                 136397
+github,                             multithreaded long distance mode,   advanced streaming,                 136397
+github,                             small window log,                   advanced streaming,                 136397
+github,                             small hash log,                     advanced streaming,                 135467
+github,                             small chain log,                    advanced streaming,                 136314
+github,                             explicit params,                    advanced streaming,                 137670
+github,                             uncompressed literals,              advanced streaming,                 167004
+github,                             uncompressed literals optimal,      advanced streaming,                 156824
+github,                             huffman literals,                   advanced streaming,                 143457
+github,                             multithreaded with advanced params, advanced streaming,                 167004
+silesia,                            level -5,                           old streaming,                      7152294
+silesia,                            level -3,                           old streaming,                      6789973
+silesia,                            level -1,                           old streaming,                      6191549
+silesia,                            level 0,                            old streaming,                      4862377
+silesia,                            level 1,                            old streaming,                      5318036
+silesia,                            level 3,                            old streaming,                      4862377
+silesia,                            level 4,                            old streaming,                      4800629
+silesia,                            level 5,                            old streaming,                      4710178
+silesia,                            level 6,                            old streaming,                      4659996
+silesia,                            level 7,                            old streaming,                      4596234
+silesia,                            level 9,                            old streaming,                      4543862
+silesia,                            level 13,                           old streaming,                      4482073
+silesia,                            level 16,                           old streaming,                      4377391
+silesia,                            level 19,                           old streaming,                      4293262
+silesia,                            no source size,                     old streaming,                      4862341
+silesia,                            uncompressed literals,              old streaming,                      4862377
+silesia,                            uncompressed literals optimal,      old streaming,                      4293262
+silesia,                            huffman literals,                   old streaming,                      6191549
+silesia.tar,                        level -5,                           old streaming,                      7160440
+silesia.tar,                        level -3,                           old streaming,                      6789026
+silesia.tar,                        level -1,                           old streaming,                      6195465
+silesia.tar,                        level 0,                            old streaming,                      4875010
+silesia.tar,                        level 1,                            old streaming,                      5339701
+silesia.tar,                        level 3,                            old streaming,                      4875010
+silesia.tar,                        level 4,                            old streaming,                      4813507
+silesia.tar,                        level 5,                            old streaming,                      4722240
+silesia.tar,                        level 6,                            old streaming,                      4672203
+silesia.tar,                        level 7,                            old streaming,                      4606658
+silesia.tar,                        level 9,                            old streaming,                      4554105
+silesia.tar,                        level 13,                           old streaming,                      4491703
+silesia.tar,                        level 16,                           old streaming,                      4381277
+silesia.tar,                        level 19,                           old streaming,                      4281514
+silesia.tar,                        no source size,                     old streaming,                      4875006
+silesia.tar,                        uncompressed literals,              old streaming,                      4875010
+silesia.tar,                        uncompressed literals optimal,      old streaming,                      4281514
+silesia.tar,                        huffman literals,                   old streaming,                      6195465
+github,                             level -5,                           old streaming,                      232744
+github,                             level -5 with dict,                 old streaming,                      46718
+github,                             level -3,                           old streaming,                      220611
+github,                             level -3 with dict,                 old streaming,                      45395
+github,                             level -1,                           old streaming,                      176575
+github,                             level -1 with dict,                 old streaming,                      43170
+github,                             level 0,                            old streaming,                      136397
+github,                             level 0 with dict,                  old streaming,                      41170
+github,                             level 1,                            old streaming,                      143457
+github,                             level 1 with dict,                  old streaming,                      41682
+github,                             level 3,                            old streaming,                      136397
+github,                             level 3 with dict,                  old streaming,                      41170
+github,                             level 4,                            old streaming,                      136144
+github,                             level 4 with dict,                  old streaming,                      41306
+github,                             level 5,                            old streaming,                      135106
+github,                             level 5 with dict,                  old streaming,                      38938
+github,                             level 6,                            old streaming,                      135108
+github,                             level 6 with dict,                  old streaming,                      38632
+github,                             level 7,                            old streaming,                      135108
+github,                             level 7 with dict,                  old streaming,                      38766
+github,                             level 9,                            old streaming,                      135108
+github,                             level 9 with dict,                  old streaming,                      39326
+github,                             level 13,                           old streaming,                      133717
+github,                             level 13 with dict,                 old streaming,                      39716
+github,                             level 16,                           old streaming,                      133717
+github,                             level 16 with dict,                 old streaming,                      37577
+github,                             level 19,                           old streaming,                      133717
+github,                             level 19 with dict,                 old streaming,                      37576
+github,                             no source size,                     old streaming,                      141003
+github,                             uncompressed literals,              old streaming,                      136397
+github,                             uncompressed literals optimal,      old streaming,                      133717
+github,                             huffman literals,                   old streaming,                      176575

From 6efce7c9ca912d9eaa367db503a1ed3ad1b8556a Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Tue, 19 Feb 2019 11:07:52 -0800
Subject: [PATCH 069/178] [fuzzer] Add test cases

---
 tests/fuzzer.c | 48 +++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 45 insertions(+), 3 deletions(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 946844e1614..e9da862b2c6 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -124,12 +124,14 @@ static U32 FUZ_highbit32(U32 v32)
 #define CHECK(fn)  { CHECK_V(err, fn); }
 #define CHECKPLUS(var, fn, more)  { CHECK_V(var, fn); more; }
 
-#define CHECK_EQ(lhs, rhs) {                                      \
-    if ((lhs) != (rhs)) {                                         \
-        DISPLAY("Error L%u => %s != %s ", __LINE__, #lhs, #rhs);  \
+#define CHECK_OP(op, lhs, rhs) {                                  \
+    if (!((lhs) op (rhs))) {                                      \
+        DISPLAY("Error L%u => FAILED %s %s %s ", __LINE__, #lhs, #op, #rhs);  \
         goto _output_error;                                       \
     }                                                             \
 }
+#define CHECK_EQ(lhs, rhs) CHECK_OP(==, lhs, rhs)
+#define CHECK_LT(lhs, rhs) CHECK_OP(<, lhs, rhs)
 
 
 /*=============================================
@@ -828,6 +830,46 @@ static int basicUnitTests(U32 seed, double compressibility)
         ZSTDMT_freeCCtx(mtctx);
     }
 
+    DISPLAYLEVEL(3, "test%3i : compress -T2 with/without literals compression : ", testNb++)
+    {   ZSTD_CCtx* cctx = ZSTD_createCCtx();
+        size_t cSize1, cSize2;
+        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
+        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2) );
+        cSize1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
+        CHECK(cSize1);
+        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_literalCompressionMode, ZSTD_lcm_uncompressed) );
+        cSize2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
+        CHECK(cSize2);
+        CHECK_LT(cSize1, cSize2);
+        ZSTD_freeCCtx(cctx);
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
+    DISPLAYLEVEL(3, "test%3i : setting multithreaded parameters : ", testNb++)
+    {   ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
+        int value;
+        /* Check that the overlap log and job size are unset. */
+        CHECK( ZSTD_CCtxParam_getParameter(params, ZSTD_c_overlapLog, &value) );
+        CHECK_EQ(value, 0);
+        CHECK( ZSTD_CCtxParam_getParameter(params, ZSTD_c_jobSize, &value) );
+        CHECK_EQ(value, 0);
+        /* Set and check the overlap log and job size. */
+        CHECK( ZSTD_CCtxParam_setParameter(params, ZSTD_c_overlapLog, 5) );
+        CHECK( ZSTD_CCtxParam_setParameter(params, ZSTD_c_jobSize, 2 MB) );
+        CHECK( ZSTD_CCtxParam_getParameter(params, ZSTD_c_overlapLog, &value) );
+        CHECK_EQ(value, 5);
+        CHECK( ZSTD_CCtxParam_getParameter(params, ZSTD_c_jobSize, &value) );
+        CHECK_EQ(value, 2 MB);
+        /* Set the number of worksers and check the overlap log and job size. */
+        CHECK( ZSTD_CCtxParam_setParameter(params, ZSTD_c_nbWorkers, 2) );
+        CHECK( ZSTD_CCtxParam_getParameter(params, ZSTD_c_overlapLog, &value) );
+        CHECK_EQ(value, 5);
+        CHECK( ZSTD_CCtxParam_getParameter(params, ZSTD_c_jobSize, &value) );
+        CHECK_EQ(value, 2 MB);
+        ZSTD_freeCCtxParams(params);
+
+    }
+    DISPLAYLEVEL(3, "OK \n");
 
     /* Simple API multiframe test */
     DISPLAYLEVEL(3, "test%3i : compress multiple frames : ", testNb++);

From 0c86d2346781c4ec21042087dfe86c78dcce51af Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Tue, 19 Feb 2019 15:49:48 -0800
Subject: [PATCH 070/178] [Windows] Move public headers to include/

---
 lib/dll/example/build_package.bat | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/dll/example/build_package.bat b/lib/dll/example/build_package.bat
index cae0a15cba7..8baabc7b2c6 100644
--- a/lib/dll/example/build_package.bat
+++ b/lib/dll/example/build_package.bat
@@ -6,14 +6,15 @@ COPY programs\datagen.h bin\example\
 COPY programs\util.h bin\example\
 COPY programs\platform.h bin\example\
 COPY lib\common\mem.h bin\example\
-COPY lib\common\zstd_errors.h bin\example\
 COPY lib\common\zstd_internal.h bin\example\
 COPY lib\common\error_private.h bin\example\
 COPY lib\common\xxhash.h bin\example\
-COPY lib\zstd.h bin\include\
 COPY lib\libzstd.a bin\static\libzstd_static.lib
 COPY lib\dll\libzstd.* bin\dll\
 COPY lib\dll\example\Makefile bin\example\
 COPY lib\dll\example\fullbench-dll.* bin\example\
 COPY lib\dll\example\README.md bin\
+COPY lib\zstd.h bin\include\
+COPY lib\common\zstd_errors.h bin\include\
+COPY lib\dictBuilder\zdict.h bin\include\
 COPY programs\zstd.exe bin\zstd.exe

From 9f9630f4550c104a0b16fbddce2944f3005f9e92 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Tue, 19 Feb 2019 16:14:02 -0800
Subject: [PATCH 071/178] [Windows] Don't use a .def file

---
 lib/Makefile        |  3 +-
 lib/dll/libzstd.def | 87 ---------------------------------------------
 2 files changed, 1 insertion(+), 89 deletions(-)
 delete mode 100644 lib/dll/libzstd.def

diff --git a/lib/Makefile b/lib/Makefile
index 6a912a6576d..f9955ba4f34 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -151,8 +151,7 @@ ifneq (,$(filter Windows%,$(OS)))
 LIBZSTD = dll\libzstd.dll
 $(LIBZSTD): $(ZSTD_FILES)
 	@echo compiling dynamic library $(LIBVER)
-	@$(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -shared $^ -o $@
-	dlltool -D $@ -d dll\libzstd.def -l dll\libzstd.lib
+	$(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -Wl,--out-implib,dll\libzstd.lib -shared $^ -o $@
 
 else
 
diff --git a/lib/dll/libzstd.def b/lib/dll/libzstd.def
deleted file mode 100644
index 668c4b1c211..00000000000
--- a/lib/dll/libzstd.def
+++ /dev/null
@@ -1,87 +0,0 @@
-LIBRARY libzstd.dll
-EXPORTS
-    ZDICT_getDictID
-    ZDICT_getErrorName
-    ZDICT_isError
-    ZDICT_trainFromBuffer
-    ZSTD_CStreamInSize
-    ZSTD_CStreamOutSize
-    ZSTD_DStreamInSize
-    ZSTD_DStreamOutSize
-    ZSTD_adjustCParams
-    ZSTD_checkCParams
-    ZSTD_compress
-    ZSTD_compressBegin
-    ZSTD_compressBegin_advanced
-    ZSTD_compressBegin_usingDict
-    ZSTD_compressBlock
-    ZSTD_compressBound
-    ZSTD_compressCCtx
-    ZSTD_compressContinue
-    ZSTD_compressEnd
-    ZSTD_compressStream
-    ZSTD_compress_advanced
-    ZSTD_compress_usingCDict
-    ZSTD_compress_usingDict
-    ZSTD_copyCCtx
-    ZSTD_copyDCtx
-    ZSTD_createCCtx
-    ZSTD_createCCtx_advanced
-    ZSTD_createCDict
-    ZSTD_createCDict_advanced
-    ZSTD_createCStream
-    ZSTD_createCStream_advanced
-    ZSTD_createDCtx
-    ZSTD_createDCtx_advanced
-    ZSTD_createDDict
-    ZSTD_createDStream
-    ZSTD_createDStream_advanced
-    ZSTD_decompress
-    ZSTD_decompressBegin
-    ZSTD_decompressBegin_usingDict
-    ZSTD_decompressBlock
-    ZSTD_decompressContinue
-    ZSTD_decompressDCtx
-    ZSTD_decompressStream
-    ZSTD_decompress_usingDDict
-    ZSTD_decompress_usingDict
-    ZSTD_endStream
-    ZSTD_estimateCCtxSize
-    ZSTD_estimateDCtxSize
-    ZSTD_flushStream
-    ZSTD_freeCCtx
-    ZSTD_freeCDict
-    ZSTD_freeCStream
-    ZSTD_freeDCtx
-    ZSTD_freeDDict
-    ZSTD_freeDStream
-    ZSTD_getBlockSizeMax
-    ZSTD_getCParams
-    ZSTD_getDecompressedSize
-    ZSTD_findDecompressedSize
-    ZSTD_getFrameContentSize
-    ZSTD_getErrorName
-    ZSTD_getFrameParams
-    ZSTD_getParams
-    ZSTD_initCStream
-    ZSTD_initCStream_advanced
-    ZSTD_initCStream_usingCDict
-    ZSTD_initCStream_usingDict
-    ZSTD_initDStream
-    ZSTD_initDStream_usingDDict
-    ZSTD_initDStream_usingDict
-    ZSTD_insertBlock
-    ZSTD_isError
-    ZSTD_isFrame
-    ZSTD_maxCLevel
-    ZSTD_nextInputType
-    ZSTD_nextSrcSizeToDecompress
-    ZSTD_resetCStream
-    ZSTD_resetDStream
-    ZSTD_sizeof_CCtx
-    ZSTD_sizeof_CDict
-    ZSTD_sizeof_CStream
-    ZSTD_sizeof_DCtx
-    ZSTD_sizeof_DDict
-    ZSTD_sizeof_DStream
-    ZSTD_versionNumber

From 7ad7ba3178a38d2ed463e66258eead90dee0f50a Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Tue, 19 Feb 2019 17:41:56 -0800
Subject: [PATCH 072/178] [libzstd] Rename ZSTD_CCtxParam_* to
 ZSTD_CCtxParams_*

---
 lib/compress/zstd_compress.c   | 12 ++++++------
 lib/compress/zstdmt_compress.c | 16 ++++++++--------
 lib/zstd.h                     | 20 ++++++++++----------
 tests/fuzzer.c                 | 18 +++++++++---------
 tests/roundTripCrash.c         |  6 +++---
 tests/zstreamtest.c            |  2 +-
 6 files changed, 37 insertions(+), 37 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 1429b84f97b..3d4091455a7 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -494,13 +494,13 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
 
     default: RETURN_ERROR(parameter_unsupported);
     }
-    return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
+    return ZSTD_CCtxParams_setParameter(&cctx->requestedParams, param, value);
 }
 
-size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams,
-                                   ZSTD_cParameter param, int value)
+size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
+                                    ZSTD_cParameter param, int value)
 {
-    DEBUGLOG(4, "ZSTD_CCtxParam_setParameter (%i, %i)", (int)param, value);
+    DEBUGLOG(4, "ZSTD_CCtxParams_setParameter (%i, %i)", (int)param, value);
     switch(param)
     {
     case ZSTD_c_format :
@@ -670,10 +670,10 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams,
 
 size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value)
 {
-    return ZSTD_CCtxParam_getParameter(&cctx->requestedParams, param, value);
+    return ZSTD_CCtxParams_getParameter(&cctx->requestedParams, param, value);
 }
 
-size_t ZSTD_CCtxParam_getParameter(
+size_t ZSTD_CCtxParams_getParameter(
         ZSTD_CCtx_params* CCtxParams, ZSTD_cParameter param, int* value)
 {
     switch(param)
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 419352e7031..dab3a9508ec 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -672,7 +672,7 @@ static void ZSTDMT_compressionJob(void* jobDescription)
         if (ZSTD_isError(initError)) JOB_ERROR(initError);
     } else {  /* srcStart points at reloaded section */
         U64 const pledgedSrcSize = job->firstJob ? job->fullFrameSize : job->src.size;
-        {   size_t const forceWindowError = ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_c_forceMaxWindow, !job->firstJob);
+        {   size_t const forceWindowError = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_forceMaxWindow, !job->firstJob);
             if (ZSTD_isError(forceWindowError)) JOB_ERROR(forceWindowError);
         }
         {   size_t const initError = ZSTD_compressBegin_advanced_internal(cctx,
@@ -864,7 +864,7 @@ static size_t ZSTDMT_expandJobsTable (ZSTDMT_CCtx* mtctx, U32 nbWorkers) {
  * Internal use only */
 size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorkers)
 {
-    return ZSTD_CCtxParam_setParameter(params, ZSTD_c_nbWorkers, (int)nbWorkers);
+    return ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, (int)nbWorkers);
 }
 
 ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, ZSTD_customMem cMem)
@@ -982,13 +982,13 @@ ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params,
     {
     case ZSTDMT_p_jobSize :
         DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter : set jobSize to %i", value);
-        return ZSTD_CCtxParam_setParameter(params, ZSTD_c_jobSize, value);
+        return ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, value);
     case ZSTDMT_p_overlapLog :
         DEBUGLOG(4, "ZSTDMT_p_overlapLog : %i", value);
-        return ZSTD_CCtxParam_setParameter(params, ZSTD_c_overlapLog, value);
+        return ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, value);
     case ZSTDMT_p_rsyncable :
         DEBUGLOG(4, "ZSTD_p_rsyncable : %i", value);
-        return ZSTD_CCtxParam_setParameter(params, ZSTD_c_rsyncable, value);
+        return ZSTD_CCtxParams_setParameter(params, ZSTD_c_rsyncable, value);
     default :
         return ERROR(parameter_unsupported);
     }
@@ -1004,11 +1004,11 @@ size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter,
 {
     switch (parameter) {
     case ZSTDMT_p_jobSize:
-        return ZSTD_CCtxParam_getParameter(&mtctx->params, ZSTD_c_jobSize, value);
+        return ZSTD_CCtxParams_getParameter(&mtctx->params, ZSTD_c_jobSize, value);
     case ZSTDMT_p_overlapLog:
-        return ZSTD_CCtxParam_getParameter(&mtctx->params, ZSTD_c_overlapLog, value);
+        return ZSTD_CCtxParams_getParameter(&mtctx->params, ZSTD_c_overlapLog, value);
     case ZSTDMT_p_rsyncable:
-        return ZSTD_CCtxParam_getParameter(&mtctx->params, ZSTD_c_rsyncable, value);
+        return ZSTD_CCtxParams_getParameter(&mtctx->params, ZSTD_c_rsyncable, value);
     default:
         return ERROR(parameter_unsupported);
     }
diff --git a/lib/zstd.h b/lib/zstd.h
index 03ba1e2dde5..98020383f1b 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -1121,7 +1121,7 @@ ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
  *  It will also consider src size to be arbitrarily "large", which is worst case.
  *  If srcSize is known to always be small, ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation.
  *  ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
- *  ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
+ *  ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
  *  Note : CCtx size estimation is only correct for single-threaded compression. */
 ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
 ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
@@ -1133,7 +1133,7 @@ ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
  *  It will also consider src size to be arbitrarily "large", which is worst case.
  *  If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation.
  *  ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
- *  ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
+ *  ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
  *  Note : CStream size estimation is only correct for single-threaded compression.
  *  ZSTD_DStream memory budget depends on window Size.
  *  This information can be passed manually, using ZSTD_estimateDStreamSize,
@@ -1346,10 +1346,10 @@ ZSTDLIB_API size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param
 /*! ZSTD_CCtx_params :
  *  Quick howto :
  *  - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure
- *  - ZSTD_CCtxParam_setParameter() : Push parameters one by one into
- *                                    an existing ZSTD_CCtx_params structure.
- *                                    This is similar to
- *                                    ZSTD_CCtx_setParameter().
+ *  - ZSTD_CCtxParams_setParameter() : Push parameters one by one into
+ *                                     an existing ZSTD_CCtx_params structure.
+ *                                     This is similar to
+ *                                     ZSTD_CCtx_setParameter().
  *  - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to
  *                                    an existing CCtx.
  *                                    These parameters will be applied to
@@ -1380,20 +1380,20 @@ ZSTDLIB_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compre
  */
 ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);
 
-/*! ZSTD_CCtxParam_setParameter() :
+/*! ZSTD_CCtxParams_setParameter() :
  *  Similar to ZSTD_CCtx_setParameter.
  *  Set one compression parameter, selected by enum ZSTD_cParameter.
  *  Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams().
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
  */
-ZSTDLIB_API size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
+ZSTDLIB_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
 
-/*! ZSTD_CCtxParam_getParameter() :
+/*! ZSTD_CCtxParams_getParameter() :
  * Similar to ZSTD_CCtx_getParameter.
  * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
  */
-ZSTDLIB_API size_t ZSTD_CCtxParam_getParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
+ZSTDLIB_API size_t ZSTD_CCtxParams_getParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
 
 /*! ZSTD_CCtx_setParametersUsingCCtxParams() :
  *  Apply a set of ZSTD_CCtx_params to the compression context.
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index e9da862b2c6..9aed11e3835 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -849,22 +849,22 @@ static int basicUnitTests(U32 seed, double compressibility)
     {   ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
         int value;
         /* Check that the overlap log and job size are unset. */
-        CHECK( ZSTD_CCtxParam_getParameter(params, ZSTD_c_overlapLog, &value) );
+        CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
         CHECK_EQ(value, 0);
-        CHECK( ZSTD_CCtxParam_getParameter(params, ZSTD_c_jobSize, &value) );
+        CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
         CHECK_EQ(value, 0);
         /* Set and check the overlap log and job size. */
-        CHECK( ZSTD_CCtxParam_setParameter(params, ZSTD_c_overlapLog, 5) );
-        CHECK( ZSTD_CCtxParam_setParameter(params, ZSTD_c_jobSize, 2 MB) );
-        CHECK( ZSTD_CCtxParam_getParameter(params, ZSTD_c_overlapLog, &value) );
+        CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, 5) );
+        CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, 2 MB) );
+        CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
         CHECK_EQ(value, 5);
-        CHECK( ZSTD_CCtxParam_getParameter(params, ZSTD_c_jobSize, &value) );
+        CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
         CHECK_EQ(value, 2 MB);
         /* Set the number of worksers and check the overlap log and job size. */
-        CHECK( ZSTD_CCtxParam_setParameter(params, ZSTD_c_nbWorkers, 2) );
-        CHECK( ZSTD_CCtxParam_getParameter(params, ZSTD_c_overlapLog, &value) );
+        CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) );
+        CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
         CHECK_EQ(value, 5);
-        CHECK( ZSTD_CCtxParam_getParameter(params, ZSTD_c_jobSize, &value) );
+        CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
         CHECK_EQ(value, 2 MB);
         ZSTD_freeCCtxParams(params);
 
diff --git a/tests/roundTripCrash.c b/tests/roundTripCrash.c
index 3f4ace8c980..3de5933185d 100644
--- a/tests/roundTripCrash.c
+++ b/tests/roundTripCrash.c
@@ -93,9 +93,9 @@ static size_t cctxParamRoundTripTest(void* resultBuff, size_t resultBuffCapacity
     int const cLevel = h32 % maxClevel;
 
     /* Set parameters */
-    CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_c_compressionLevel, cLevel) );
-    CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_c_nbWorkers, 2) );
-    CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_c_overlapLog, 5) );
+    CHECK_Z( ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_compressionLevel, cLevel) );
+    CHECK_Z( ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_nbWorkers, 2) );
+    CHECK_Z( ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_overlapLog, 5) );
 
 
     /* Apply parameters */
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index ac300890415..6d3cbffb111 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1703,7 +1703,7 @@ static size_t setCCtxParameter(ZSTD_CCtx* zc, ZSTD_CCtx_params* cctxParams,
                                int useOpaqueAPI)
 {
     if (useOpaqueAPI) {
-        return ZSTD_CCtxParam_setParameter(cctxParams, param, value);
+        return ZSTD_CCtxParams_setParameter(cctxParams, param, value);
     } else {
         return ZSTD_CCtx_setParameter(zc, param, value);
     }

From 2e8c19f6a3eeeb701055ad99d1460a9b116c93be Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Wed, 20 Feb 2019 10:46:14 -0800
Subject: [PATCH 073/178] Update changelog

---
 CHANGELOG | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/CHANGELOG b/CHANGELOG
index 2a394227833..0c09b4ad363 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,7 @@
+dev
+api : Rename ZSTD_CCtxParam_getParameter to ZSTD_CCtxParams_getParameter
+api : Rename ZSTD_CCtxParam_setParameter to ZSTD_CCtxParams_setParameter
+
 v1.3.8
 perf: better decompression speed on large files (+7%) and cold dictionaries (+15%)
 perf: slightly better compression ratio at high compression modes

From 4b0024a97d32f60ff72be8004d73c8a8b96e23eb Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Wed, 27 Feb 2019 17:15:27 -0800
Subject: [PATCH 074/178] [fuzz] Add --enable-fuzzer for clang fuzzing

---
 tests/fuzz/fuzz.py | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/tests/fuzz/fuzz.py b/tests/fuzz/fuzz.py
index 8ce293a3a69..69376298518 100755
--- a/tests/fuzz/fuzz.py
+++ b/tests/fuzz/fuzz.py
@@ -192,11 +192,21 @@ def build_parser(args):
         default=LIB_FUZZING_ENGINE,
         help=('The fuzzing engine to use e.g. /path/to/libFuzzer.a '
               "(default: $LIB_FUZZING_ENGINE='{})".format(LIB_FUZZING_ENGINE)))
-    parser.add_argument(
+
+    fuzz_group = parser.add_mutually_exclusive_group()
+    fuzz_group.add_argument(
         '--enable-coverage',
         dest='coverage',
         action='store_true',
         help='Enable coverage instrumentation (-fsanitize-coverage)')
+    fuzz_group.add_argument(
+        '--enable-fuzzer',
+        dest='fuzzer',
+        action='store_true',
+        help=('Enable clang fuzzer (-fsanitize=fuzzer). When enabled '
+              'LIB_FUZZING_ENGINE is ignored')
+    )
+
     parser.add_argument(
         '--enable-asan', dest='asan', action='store_true', help='Enable UBSAN')
     parser.add_argument(
@@ -364,13 +374,17 @@ def build(args):
         '-DFUZZ_RNG_SEED_SIZE={}'.format(args.fuzz_rng_seed_size),
     ]
 
-    mflags += ['LIB_FUZZING_ENGINE={}'.format(args.lib_fuzzing_engine)]
-
     # Set flags for options
+    assert not (args.fuzzer and args.coverage)
     if args.coverage:
         common_flags += [
             '-fsanitize-coverage=trace-pc-guard,indirect-calls,trace-cmp'
         ]
+    if args.fuzzer:
+        common_flags += ['-fsanitize=fuzzer']
+        args.lib_fuzzing_engine = ''
+
+    mflags += ['LIB_FUZZING_ENGINE={}'.format(args.lib_fuzzing_engine)]
 
     if args.sanitize_recover:
         recover_flags = ['-fsanitize-recover=all']

From 820af1e07855019c95299e27666321fc8b096ebd Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Thu, 28 Feb 2019 00:42:49 -0800
Subject: [PATCH 075/178] Provide an API function to estimate decompressed
 size.

Introduces a new utility function `ZSTD_findFrameCompressedSize_internal` which
is equivalent to `ZSTD_findFrameCompressSize`, but accepts an additional output
parameter `bound` that computes an upper-bound for the compressed data in the frame.

The new API function is named `ZSTD_decompressBound` to be consistent with
`zstd_compressBound` (the inverse operation). Clients will now be able to compute an upper-bound for
their compressed payloads instead of guessing a large size.

Implements https://github.com/facebook/zstd/issues/1536.
---
 doc/zstd_manual.html             |  9 ++--
 lib/decompress/zstd_decompress.c | 77 +++++++++++++++++++++++++++++---
 lib/zstd.h                       |  9 ++--
 tests/fuzzer.c                   | 19 ++++++++
 tests/symbols.c                  |  1 +
 5 files changed, 101 insertions(+), 14 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index c7962e7de0c..a11a9589229 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -127,10 +127,11 @@ 
zstd 1.3.8 Manual

 


 
 
Helper functions
#define ZSTD_COMPRESSBOUND(srcSize)   ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
-size_t      ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
-unsigned    ZSTD_isError(size_t code);          /*!< tells if a `size_t` function result is an error code */
-const char* ZSTD_getErrorName(size_t code);     /*!< provides readable string from an error code */
-int         ZSTD_maxCLevel(void);               /*!< maximum compression level available */
+size_t      ZSTD_compressBound(size_t srcSize);                    /*!< maximum compressed size in worst case single-pass scenario */
+size_t      ZSTD_decompressBound(const void* src, size_t srcSice); /*!< maximum decompressed size of the compressed source */
+unsigned    ZSTD_isError(size_t code);                             /*!< tells if a `size_t` function result is an error code */
+const char* ZSTD_getErrorName(size_t code);                        /*!< provides readable string from an error code */
+int         ZSTD_maxCLevel(void);                                  /*!< maximum compression level available */
 


 
Explicit context

 
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 601bfe70434..a29122a837c 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -434,12 +434,7 @@ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t he
 }
 
 
-/** ZSTD_findFrameCompressedSize() :
- *  compatible with legacy mode
- *  `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame
- *  `srcSize` must be at least as large as the frame contained
- *  @return : the compressed size of the frame starting at `src` */
-size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
+static size_t ZSTD_findFrameCompressedSize_internal(const void *src, size_t srcSize, size_t* bound)
 {
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
     if (ZSTD_isLegacy(src, srcSize))
@@ -464,6 +459,7 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
         remainingSize -= zfh.headerSize;
 
         /* Loop on each block */
+        unsigned nbBlocks = 0;
         while (1) {
             blockProperties_t blockProperties;
             size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
@@ -474,6 +470,7 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
 
             ip += ZSTD_blockHeaderSize + cBlockSize;
             remainingSize -= ZSTD_blockHeaderSize + cBlockSize;
+            nbBlocks++;
 
             if (blockProperties.lastBlock) break;
         }
@@ -483,10 +480,78 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
             ip += 4;
         }
 
+        if (bound != NULL) *bound = (nbBlocks * zfh.blockSizeMax); /* set to block-based bound */
+
         return ip - ipstart;
     }
 }
 
+/** ZSTD_findFrameCompressedSize() :
+ *  compatible with legacy mode
+ *  `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame
+ *  `srcSize` must be at least as large as the frame contained
+ *  @return : the compressed size of the frame starting at `src` */
+size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
+{
+    return ZSTD_findFrameCompressedSize_internal(src, srcSize, NULL);
+}
+
+
+/** ZSTD_decompressBound() :
+ *  currently incompatible with legacy mode
+ *  `src` must point to the start of a ZSTD frame or a skippeable frame
+ *  `srcSize` must be at least as large as the frame contained
+ *  @return : maximum decompressed size of the compressed source
+ */
+size_t ZSTD_decompressBound(const void* src, size_t srcSize)
+{
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
+    if (ZSTD_isLegacy(src, srcSize))
+        return ERROR(version_unsupported);
+#endif
+
+    size_t totalDstSize = 0;
+
+    /* Loop over each frame */
+    while (srcSize >= ZSTD_FRAMEHEADERSIZE_PREFIX) {
+        U32 const magicNumber = MEM_readLE32(src);
+
+        if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
+            size_t const skippableSize = readSkippableFrameSize(src, srcSize);
+            if (ZSTD_isError(skippableSize))
+                return skippableSize;
+            if (srcSize < skippableSize) {
+                return ZSTD_CONTENTSIZE_ERROR;
+            }
+
+            src = (const BYTE *)src + skippableSize;
+            srcSize -= skippableSize;
+            continue;
+        }
+
+        {   unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
+            if (ret == ZSTD_CONTENTSIZE_ERROR) return ret;
+
+            size_t bound;
+            size_t const frameSrcSize = ZSTD_findFrameCompressedSize_internal(src, srcSize, &bound);
+            if (ZSTD_isError(frameSrcSize)) {
+                return ZSTD_CONTENTSIZE_ERROR;
+            }
+
+            size_t frameBound = (ret == ZSTD_CONTENTSIZE_UNKNOWN) ? bound : ret;
+            /* check for overflow */
+            if (totalDstSize + frameBound < totalDstSize) return ZSTD_CONTENTSIZE_ERROR;
+            totalDstSize += frameBound;
+
+            src = (const BYTE *)src + frameSrcSize;
+            srcSize -= frameSrcSize;
+        }
+    }  /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */
+
+    if (srcSize) return ZSTD_CONTENTSIZE_ERROR;
+
+    return totalDstSize;
+}
 
 
 /*-*************************************************************
diff --git a/lib/zstd.h b/lib/zstd.h
index 98020383f1b..31341101773 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -148,10 +148,11 @@ ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t
 
 /*======  Helper functions  ======*/
 #define ZSTD_COMPRESSBOUND(srcSize)   ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
-ZSTDLIB_API size_t      ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
-ZSTDLIB_API unsigned    ZSTD_isError(size_t code);          /*!< tells if a `size_t` function result is an error code */
-ZSTDLIB_API const char* ZSTD_getErrorName(size_t code);     /*!< provides readable string from an error code */
-ZSTDLIB_API int         ZSTD_maxCLevel(void);               /*!< maximum compression level available */
+ZSTDLIB_API size_t      ZSTD_compressBound(size_t srcSize);                    /*!< maximum compressed size in worst case single-pass scenario */
+ZSTDLIB_API size_t      ZSTD_decompressBound(const void* src, size_t srcSice); /*!< maximum decompressed size of the compressed source */
+ZSTDLIB_API unsigned    ZSTD_isError(size_t code);                             /*!< tells if a `size_t` function result is an error code */
+ZSTDLIB_API const char* ZSTD_getErrorName(size_t code);                        /*!< provides readable string from an error code */
+ZSTDLIB_API int         ZSTD_maxCLevel(void);                                  /*!< maximum compression level available */
 
 
 /***************************************
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 9aed11e3835..00e7f5442a8 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -376,6 +376,20 @@ static int basicUnitTests(U32 seed, double compressibility)
     }
     DISPLAYLEVEL(3, "OK \n");
 
+    DISPLAYLEVEL(3, "test%3i : tight ZSTD_decompressBound test : ", testNb++);
+    {
+        size_t bound = ZSTD_decompressBound(compressedBuffer, cSize);
+        if (bound != CNBuffSize) goto _output_error;
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
+    DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test missing Frame_Content_Size : ", testNb++);
+    {
+        size_t bound = ZSTD_decompressBound(compressedBuffer, cSize);
+        if (bound != CNBuffSize) goto _output_error;
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
     DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize);
     { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
       if (r != CNBuffSize) goto _output_error; }
@@ -901,6 +915,11 @@ static int basicUnitTests(U32 seed, double compressibility)
         if (r != CNBuffSize / 2) goto _output_error; }
     DISPLAYLEVEL(3, "OK \n");
 
+    DISPLAYLEVEL(3, "test%3i : get tight decompressed bound of multiple frames : ", testNb++);
+    {   unsigned long long const r = ZSTD_decompressBound(compressedBuffer, cSize);
+        if (r != CNBuffSize / 2) goto _output_error; }
+    DISPLAYLEVEL(3, "OK \n");
+
     DISPLAYLEVEL(3, "test%3i : decompress multiple frames : ", testNb++);
     {   CHECK_V(r, ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize));
         if (r != CNBuffSize / 2) goto _output_error; }
diff --git a/tests/symbols.c b/tests/symbols.c
index 600d81670a5..4d9c6fc0c9e 100644
--- a/tests/symbols.c
+++ b/tests/symbols.c
@@ -31,6 +31,7 @@ static const void *symbols[] = {
   &ZSTD_getFrameContentSize,
   &ZSTD_maxCLevel,
   &ZSTD_compressBound,
+  &ZSTD_decompressBound,
   &ZSTD_isError,
   &ZSTD_getErrorName,
   &ZSTD_createCCtx,

From 97d3d28dabeccd73d2179ff0b328a298404534f7 Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Thu, 28 Feb 2019 01:24:54 -0800
Subject: [PATCH 076/178] Fix decl-after-stmnt build error

---
 lib/decompress/zstd_decompress.c | 12 ++++++------
 tests/truncatable-input.txt      |  1 +
 2 files changed, 7 insertions(+), 6 deletions(-)
 create mode 100644 tests/truncatable-input.txt

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index a29122a837c..b9362eb6129 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -448,6 +448,7 @@ static size_t ZSTD_findFrameCompressedSize_internal(const void *src, size_t srcS
         const BYTE* const ipstart = ip;
         size_t remainingSize = srcSize;
         ZSTD_frameHeader zfh;
+        unsigned nbBlocks = 0;
 
         /* Extract Frame Header */
         {   size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize);
@@ -459,7 +460,6 @@ static size_t ZSTD_findFrameCompressedSize_internal(const void *src, size_t srcS
         remainingSize -= zfh.headerSize;
 
         /* Loop on each block */
-        unsigned nbBlocks = 0;
         while (1) {
             blockProperties_t blockProperties;
             size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
@@ -505,13 +505,12 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
  */
 size_t ZSTD_decompressBound(const void* src, size_t srcSize)
 {
+    size_t totalDstSize = 0;
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
     if (ZSTD_isLegacy(src, srcSize))
         return ERROR(version_unsupported);
 #endif
 
-    size_t totalDstSize = 0;
-
     /* Loop over each frame */
     while (srcSize >= ZSTD_FRAMEHEADERSIZE_PREFIX) {
         U32 const magicNumber = MEM_readLE32(src);
@@ -529,16 +528,17 @@ size_t ZSTD_decompressBound(const void* src, size_t srcSize)
             continue;
         }
 
-        {   unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
+        {   size_t bound;
+            size_t frameBound;
+            unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
             if (ret == ZSTD_CONTENTSIZE_ERROR) return ret;
 
-            size_t bound;
             size_t const frameSrcSize = ZSTD_findFrameCompressedSize_internal(src, srcSize, &bound);
             if (ZSTD_isError(frameSrcSize)) {
                 return ZSTD_CONTENTSIZE_ERROR;
             }
 
-            size_t frameBound = (ret == ZSTD_CONTENTSIZE_UNKNOWN) ? bound : ret;
+            frameBound = (ret == ZSTD_CONTENTSIZE_UNKNOWN) ? bound : ret;
             /* check for overflow */
             if (totalDstSize + frameBound < totalDstSize) return ZSTD_CONTENTSIZE_ERROR;
             totalDstSize += frameBound;
diff --git a/tests/truncatable-input.txt b/tests/truncatable-input.txt
new file mode 100644
index 00000000000..02b78f51892
--- /dev/null
+++ b/tests/truncatable-input.txt
@@ -0,0 +1 @@
+;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C1025E554001?:4@<1D587AA6708;9>8D01<0M?1551A5402I03>11430715FEC1825A>?264=3011M<=C=3?453;7911?700=39<4HJk1098>A054544C31V4H98061EN805851?4:=0=61254402377841;:300435;438901:>361803D:6640D<:I1=@00462;629>;H700;AKD1720143K9B43100F0972121:2?2Q285C0018634S:<31J07721750E1822775E24=31195Q04;5;2QY21?30?450;991;1>412646:500N<3>;D0=40793;53B25>504O912H363696:B3677J:0092;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C1025>3<586>A4=31195Q04;5;2QY;32FF;32FF;32FF;3237034859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C1025E554001?:4@<1D587AA6708;9>8D01<0M?1551A5402I03>11430715FEC1825A>?264=3011M<=C=3?453;7911?700==A;32FF;32FF;32FF;32G>8<;32FF;32FF;32FF;3;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C102515;32FF;32FF;32FF;790<1:6258121>0@=15@==7020:69210<>1;32FF;32FF;32;32FF;32;32FF;32FF;32FF;32F6;32FF;32FF;32FF;32F;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32;32FF;32FF;32F5065:118300::F64401<0321?25K56915?46;:N0O119@<64716;3A4=<90130=417@2MB7G15495610>F67M=2?:3772533:<0@43;32FF;32FF;32FF;32FF;32FF;32608?35>0673;:69210<>15;32FF;3<0>3121?;32FF;32F;32FF;32F;32FF;32<5?2;32FF;32FF;3>C5124662C41?1A64;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C1025E554001?:4@<1D587AA6708;9>8D01<0M?1551A5402I03>11430715FEC1825A>?264=3011M<=C=3?453;7911?700=39<4HJk1098>A081:004518836;760;538:31L14365324B37C3H8P004002J81C2;22865G13415X1:;@B0029625MP0H0421=322:2=5?@:928610<324110G4@AD3:941>544C31V4H98061EN805851?4:=0=61254402377841;:300435;438901:>361803D:6640D<:I1=@00462;629>;H700;AKD1720143K9B43100F0972121:2?2Q285C0018634S:<31J07721750E1822775E24=31195Q04;5;2QY21?30?450;991;1>412646:500N<3>;D0=40793;53B25>504O912H363696:B3677J:0092;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371;32FF;32FF;32FF;32FF;;=9:;32FF;32@:<760125B101K032300331J5F;6241;1D85:00?41040005?6F17305F0;5;B604028=:0073C26@21T06274OJ2J9@29?=039:503010425@719127667=402FF;0:7;?105=;32FF;32FF;32FF;32FF;31859389;:6;32FF;32FF;32FF;32F;32FF;32FF;3NF5H;32FF;32FF;32FF;32F;32FF;32FF;32FF;32F;32FF;32FF;32FF;32F0431253G500130H00286@1H<;1B6:180=04A1010683706062:005536C436?G650A<33:BO4;0117?=375<485=2V5673A2522A225K97?512:1@P76616P842340@03177K01V2<30B1?13>E5540I6;32FF;32;32FF;32F7?@600;283402<;32FF;32FF;32FF;;32FF;32FF;3;32FF;32FF;32FF;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C1025F7=;32FF;32F073973053;32FF;32FF;32FF;3=6125440237;32FF;32F3063224:61=2645:?26;@<43>2<6:G7:B7C40211126:33026O:220UZdC:0]a7=0777G1B7?1=4F9>47;@;002738E06092437065<=0649D68\N16380921DE70H2413034::1<21A3IH3RG9=4;32F;32FF;::14:60;396844:93415@707:@<;=03>1>4843432B1H082:010C>4E63F1@719127667=48;?F15NC2=:19@105;32FF;32FF;32FF;3;32FF;32FF;32FF;32FF;;32FF;Q04;5;2QY;32FF;32FF32F;32E3012611;32;32FF;32FF;32;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C102508;32FF;32FF;3@6=;32FF;32F;32FF;32FF;32FF;32FF;32FF;032H1C197313;32FF;303:64<468AI0;32FF;32FF;32FF25?6Q=9=8?36A6U;32FF;32F;32FF;32FF;;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C1025A6;32FF;32FF2487;3=25299=3<68F52:D;32D10080?4472E60:19802<30:B03:19=@755441705=@73:63=<05^70<24FX;008330;251641:B>3OF12A28<:231<21:23H4=4215<0171825>201803203@0800<17939?0<96424;;2D320665550362:1;72:3Hd372=99;0I967516J21320C0=4<045238331:7N531<5:;C320E13303>>:1:0;101579:D334243409>O62:H91@5:14N56@04<67504=42Q3R3F0_1107657505<540@030@272209;53C2210]60@40G:1G118@4B37?299h4652B;53E@2K09:717M031990:8<1999@;0=17:;32FF;32FF;32FF;32FF;31859389;:692;32F;32FF;32FF;8@@?2876:3H2A8N<0I88782A2:B0431253G5001<53106106H1261@6G9629B4048E458025V17I1:0AE1?7<7a:41<1;0J3208:7<50800394@6181;168201<14F3299C6?495926>2=6>E?2BP435130334:6866M382944Q171>7D<9230?1:832FF<@18AABOS120620<;;I8B11213965365I>K^R0M5I7304619=21670764656B6I6I6099=076@?174@@02;4::41B23017B4261943@81@605P032;R3B46<36:183630409^;F8?346250202E0715228:C4<3@34X90081:V5992>802;02>=L1F3B60;32F;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C102588P2@32:;60707608243;<5:H95883ER088380]36J0075221189F15@1176:4443g:G295L8020406;9NS<51;<8HF1>6A20<:71F?109154=31C411=1BI2F61303A=53189M654=;03519=;8151CQ6<900@457F2:3B5Q2<051H0;G;1B@42004218201238611220710003B40B7J2526U5C38634<24C2K612814<;3672;235103GG118371C:?22A510104@<1D587AA6708;9>8DB47J27083;32FF;32FF;32FF;3;32FF612631460D57;2d;32FF;32FF;32FF;003L27]55;9A3L6:90528;32FF;32FF;32FF;32F;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C10250;32F>5B6156N2@9@1>6;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<111<35553@25207I68072522A261>404;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C1025E554001?:4@<1D587AA6708;914<0;32FF;32F;32FF;32F;32FF;32FF;32FF;32FF;3076@?17415;32FF;32FF;32FF;790<1:62526351;32FF@:6242731;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C1025E554001?:4@<1D587AA67;32FF;32FF;32FF;32FF;32FF;F;32;32F;32FF;32FF;32F17>88P2@32:;60707608243;<5:H95883ER088380]36J0075221189F15@1176:4443g:G295L8020406;9NS<51;<8HF1>6A20<:71F?109154=31C411=1BI2F61303A=53189M654=;03519=;8151CQ6<900@457F2:3B5Q2<051H0;G;1B@42004218201238611220710003B40B7J2526U5C38634<24C2K612814<;3672;235103GG118371C:?22A510104@<1F;3185790<1:6258121>0;32F;32FF;32FF;32FF;;8150=51426;6L068;6>89546;32FF;32CC80?;32FF;32FF;32;32FF;32FF;1:4=<10B3:3O753224260?9:C2193@;32FF;32FF3278AB;32FF;32FF;E:58>2;720441529?>90D<=;129090=0<8@50G0M<649YEA252;3809D700JJ1>173E23C7<=;E1V6D>0717550F110FC7<1<6:7;M96G4E0<321@65290752;196>2T;?1@79>8>70349?F0337T2203;050038D29<623?1C;2U?E932?282C43C54666@?1030<30M031D986:5>01Q073:K04413565256M424>6131=1@13614=H3180:914A946;32B0:F20@D;32FF;32FF75E=?A111<03;:<9<321<6:8113056402H363696:B3677J:0092;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C102515;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C102536183D131:836=602;32FF;32FF;32FF;32FF;7B0D432FF;32FF1?25K56915?46;:N0O119@<64716;3A4=<90130=417@2MB7G15495610>F67M=2?:3772533:<0@43;32FF;32FF;32FF;32FF;32FF;32608?35>0673;:69210<>15;32FF;3<0>3121?;32FF;32F;32FF;32F;32FF;32<5?2;32FF;32FF;3>C5124662C41?1A64;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C10258<;32FF;32FF;32FF;3;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;3215937159;32FF;32FF;32FF;3;32FF;32FF;32FF;3;32FF;32FF;32FF;324:31A81>134167121>0;32FF;32FF;A;32FF;32FF;32FF;32G>8<;32FF;32FF;32FF;3;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258276G4:9:1636E=042103610I@81:H349=56350>303692503623A2Z054151;:?1:30KC6201:=53309@90;32FF;32FF;32FF9415847A16?:1?7914;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C1025E554001?:4@<1D587AA6708;9>8D01<0M?1551A5402I03>11430715FEC1825A>?264=3011M<=C=35M531=547B031504334A05;;=6A00?85;32FF;32FF;32FF;32FF;32FF;32F;32FF;32FF;3210?7321180?033;20671I8155@403322434546<_36069622877<5<3O6@5?C;01^80>23633233G:=3=568;>1?226:G:G2?C5?=40C;:<204?476>803B941;A420@150<364<43L4>6322<6?5EF<094940>44>M<424120164362K40812H:7=231B420824:56:A3D8828C6<010QG6B6H77?32O07:R83FO3?;C;7:E3><=40475211:5006098<@1<7627E:0>235@8072M:4210C3124>:8@T=4T1>5E=57;5@0362>?376?5<3931833a?332FF;32EO113552;Q06@4@59?;62:<520031131<=6876640>D50249;6151;756AF:14:2113253B?0C02O3K11C224;;S00243?2347d8<:10H46T=212<511?624H3C025445220587AA6708;9>8D01<2FF;;32FF;2216J74<0083C02;206G0=2<93;51:0CVE55200257430G>9920C0:7005>1373<;194>=4H952=2=:9?4U:B112>KCI0E0>:10=5204A1517002=3:0M11?H;0A;195H0O1:5G1>05951C453002500326007?7;1;7?6:48:A5@260176084693008=26FG81>B1F9145P037J3950175302807V4682>338<38@651026BGG2445E12G7401;M850=41?26;6L364B8<336?0744166081O3\8I>6682T16;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C1025E554001?:4@<1D587AA6708;9>8D01<0M?1551;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C1025LC?K31<5046091091J070A2J4A03505163;722062121M54154741D0:94184122;Y:00284190;:I5?DE4472711;<9866?4;<8;H;39530A11K92@3?05:6524H391:46O880173O<2?3:4@86193544807179J28511>105:<5B3D3003<=A6A;_5340?@C07552C10;@;79245O870331=3J214;F:62G917760F01522C0EF1>1?75E611H133B63530<;32FF;32FF;32FF;3A<33:BO4;01144620T>>;13125??3264@310G60345?:<2B32203TC4<7018<5<10959EAL396846<25P2C1OBO123L110:73F>?89132855NM07LE9446J3A4722A7734898N38<5844?65N=221810836543K030;1:089<2;322P39:11;=0C;1076539^0<75?24404826F162C9I92<235286J<3144782A5055:30>Y3GB3400<3475217I12=35P019<74:0117?=375<485=2;32FF;31H9000>21;32FF2<1K240;32FF;32FF;;32FF;3>2222;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;321G4D73?U?8167T7;32FF;32AA67;32FF;32FF;3;32FF;32FF;32>915725;583540862422E85@00213929;34=G65A81G?5912450@104320:B4C41909=298B641P6I0;0;91C901>2>4@359417;2;1K0Y;>1`72111576626;LF@@>751H753E100=C0:481604054091063;>817D124130K5572:::31320=9678=1560<241=07MH:0<13A@C;5=1G3D;07=1;1308BA2545727\@11<1G09F0031@30?;4H05760H81130AP2=:270568@3IA2N9:65=6544;654=1G;270?12=14:043B304:700=?723L68270>>64>B;1709934mS561=11:A322:2614377042A;605DE6GD7521<81;:F41:4m?6>3E3<92>1=0;55XB2=9<00134G244Q>300@7770260A008531B4C521E20069A25C4@1=1CAB739:A55B405F29222=:k1098>A054544C31V4H98061EN805851?4:=0=61254402377841;:300435;438901:>361803D:6640D<:I1=@00462;629>;H700;AKD1720143K9B43100F0972121:2?2Q285C0018634S:<31J07721750E1822775E24=31195Q04;5;2QY21?30?450;991;1>412646:500N<3>;D0=40793;53B25>504O912H363696:B3677J:0092;32FF;32FF;32FF;32@8<192@D36<66<9210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C1025026227D046M501;32FF;32FF;32FF0081712M14;32FF;5353;:0;5F?5211274>?0:3?;32FF;32FF;32@7]21:5I1B332A05471:53328413:5;:>475;0@A111j4>7D67357>39818C6=:C41;74360E2355907290211960=093V:4850>708M2;9G317129@1202<73Y14E57156?1103;:@1557B020E9:397412H288?83K:A=21:2T020320;>=61;:75<7>?4B8<846305:2205218929@A456:<4428<630;2AE2202310103<241926311410N707JC26395I0310:=8628903?46C92F0C3?1186@N58=02982I210<233:570152320=C?62859:387586X3=:=;03>9121;C5424H@TL5>>1537O0D4D:145K125??326A3A550;32FF;32FF;32FF;3255B405F29222=:k10985841T419170J37324;32FF;32FF;32F06703112E2K51315510346=L=2==C5N1FE1790034L70402221P>7B=1?1Z9?12gH174E040041C62::3<01843590E5?2955=7D5;B8L05@5K<542DR12102D1?18a79=264E24H5348:040@A130294GG@3:072?1;328V52816652<;8B3;3OI70[1?3BD649311:>?E55F1T0311[68F48B433084B81003036740:0913A81B33L4A;3052834C;?522R>00:6BA708e24B==:42AN30?80F3:R412340210900892E108=4339F10929E38]B90929784;37<@3=50<6:126<=A54281<11U3824611L3206\FF;32FF;32FD:1:;3N>20:4@MR4>h0>7F663:@47503J0=3@5<;68AA39226;5A51321111H=4;:5@0@1;238>6<43C915B543?604:137601I200@02221218;746D45450325A4S@00A641[8;5GF7408D;R11:@f5560292;32FFJD;617400>;1C3J;31859389;:69;32FF;32FF;32F389;:6;32FF;32FF;358G273N414C0;22A8<320971\040:1A21:K2;498672FF;32FF;321593136CD>011>91=1<37;5@0362>?376?5<3931833a?332FF;32EO113552;Q06@4@59?;:5>01Q073:K04413565256M424>6131=1@13614=H31314656254B37C3H8P0040;32FF;AF;32FF;32FF;32F;32FF;32FF;32FF;;<30E3=14656254B37C3H8P0040;32FF;AF;32FF;32FF;32F;32FF;32FF;32FF;;<30E3=14656254B37C3H8P0040;32FF;AF;32FF;32FF;32F;32FF;32FF;32FF;;<30E3=14656254B37C3H8P0040;32FF;AF;32FF;32FF;32F;32FF;32FF;32FF;;<30E3=14656254B37C3H8P0040;32FF;AF;32FF;32FF;32F;32FF;32FF;32FF;;<30E3=14656254B37C3H8P0040;32FF;AF;32FF;32FF;32F;32FF;32FF;32FF;;<30E3=14656254B37C3H8P0040;32FF;AF;32FF;32FF;32F;32FF;32FF;32FF;;<30E3=14656254B37C3H8P0040;32FF;AF;32FF;32FF;32F;32FF;32FF;32FF;;<30E3=14656254B37C3H8P0040;32FF;AF;32FF;32FF;32F;32FF;32FF3G7304HF606<;32FF;:3>:9203=54B83;31859416H6ER@F4;32FF;32FF;32FF;;32FF;32FF;32F;32F;32FF;32FF;32;32FF;32FF;32FF;?T12E81k21=87=22?906@707:@`;32FF;32FF;32FF;32F6>30@4>6@4513<>2FF;;32FF;324000<304A656?:14A22227;052611?01:2137480083B:3>Q560>40;I>53:9757203760J24<;66?0:B91>1C10>;?08>102AJ86@4057190553<@09A36E3971D3E4L1I54MK106352F^3098A336I033;6;=1=10;84101O422F5077743Q0T51617:943=22002?1F427:005010A4:GP>>3851532@@38110394H:8:2Bd12831D?3808>40<3=J=E2>;W0<52=022835024115A02>51514160=050623K62014152J12C1;Q53C831M00;4?0013H00E>41:8:;7C40@144;F62529>2G9B335152D@0223KE1V2444A19M03J5923=;36:166450F719933C3D66H880HG04>5:50?8501KA883=Y007A5E24723556A1I2:1250?5F324H=00082111090810A95:>242<13<6012:40B3H0@:85510027C7D:895>:44=536063265930;390=?713:234;425515F0>574274050<120601@024L373221J7R555A;=86@TD72;0FL05274>102C778;C5434E9=381370353325F121030B3:04A8YL794021>084210961:22>1G280;8>;D;32FF;32FF;3143H020@949I2=2?481544<6>00;25>12648=6@356>4189D66<4><01H=S<04<01:3324I=>3?22T049O00805432<73031305K?2308793;17074@L95652100611?20V540705;8:5E420645R121913:3D101022@M32T13<15H22<437;30>:8:;3KDL18434;=7C6<4A65229732:3;32FF;3807D67357>39818C6=:C41;74360E2355907290211960=093V:4850>708M2;9G317129@1202<73Y14E57156?1103;:@1557B020E9:397412H288?83K:A=21:2T020320;>=61;:75<7>?4B8<846305:2205218929@A456:<4428<630;2AE2202310103<241926311410N707JC26395I0310:=8628903?46C92F0C3?1186@N58=02982I210<233:570152320=C?62859:387586X3=:=;03>9121;C5424H@TL5>>1537O0D4D:145K125??326A3A550;32FF;32FF;32FF;3255B405F29222=:k10985841T419170J37324;32FF;32FF;32F06703112E2K51315510346=L=2==C5N1FE1790034L70402221P>7B=1?1Z9?12gH17;32FF;32FF;32FF;3:3623D37B==K651025:03=70:610167380681F;;=9:;32F;32FF;32FF;32F;32FF;32FF;32FF;32FF;316@E04P06>642350190=725;32FF;3215937079:=811935;32FF;32FF;32FF;32FF;32FF;B1F9140:0?900A::6<26;K44<==16001605:5c2462;1297;0D5241^41=688595>G1B7?1=4F9>47;@;002738E06092437065<=0649D68\N16380921DE70H2413034::1<21A3IH3RG9=4;32F;32FF;::14:60;396844:93415@707:@<;=03>1>4843432B1H082:010C>4E63F1@719127667=48;?F15NC2=:19@105;32FF;32FF;32FF;3;32FF;32FF;32FF;32FF;;32FF;Q04;5;2QY;32FF;32FF32F;32E3012611;32;32FF;32FF;32;32FF;32FF:=10;K2281113::=5>B;8=4603;000A4216063D2192612704A75223C9:A056C>4K;215B;;?0006=B:CL233311P28;524G1086483E0C4BP:82I;;=399108>92072L9>61870B17246:484880D:A98354:308<2F=3L3722G56=2F38@A43=16A6K:AN6015274PP90:2BB06@0001?:9C<1D82=3912@7FJ<37U73C@50;;32FF;32FF;32F082;0:11072317I667O27C16M;7<11A8A51150148107LE9446J3A4722A71B32M:2=5?@:928610<32411;32FF;32FF;3433146G028C42D021<811:02=34400843669?105=;32FF;6<;:>34>1167;32FF;32FJ53;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C102515;32F>4OK8222PB<5000<1:6258121>0;32FF30?4FF;32FF;32;32FF;32FFFF;32FF;32FF;3185910;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C102588P2@32:;60707608243;<5:H95883ER088380]36J0075221189F15@1176:4443g:G295L8020406;9NS<51;<8HF1>6A20<:71F?109154=31C411=1BI2F61303A=53189M654=;03519=;8151CQ6<900@457F2:3B5Q2<051H0;G;1B@42004218201238611220710003B40B7J2526U5C38634<24C2K612814<;3672;235103GG118371C:?22A510104@<1D587AA6708;9>8DB47J27083;32FF;32FF;32FF;3;32FF61263088380]36;322P39:1124303;H1:10=949A?44334L79054=0BH;32FF;32FF;326156N2@9@1102515;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C10259YQ@IK34;50300C2M1@29A2185>14E7861=B8;RH5313EJ10<15G196?59C5O376166;@53080103133?25>018R1A112058466?542151>=01B32OR1142E9G30911413=@208086:3056=]6Y55217214222F143:M711392937540200333:0533>51106=B1=@4>358K15:70>80211<34319155=453A402FB?9>98B5h419D0?3162211=109L0A1R>13F0B11=8547@I11082<40O1:57=930@B0:12171@>38C17M@215B34<07009;643F8P4[515A464O6J13:?A017A6C3@32SB<2L25A61<81:7K>3>:636041[4L8A??;0;32FF;32FF;32FF00EU363;5A308:0F441N38<5844?65N=22181083;32FF;32FF;32FF;35434O3=?25235;R504?H6160955G04?73<0104105M41:1<32611G3011>224024677G>5<=1=:82491>@>425312DG02B0571>26P>51785>93<04330582008?5>192079904<1;A1:3:HH00598?=<;1010<=>0:00050:8465C?30R4B11583G29;@C940:45><72F4802=324=;289801302?443D61256H2603SO35b:6<0121?48031620;D=?65Q6W3:01B3?01523CJI84871146111430417555<44;>C9?31185317=0BB058809>9761H8?704:6:2199;8SC1I23I1156L055G561B3<:9882286913A<0143<;62C34d00;2>064?2R4D;0@2F30=1332FF;0:7;?10@616413@4825A>?264=3011M0A45118C6E:6B340AHB587AA6708;32FF;32FF;3;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C102506162063[0492:49090IC25970192B5;3B220=5=M;15:5A51c0452K5F12>61?1504;0:]661=05Bb5901>0>9:J02810H:04=4<:K2?2420;50@732>839853J<38:@F371;<372<0712;182111166>405C=A62=354:L67348221A1R0G15H2070F941186<624477;3:4782G51405=00:;3128652090206G:60=?49>45361345@45=06A724;@:22966A11=;3H@000?;205W63>9;2865A:3244=5A225A<47536517E2=:4;2520554432;32FF;32FF;32FF;31052D;6L@23I;9414802I1E0:15>1>T08:E600F;02?5>1HH287>341601I4=5M00K4321445?JF515<<>1067<>309;81HWT5426?:L4600470=03APBN214E?78F;A9I4161C<22FF;32FF;32FF;32FF00>9:J02565256M424>6131=181=6442B2469043?45Q<A85D;04?89324<=A0Q02I8=57:46A1311517M;501H=19600M>9947=];0B0G6394353;<=210?>721;606@>211I2[203080E=8147012506092\02FL4A2204>0GS5;A372N903C12231:D6B01K358101224<<60>K498011210E8F:3<54<:4639:N1F8G36C>740941;?0?85J33B2B@D8331j3:206H?@190U4@0LA208;0963=_:1I01J1547L261554L;6;0413K>M2I>;<976008>3434K179411IC6483=B585<8C5117:4111496;0?:@0277:2;251:D6881<7447;5132>4J7;33B0000117:A832=D72C247]96<73700053833872413450=607:1230>011817G30372904860=57;2F<51J25>832871<2A=014?3243c2:617205<>1SV77QD>4@54@654C25;714=502:@45143726:225:29<00021700C06127736J11B=0g25307G0344A=1J;0629=I<480300<61H>20386F;318G913<4:218>4618<133368<4=@0?<96802:967600200991744;100<4D13346>30?=127;30713010202G402?:?791=01730>2=62511021590H270121=B8[8641053@02=700::4044;2170705H506919162B1C3@294E?B5?I96534=@374<06596:17862485C?1B325CEA2OF34M61B8AN2:3:?2]N01?1>0H;32FF;32FD:1:B080632;32FF;32FF;618?;32FF;31859389;371596;19C102534>1167;32FF;32FJ53;32FF;32FF;32FF;32FF;31859389;:69210<>15:022;32FF;37?1A75:2147:6@?73678A50<4>69082574J90KB86339;21<0373C35;510=31:043522IH315:14D22178C62485:12?597B:I427251280T13441=2312634DB2;=<21_843>224J038C=0:>5>0052C70AB2<2UD7157430?3<2=762032F;32FF;32FF0FF;3>C5124662C4571>2F5H;32FF;32FF;32FF;32FF;32FF;311;3515;32F;32FF;32FF;32FF;32FF;3EG025:2;N29A=093V6<24?00<4883536=3973053;32FF;32FF;B8[8641053@023000B86;42243>06162063[0492:49090IC25B0?=656=>01P6520;2H1M03;32FF;32FF;32FF;3;32F?:C30100210@048>52:5267>34J2596:569H561C05E164@030977G46314:A2<2@011>:0=7D26C03>5619140=01<8@0003PU25C41D8;2972@054151;:?2=666>68D0C73011>:0=7D26C03>E2>;W0<52=02283;32FF;3624929K5?;0050;0413K>M2I>;<976008>3434K179411IC6483=B585<8C5117:4111496;0?:@0277:2;251:D6881<7447;5132>4J7;33B0000117:A832=D72C247]96<73700053833872413450=607:1230>011817G30372904860=57;2F<51J25>832871<2A=014?3243c2:617205<>1SV77QD>4@54@654C25;714=502:@45143726:225:29<00021700C06127736J11B=0g25307G0344A=1J;0629=I<480300<61H>20386F;318G913<4:218>4618<133368<4=@0?<96802:967600200991744;100<4D13346>30?=127;30713010202G402?:?791=01730>2=62511021590H270121=B8[8641053@02=700::4044;2170705H506919162B1C3@294E?B5?14<0;32FF;32F;;32FF;32F07:@`;32FF;32FF;0R661_06;32FF;32;6L068;6>895F;32FF;0:7?G65;32FF;32FF;32FF;32132VA@1:4<49176:4443g:G295L8020406;9NS<51;<8HF1>6A20<:71F?109154=31C411=1BI2F61303A=53189M654=;03519=;8151CQ6<900@457F2:3B5Q2<051H0;G;1B@42004218201238611220710003B40B7J2526U5C38634<24C2K612814<;3672;235103GG118371C:?22A510104@<1D587AA6708;9>8DB47FF;32FF;3215=0=61254402377841;:300435;438901:>361803D:6640D<:I1=@00462;65800<61H>20386F;97?512:1@P76616P84G03<65123FF;32;32FF;324304A<341567<2FF;3;316<0;8114118E0;8H@1B?2865:12?5947078:33GK:6O:570152320=C?62859:387586X3=:=;03>9121;C5424H@TL5>>1537O0D4D:145K125??326A3A550;32FF;32FF;32FF;3255B405F29222=:k10985841T419170J37324;32FF;32FF;32F06703112E2K51315510346=L=2==C5N1FE1790034L70402221P>7B=1?1Z9?12gH17;32FF;32FF;32FF;3:3623D37B==K6510252070:4901:M7202<21=32406A4I:14><15583:BG94=315E517<39B2H>IA032F3:2:476;32FF;32DE70H29B7:<:77253;07>>44;629=I<480300<61H>20386F;318G913<4:218>4618<133368<4=@0?<96802:967600200991744;100<4D13346>30?=127;30713010202G402?:?791=01730>2=62511021590H270121=B8[8641053@02=700::4044;2170705H506919162B1C3@294E?B5?I96534=@374<06596:17862485C?1B325CEA2OF34M61B8AN2:;4?0013H00E>41:8:;7C40@1A6A;_541A5F;32FF;B3D86=A5:30>A:180=04A1010683706062:005536C436?G650A<33:BO4;0117?=375<485=2V5673A2522A225K97?512:1@P76616P842340@03177K01V2<30B1?13>E554001?:4@<1D587AA6708;9>8D01<0M?1551A5402I03>11430715FEC1825A>?264=3011M<=C=35M531=547B031504334A05;;=6A00?85;32FF;32FF;32FF;32FF;32FF;32F;32FF;32FF;3210?7321181M0A45118C6E:6B340AHB587AA6708;32FF;32FF;3;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C10252222;32FF;3231?C8>671I81;32FF;32FF;32FF;3D=612<6527131F:H572;32F0D=?40<502E260N022<2455241313@02:B68223303112E2K51315510346;32F3623D37B==K6510258U3;780;734A?039079;:69210<>;512S39k8:67:96?HJ611:106052084:8>233>13221CF411820068058<;417?70=7<8041411211B3K15M3<816531C?D1CB632703D727337M001:835;54649451099=73:98;101:72;0200H]=D3;02963B01918;1172794I77127@874<31:156ZB<;06:A0122=2B=624==::?2180B;113I63751FD7L1028123@22BE<0GB393392741192?50BBm;62254141RE:H3=D1461143562O2>95451G45=1;5=;L21183366006<6Q=08;10=?591G6791B2307<4706Lc4=1>6F@=MI006H;19453B7005;8G9219=E338PI75L80200U0977;08844=E47?3?G913<4:218>40G500Y10549F31<24<0?6;32F<2590H27017401703989;52C;=5@0334;0502FF;323F2:>@;1H49=9M220;323C20OM80A18;:@3L3701;:F44<0;32FF;32F;;32FF;32F07:@`;32FF;32FF;0R661_06;32FF;32;6L068;6>895F;32FF;0:7?G65;32FF;32FF;32FF;32132VA@1:4<49176:4443g:G295L8020406;9NS<51;<8HF1>6A20<:71F?109154=31C411=1BI2F61303A=53189M654=;03519=;8151CQ6<900@457F2:3B5Q2<051H0;G;1B@42004218201238611220710003B40B7J2526U5C38634<24C2K612814<;3672;375<485=2V5>=9240;2A225K97?512:11DE70H2413034::12FF;31859389;:6;32FF;32FF;32FF;32F;32FF;32FF;3NF5H;32FF;32FF;32FF;32F;32FF;32FF;32FF;32F;32FF;32FF;32FF;32FFL4=;347;32FF;32FF;32FF;32FF;32FF;F;32;32F;32FF;32FF;32F17>88P2@32:;60707608243;<5:H95883ER088380]36J0075221189F15@1176:4443g:G295L8020406;9NS<51;<8HF1>6A20<:71F?109154=31C411=1BI2F61303A=53189M654=;03519=;8151CQ6<900@457F2:3B5Q2<051H0;G;1B@42004218201238611220710003B40B7J2526U5C38634<24C2K612814<;3672;230323GG118371C:?22A510104@<1D587AA6708;9>8DB47J27083;32FF;32FF;32FF;3;32FF61263088380]36;322P39:1124303;H1:10=949A?44334L79054=0BH;32FF;32FF;326156N2@9@1102515;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C10259YQ@IK34;50300C2M1@29A2185>14E7861=B8;RH5313EJ10<15G196?59C5O376166;@53080103133?25>0131=547B031504334A011104O7:0F6>330C2P041C62::3<01843590E5?2955=7D5;B8L05@5K<542DR12102D1?18a79=264E24H5348:040@A130294GG@3:072?1;328V52816652<;8B3;3OI70[1?3BD649311:>?E55F1T0311[68F48B433084B81003036740:0913A81B33L4A;3052834C;?522R>00:6BA708e24B==:42AN30?80F3:R412340210900892E363B8AN2:3:2FF;32FH;32FF;321;32FF;32FF;>0;32FF;32FF;;32FF;32FF;32159371596;19C102515;32FF;32F25A00630@061=J16;5A51321828:2376S6>17036>02271H0>>681X200@H0J5>1596730;=2:@<:112O;;67513611>760?5F<801@00:;<5:6E=689138E4B3;0565;01:;A@052OU10??@68:3;6:@=J:2@12308=6@4421H565142:D215086G;2440Z@?51856F39K9=3257E451D<<8A;C10933670Z17=R3@;><09e064=242:EV981893756089212@442<:84=3A3758310G2001:3C4;1H00;64F2=5112<1160=151007B:^55279;334<<032103C311<7P1;A4=7=82842338139127;3228DD1;@219040@0?P2<2=6>E?2BP435130334:6866M382944Q171>7D<9230?1:832FF<@18AABOS120620<;;I8B11213965365I>K^R0M5I7304619=21670764656B6I6I6099=076@?1740;32FF;389;:6921:557;02811A5I3096;<:132?;009>808>1<66462503511V02B137?14666732238611220710023?1C;2U?E932?2822FF;0:7;?105=;32FF;32FF;32FF;32FF;31859389;:6;32FF;32FF;32FF;32F;32FF;32FF;3NF5H;32FF;32FF;32FF;32F;32FF;32FF;32FF;32F;32E3012611;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0@=15@==7020:69210<5K569156104:8=250?567L1U082345809:47BG43602?L:>95=U0@112C75430;E212271<@202>7;8D0ED8841240=3294634BD59^1==0023111?84;005611128D5307290O;40:1<5@94V6@[:5=9BR180412802117142O400370602:7004@B10;;<=0CB8769769A?1H2[5:=6?62178>042:3454JF7586842:>2<900A3KX4JA6?1320640G?87::73B7004>6;0E0=0218313?0>7>296691145233<7;440J3:7B0K173<8773?1<120=01372?421140:2262:480940404A1;1A46C2770609=511125121;50P8345\27392N5115>932FF;32FF;32F17>88P2@32:;60707608243;<5:H95883ER088380]36J0075221189F15@1176:4443g:G295L8020406;9NS<51;<8HF1>6A20<:71F?109154=31C411=1BI2F61303A=53189M654=;03519=;8151CQ6<900@457F2:3B5Q2<051H0;G;1B@42004218201238611220710003B40B7J2526U5C38634<24C2K612814<;3672;230323GG118371C:?22A510104@<1D587AA6708;9>8DB47J27083;32FF;32FF;32FF;3;32FF61263088380]36;322P39:1124303;H1:10=949A?44334L79054=0BH;32FF;32FF;326156N2@9@1102530527736J11B=0g2530421=322:2=5?D59@@<=40475211:5006098<@1<7627E:0>235@8072M:4210C3124>:8@T=4T1>5E=57;5@0362>?376?5<3931833a?332FF;32EO113552;Q06@4@59?;62:<520031131<=6876640>D50249;6151;756AF:14:2113253B?0C02O3K11C224;;S00243?2347d8<:10H46T=212<511?624H3C025445220587AA6708;9>8D01<2FF;;32FF;2216J74<0083C02;206G0=2<93;51:0CVE55200257430G>9920C0:7005>1373<;194>=4H952=2=:9?4U:B112>KCI0E0>:10=5204A1517002=3:0M11?H;0A;195H0O1:5G1>IB0652104U55=L02M81A?09253@=5@F<:61160=@1E777:G<0982265044920;<8:4?F330]8674732G31=63]1D9015D73792:>332F;32FF4101O4210<>15;32F;32FF;32FF;32FF;32FF;3EG025:2;N29A=093V6<24?00<4883536=3973053;32FF;32FF;B8[8641053@023000B86;42243>06162063[0492:49090IC25B0?=656=>01P6520;2H1M03;32FF;32FF;32FF;3;32F?:C30100210@048>52:5267>34J2596:569H561C05E164@030977G46314:A2<2@0;0:]661=05Bb59413C3FF2E75>D1038484201_K43;8I@9186072940940E3<5D07H1=?20:71>5?:5[13F<=10=>1C255229732:3;32FF;3807D67357>39818C6=:C41;74360E2355907290211960=093V:4850>708M2;9G317129@1202<73Y14E57156?1103;:@1557B020E9:397412H288?83K:A=21:2T020320;>=61;:75<7>?4B8<846305:2205218929@A456:<4428<630;2AE2202310103<231267CFL0597GG@38<3Q?46160C0M453?770K4163653G1@23004643043<2;68203B57444<4K1C>QZ302J<97103>F25@87093011208062037B=0>L2:5>6;9J2032H:5:2K5;;72195<173860R90<391;876=3056823@;0632FFP15439::25182EO11355>34J2596:F;32FF;32F=:k10:5006098<@1<76279=I<480300<61H>20090242299a60F130782A29M6542:1@P76616P<511?624H373F>?891328554<011A5012;541=15@=9;:6;32FF;3225312DG02B0571>26P>]36J007522118G;1B@420042182012:F2;a@5570NN800943933026U5C383NF5H;32FF;32FF;3:4450>19=@;0E0=0218313?0>7>296691145233<7;440J3:7B0K173<8773?1<120=01372?421140:2262:480940404A1;1A46C2770609=511125121;50P8345\27392N5115>932FF;32FF;32F17>88P2@32:;60707608243;<5:H95883ER088380]36J0075221189F15@1176:4443g:G295L8020406;9NS<512M623C75D6?9F103F7F3X=0>472E60:19802<30:1B8AN2:3:C55FFJD;617<:3:282A5055:30>Y3GB;57K855g30:572FF;32FF;32FF;3213NC2=:19@105;3Q018=?32@9745;>LC?59F;0:7;?10@616413@2J4;1=:=?62F635206;6<9061>78421<1;:72:3Hd372=99;0I967516J213208041411211B3K15M3<816531C?D1CB632703D727337M001:835;54649451099=73:98;101:72;0200H]=D3;02963B01918;1172794I77127@874<31:156ZB<;06:A0122=2B=624==::?2180B;113I63751FD7L1028123@22BE<0GB393392741192?50BBm;62254141RE:H3=D1461143562O2>95451G45=1;5=;L21183366006<6Q=08;10=?591G6791B2307<4706Lc4=81X200@H0J5>15967:1@P76616P847;?105=;32FF;32FF;32FF;32FF;7A0<1:6258121>0;32FF;06266D0E0>:10=6;19C10255T42<:081:921368>0A3O22ACH0EU363;503505163;722062124@17014M61B8AN2:3:?2]N0101<340K0;21328E529>2GAI233C025445220587A1I30=F5;3B220=5=M;15::A0122=2B=624==::?2180B;113I63751FD7L1028123@22BE<0GB393392741192?50BBm;62254141RE:H3=D1461143562O2>95451G45=1;5=;L21183366006<6Q=08;10=?591G6791B2307<4706Lc4=1>6FC5424H@TL50062L526110985841T4132FF;32FF;318597;32FF;32F3063224:61=2645:?26;@<43>2<6:G7:B7C40211126:33026O:220UZdC:0]a7=0777G1B7?1=4F9>47;@;002738E06092437065<=0649D68\N16380921DE70H2413034::1<21A3IH3RG9=4;32F;32FF;::14:60;396844:93415@707:@<;=03>1>4843432B1H082:010C>4E63F1@719127667=48;?F15NC2=:19@105;32FF;32FF;32FF;3;32FF;32FF;32F24H=00082=42CG23;L:39601O4210<>15;32F;32FF;32FF;32FF;32FF;3E;:AA7;32FF;3;:6;32FF;32FF;37=A<:6007608243;<5:@4D0:4780040;9H10D1202<21=356824100P037T3><34319155=453A4025<7>?4B8<846300346:8;209A45;4E4757024802:3817>4>K0P=1210J6?2<72:314?0@75379C9@5<0166B55=U61C095I228J363I3BH;:]<@2R23:21207=5411A3<<1@6319022FK0?1145:248;2;4=6H4A380J31921B;@04@6417106@?9D125A:X510:17362:;1:41162267459M06703287?8E0000?C071596;19C?1<0<;044300503I003AP:745;>LC?59F203764HIF=<6G66495350232B473676002009917;756AF:14:21132FF2487;3=25299=3<41:8:;7C40@1A6A;_541A5F;32FF;B3D86=A5:30>A:180=04A1010683706062:00005611128101:72;0200H]=D=7?9430977G46314:A2<2@190144800038:1859386L1F4420:<5B6=>2=3<=A6A;_5340?6062:0?38120X15266433W5D1512F027492351G2@40167PIDD38804B5>263366D71D;5=A02?7G;9289>:?0710122@08K50406C76577L01?5;6611;:484148212276A2?1N27E2010299<2438265=32335;007=11651H641025JDF2>29E1182T4150>:4071D6>3>0A1@10938>1?8A>2606I?=70145C1;21?0;M1:[06814C65>;7;3=76?7121;792828105717<41F=k0101439082>11132C23:83?261517>32194C?A8070=B47125C3802C2A65023033286;32FF;32FF;;11:43;YD062L24H00286@05:2205218929@A457M:6>=@=5<7>?4B8<846302FF;32FF;310938>1?594@13<7328B?421<1;:AA7;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:62526351;32FF@:6242731;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C1025Q5;82@525458@24=0832>16;;0S@6605347=;7411508021T6>J1703<:10E8263P;0810784067;?1G235??A02H04855C39;2747>9<<664;251C011:4A75020>:;26B45>54417<<:21<1;;3I@?=21?7080011?6415@>170675032G90F31728==5710G30I4=0;137809L11I131G0JE1;2:D5B19P860B0>J420:15879588D310]4:32:62076=208107:865911=9245<;0T08M?44B=2D320>210<133134::0:>0604181956104:8=25733232420@87JA8320;3K?48:01732=706:B0;213>070F<5539171O2>2024E24520=00K=0G1GA3;5R0K350433?9160E68683=0A8:0;A03;D11G620N=9?4B9870246G83Bi95340443?OK5201=4D1013?213C33>2R<:112835@2=102037:3924;4<2I3737611N>590A57<:720=80E0A548005=C7<164?53X566@101A>445J542079I930:3E>10G8<<3110L1<;2132309;81HWT54EF1>1?75E611H133B63530<;32FF;32FF;32FF;3A<33:BO4;01144620T>>;13125??3264@310G60345?:<2B32203TC4<7018<5<10959EAL396846<25P2C1OBO123L110:73F>?89132855NM07LE9446J3A4722A7734898N38<5844?65N=221810836543K030;1:089<2;322P39:11;=0C;1076539^0<75?24404826F162C9I92<235286J<31441596;116?3025C79@3971023C04533F;32FF;3212F?:C30100RF76<37>8=386F;318752626:2020Q178A??;02[0?0141;A420@17061?1504;0:]661=05Bb5901>0>9:J02810H:04;32FF;32FF;35879U?8167T7;32FF;32AH373F>?89:H3407:>N263A5242V16<4938A61OHB12021?1822SA3C34D553:03962F24=?4033<0O8G3?C;R@3E2I36P:T472E083:001216503606479@@708M2;9G31@2<6353241P2@32:2124662C41?1A646811;7P796b>84:46=538<0?16572;9>NPBD24105158232;<3DYC3035=087:555_30811763E1F272<0:<@6412221531C8063=75H297=8346=5122752<202375943869157;7521@1A4@44195:?10V3A2058@D8T;7W65:]1IF6=;22783060@33F=209;Y4U758Z32?44Q=23666819A0;5@:C892<15:8741A0;CC54@2C?68>33016:E:1B<2700=59=?684E522:12D098@223>742R:701Y72@69;2823;4Q579;4206:58384081B<<7\=7>1929;6J@;728929S2H6A0;O7031;Q:73\D1A\92E0:632>=0@02;0H31@5625C5F;32FF;32FF;32FC;`<2E332;622897;8525945=:8O5021A453816978<6054C9C54666@?1030<30M031D986:5>01Q073:K04413565256M424>6131=1@13614=H3180:914A946;32B0:F20@D;32FF;32FF75E=?A111<03;:<9<321<6:8113056402H363696:B3677J:0092F;32FF;32FF;32FF45118C6E:6B340AH84750G2?>8E5705804?;270122794I77127@?C8>671I81;32FF;3238D29<623?2FF;7=322:2=5?D59@2:038=424:0@C28;5802159389;:69210<>1563I3BH;F5H;32FF;0502185691H10Q00S:13432HA56:K28070=B47125C3802C2A51M9>91141=9645;1445><72F48089D66<4><01H=S<04<0;7<11A8A51150148107LE9446J3A46<0121?48031620;D;32FF;32FF;318593966A;_541A5F;32FF;B3D86=A5:30>A:180=04A1010683706062:00005611128101:72;0200H]=D=7?9430977G46314:A2<2@190144800038:1859386L1F4420:<5B6=>2=3<=A6A;_5340?6062:0?38120X15266433W5D1512F027492351G2@40167PIDD38804B5>263366D77>88P2@32:;607076859389;:69210<>;32F802:1D17V5@G2783060@33F=232FF;32FF;32FF;7905A15=3@3;2D907@17014M61B8AN2:3:?233B2B@D8331j3:206H?@190U4@0LA208;0963=_:1I01J1547L261554L;6;0413K>M2I>;<976008>3434K179411IC6483=B585<8C5117:4111496;0?:@0277:2;251:D6881<7447;5132>4J7;33B0000117:A832=D72C247]96<73700053833872413450=607:1230>011817G30372904860=57;2F<51J25>832871<2A=014?3243c2:617205<>1SV77QD>4@54@654C25;714=502:@45143726:225:29<00021700C06127736J11B=0g25307G0344A=1J;0629=I<480300<61H>20386F;318G913<4:218>4618<133368<4=@0?<96802:967600200991744;100<4D13346>30?=127;30713010202G402?:?791=01730>2=62511021590H2701103;:2@;44=5M00K432148IF150J0A713320271D0?34200=@340<;2=424:0@26A3A550;3280LB:>86<6389C24342<11?070;062>4711::<2W?1<2:1650FH011330568864I72;6>J9;56233666:0906B10=47=812453?B8440M:91604<1:205:::204386>:074A8>4985I34D7@00805:4?>;08D39J0?73GM93CGK704D;9I7=308C553670B?53G;31531AA73412A012B;125E55607Hm6:31?372C45?3LA46;8:474J7407623:551?6E3?3>7N3<00576>09H1;60225S8175298:110A1S361:?1215221>68?23225:30547130<56G368CC6029009>;?604=254J2B30<:4C4C510203034><39120?111E7001<231=6405`525?720>2866I601043062N8C:=3975Z54B<0;5674;:>2D92G181423Ka03826;74068:493?6?13O84317G62333:6A34:93>800@>05621311A=@0364F4;0542;0;355B=0:4N617714I5A7<6;1430k7EQ7301>B0_;;0597973830?3258=02982I210<233:570152320=C?62859:387586X3=:=;03>9121;C5424H@TL5>>1537O0D4D:145K125??326A3A550;32FF;32FF;32FF;3255B405F29222=:k10985841T4191FF;32FF;3NF5H;32FF;08?;53344>930802159389;:6921045143F;32FF;32FF;32F0431253G500130H00286@1H<;1B6:180=04A1010683706062:005536C436?G650A<33:BO4;0117?=375<485=2V5673A2522A225K97?512:1@P76616P842340@03177K01V2<30B1?13>E554001?:4@<1D587AA6708;9>8D01<0M?1551A5402I03>11430715FEC1825A>?264=3011M<=C=35M531=547B031504334A05;;=6A00?85;32FF;32FF;32FF;32FF;32FF;32F;32FF;32FF;3210?73211861F812=12511C653;937D15B20D2413K24BE0:10041A4;41I8=01303F6L0=>7?KC;4H025U41;11K16A;70F52F720425:8627:714<:I0001@1B92056202@906>1:16J::25@;13:2?0127:P65424A55=M0CDC37:3A0696B0<41053=99900;@533<050941A5715^1P8K000201A4;7:\157B078C774386<4IE221F893111M84H532541E3H65818L01H1076061;A71732126101R098232Q8750665A878;D954F1619@<32K4W01<2<1>80E20970224:<2045391<81:4151107>00535@0=:52G14W13=94418??>3:10=5730N>05012W3B:46:72=30:0224FD65E574b;=ED:65;C04231@Q;A1981>@2@:035D5FD>;N476174823<>:324]36J0075221189F15@1176:4443g:G295L8020406;9NS<512M623C75D6?9F103F7F3X=0>472E60:19802<30:1B8AN2:3:C55FFJD;617<:3:282A5055:30>Y3GB;57K855g30:572FF;32FF;32FF;3213NC2=:19@110@53<37LO0D4D:143><34319155=45201<14F3299C6?49:M23S:68424C=5D2723>>9;B@<<715S540<603241GhC:620>52B011901>2B>;36009D6?9F103F>@16L2?>629FF;32F97?G65;32FF;23B;5205:5c32FF;32FF0=;19:3336956741I1:1<=f1>755:4134G4<21;0067><:I51E0EBN12O26=2I08O9;B116=77?0022@?1:1:CJ7<6=71>=2=77M37J14SC1I23I1156L0524404826132V>230025003262;34F;32FF;32FF;32FF;32FF;F;32;32F;32FF;32FF;32F17>88P2@32:;60707608243;<5:H95883ER088380]36J0075221189F15@1176:4443g:G295L8020406;9NS<51;<8HF1>6A20<:71F?109154=31C411=1BI2F61303A=53189M654=;03519=;8151CQ6<900@457F2:3BI5<599908=4339F10929E38]45<@@28?@D6<250P;1@71515>3262>100572<6>19173:003811502:5181;02832>025P2070A4?46J=0J535]B4P62>N0392401GF222574<5:000446442423=14J:0@0B659444A87;<3304@8980E:B00:D2CK0320R?J?6F06>A38DK297E4Q04:72:;62K90;184;875458840B0333G5H:I3:5147>64391542A62I62:78W0722N@095010<1771510=WT07012204210610391KD14172?6863C48D3>:12381A43151B47:D<32;5572410C?15GEA2B032B48P4932J12;>213:A21777031C05950G4H:4G162>:20600=11602;23273G531KC=J692685Q5244=J5J17NI0008<4=8268040703:4NCB118;020829>05=54;1S82A440241=5644493:47:D55>:7:0H30007122033J30A3A514071D068531326;KF08159414601022:57D0156;J8;3;D28228462C5@1:1=:210;283412FB1C236_<12416>686@0@512564705N3A0H0<7?63603D9166213081541;712B6219<6B5:353:166?=;4T2?11O<5311;:0=0>0017M3<;:?15C6731>01?H78;=?5>1>20G2621@6063G543CA4249>0?6Dc94140417<5943;0H3C173:375320C0857097=43\BM;11?:=B34;222034<020085913A0;1005@94AF599F5DX86182G<1:@5631\<62795<1411A=4>5C94P13<1>612@J3:><4979194B0=A:7B:2;0160725L7;:355355;UL70F4551371L211238SH<33=72;5481D63=27:3C04092@P62C3800L0:0:7232012U57144272BD:3574032450::=;1:=212BZE360272760;A65;2?Y@456C0F03532=4507=1<82H=L71G3@8:3h54=452031?734J1:4210H5119]@1301741>A:7::402A:410<>1SV77QD>4893243>06162063[0492:49090IC25970192B5;3B220=5=M;15:5A51c0452K5F12>61?1504;0:]661=05Bb5901>0>9:J02810H:04=4<:K2?2420;50@732>839853J<38:@137>D19720470709<08H0J2_702A6;@7=96;19C1025?4;A7C690600C11370=2231600J==27<3020027270F10053751<<:;1605742<346?4?81036615:134<:78I:103NP32210533436;650:846?2S0E=CA31517?33A2:JN02;2=32BC170600DA611LE48610903A103>B04671110;38@E256F4?6=1>B3?01:10>6010>5:=78<;0@011792622C0037A3BG364A2>9358063;><30EQ24<<08<5E46806S08@0B3472G23=1186CP93j?17D81726KL20<=0530=2:5103D20@322HJJ9662073>826?>0355162:8F0>B<44:05E:<200EX04302404F<1>612@J3:><4979194B0=A:7B:2;0160725L7;:355355;UL70F4551371L211238SH<33=72;5481D63=27:3C04092@P62C3800L0:0:127;3071301327542N256083B2<46J;J>@24111020?174I2I25:0O52451B107192051:10130T0754251M10>90C10H8Y0:1;0:0J>2:84662O96L914524I500>316509;75@A1K0\GE69@D2262526U5C386114C0?8?B1E1141<38?45302P2@32YQ@IK34;50300C2M1@;32FF;32FF;32F7480083B:3>Q560>45A07@322E>2005536C436?G625307G034B;53E@2K09:;2278306005E:<200EX04VA@1:45T847;?10505L4607098H43>2D1<9932FF;;<30E3=14656254B37C3H8P0040;32FF;AF;32FF;32FF;32F;32FF;32FF;32FF;;<30E3=14656254B37C3H8P0040;32FF;AF;32FF;32FF;32F;32FF;32FF;32FF;;<30E3=14656254B37C3H8P0040;32FF;AF;32FF;32FF;32F;32FF;32FF;32FF;;<30E3=14656254B37C3H8P0040;32FF;AF;32FF;32FF;32F;32FF;32FF;32FF;;<30E3=14656254B37C3H8P0040;32FF;AF;32FF;32=307<40>10H8Y0:1;0:0J>R?>1216>5909E>=745=01OG67003703161D274744712MC159E708?I4300860@2757?2B10720510071940e<:0KM155=7142BE27<1826J331dE09?Q0B31033>242<1470709<08H0J2496;0?:@0278>1?=515229784;37<@38C=0:>5>0052C700@36928190701B91?45Q<A8216762450307550;37621953975@8=277:1>>05H13312930:133k=0332324=>19;4374162K1602533710181C3P3212\8:O4054101331:823542H536D860641191111=3393G703303537;F9954U:0237=;4A10665I3B9>D19287A:93FM?40;13E?061053262>@0A85;B748;8150080>44;4D=97@0003601717=0;007B501:6=K:;0>=14712232F87;17A2835;:9021041G6L2217Q3=26<2056567@27597:<1<3227133341:84:20=59:B0014FA92351<;43K6;C5193642717P77U6042C475>;=A3@?2N;13=L0A1<:003902;:47JA0<0333T295660=0;0=03659250212:2?E10I1:4714200500>120253864:52QS@34<4090@15;32??2;G22>2=00;@58:110860123861E212271<8F:01B;658=01161@29E4D068531326;KF08Z9?12gH174E048641053@02=700::4010<>15;32FF;32F87F282Q07>1563I3BH;F5H;32FF;0502185691H10Q00S:13432HA56:K28070=B47125C3802C2A51M9>91141=9645;1445><72F48089D66<4><01H=S<04<0;7<11A8A51150148107LE9446J3A46<0121?48031620;D;32FF;32FF;318593966A;_541A5F;32FF;B3D86=A5:30>A:180=04A1010683706062:00005611128101:72;0200H]=D=7?9430977G46314:A2<2@190144800038:1859386L1F4420:<5B6=>2=3<=A6A;_5340?6062:0?38120X15266433W5D1512F027492351G2@40167PIDD38804B5>263366D77>88P2@32:;607076859389;:69210<>;32F802:1D17V5@G2783060@33F=232L79054=0BH;25L7;:355355;UL33?c0452F3623D37B==K651025FF;3NF5H;32FF;F?109154=31C41C6:1;<:31B453;B1211>22144;609013335787;;25143B3a124>6D75411033?5010281C034480276DA=M903E795577F820<<21A=:17A9571A90X09004742?5>9=8MI3@1@@D1176022420B;>244K507076010115328::6669413631?7I14912E773221261
\ No newline at end of file

From c9d674b60d76ca2bae4edd84599298da984da9a5 Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Thu, 28 Feb 2019 01:28:29 -0800
Subject: [PATCH 077/178] Remove autogenerated test file

---
 lib/decompress/zstd_decompress.c | 3 ++-
 tests/truncatable-input.txt      | 1 -
 2 files changed, 2 insertions(+), 2 deletions(-)
 delete mode 100644 tests/truncatable-input.txt

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index b9362eb6129..a1f656ea4c3 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -530,10 +530,11 @@ size_t ZSTD_decompressBound(const void* src, size_t srcSize)
 
         {   size_t bound;
             size_t frameBound;
+            size_t frameSrcSize;
             unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
             if (ret == ZSTD_CONTENTSIZE_ERROR) return ret;
 
-            size_t const frameSrcSize = ZSTD_findFrameCompressedSize_internal(src, srcSize, &bound);
+            frameSrcSize = ZSTD_findFrameCompressedSize_internal(src, srcSize, &bound);
             if (ZSTD_isError(frameSrcSize)) {
                 return ZSTD_CONTENTSIZE_ERROR;
             }
diff --git a/tests/truncatable-input.txt b/tests/truncatable-input.txt
deleted file mode 100644
index 02b78f51892..00000000000
--- a/tests/truncatable-input.txt
+++ /dev/null
@@ -1 +0,0 @@
-;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C1025E554001?:4@<1D587AA6708;9>8D01<0M?1551A5402I03>11430715FEC1825A>?264=3011M<=C=3?453;7911?700=39<4HJk1098>A054544C31V4H98061EN805851?4:=0=61254402377841;:300435;438901:>361803D:6640D<:I1=@00462;629>;H700;AKD1720143K9B43100F0972121:2?2Q285C0018634S:<31J07721750E1822775E24=31195Q04;5;2QY21?30?450;991;1>412646:500N<3>;D0=40793;53B25>504O912H363696:B3677J:0092;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C1025>3<586>A4=31195Q04;5;2QY;32FF;32FF;32FF;3237034859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C1025E554001?:4@<1D587AA6708;9>8D01<0M?1551A5402I03>11430715FEC1825A>?264=3011M<=C=3?453;7911?700==A;32FF;32FF;32FF;32G>8<;32FF;32FF;32FF;3;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C102515;32FF;32FF;32FF;790<1:6258121>0@=15@==7020:69210<>1;32FF;32FF;32;32FF;32;32FF;32FF;32FF;32F6;32FF;32FF;32FF;32F;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32;32FF;32FF;32F5065:118300::F64401<0321?25K56915?46;:N0O119@<64716;3A4=<90130=417@2MB7G15495610>F67M=2?:3772533:<0@43;32FF;32FF;32FF;32FF;32FF;32608?35>0673;:69210<>15;32FF;3<0>3121?;32FF;32F;32FF;32F;32FF;32<5?2;32FF;32FF;3>C5124662C41?1A64;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C1025E554001?:4@<1D587AA6708;9>8D01<0M?1551A5402I03>11430715FEC1825A>?264=3011M<=C=3?453;7911?700=39<4HJk1098>A081:004518836;760;538:31L14365324B37C3H8P004002J81C2;22865G13415X1:;@B0029625MP0H0421=322:2=5?@:928610<324110G4@AD3:941>544C31V4H98061EN805851?4:=0=61254402377841;:300435;438901:>361803D:6640D<:I1=@00462;629>;H700;AKD1720143K9B43100F0972121:2?2Q285C0018634S:<31J07721750E1822775E24=31195Q04;5;2QY21?30?450;991;1>412646:500N<3>;D0=40793;53B25>504O912H363696:B3677J:0092;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371;32FF;32FF;32FF;32FF;;=9:;32FF;32@:<760125B101K032300331J5F;6241;1D85:00?41040005?6F17305F0;5;B604028=:0073C26@21T06274OJ2J9@29?=039:503010425@719127667=402FF;0:7;?105=;32FF;32FF;32FF;32FF;31859389;:6;32FF;32FF;32FF;32F;32FF;32FF;3NF5H;32FF;32FF;32FF;32F;32FF;32FF;32FF;32F;32FF;32FF;32FF;32F0431253G500130H00286@1H<;1B6:180=04A1010683706062:005536C436?G650A<33:BO4;0117?=375<485=2V5673A2522A225K97?512:1@P76616P842340@03177K01V2<30B1?13>E5540I6;32FF;32;32FF;32F7?@600;283402<;32FF;32FF;32FF;;32FF;32FF;3;32FF;32FF;32FF;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C1025F7=;32FF;32F073973053;32FF;32FF;32FF;3=6125440237;32FF;32F3063224:61=2645:?26;@<43>2<6:G7:B7C40211126:33026O:220UZdC:0]a7=0777G1B7?1=4F9>47;@;002738E06092437065<=0649D68\N16380921DE70H2413034::1<21A3IH3RG9=4;32F;32FF;::14:60;396844:93415@707:@<;=03>1>4843432B1H082:010C>4E63F1@719127667=48;?F15NC2=:19@105;32FF;32FF;32FF;3;32FF;32FF;32FF;32FF;;32FF;Q04;5;2QY;32FF;32FF32F;32E3012611;32;32FF;32FF;32;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C102508;32FF;32FF;3@6=;32FF;32F;32FF;32FF;32FF;32FF;32FF;032H1C197313;32FF;303:64<468AI0;32FF;32FF;32FF25?6Q=9=8?36A6U;32FF;32F;32FF;32FF;;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C1025A6;32FF;32FF2487;3=25299=3<68F52:D;32D10080?4472E60:19802<30:B03:19=@755441705=@73:63=<05^70<24FX;008330;251641:B>3OF12A28<:231<21:23H4=4215<0171825>201803203@0800<17939?0<96424;;2D320665550362:1;72:3Hd372=99;0I967516J21320C0=4<045238331:7N531<5:;C320E13303>>:1:0;101579:D334243409>O62:H91@5:14N56@04<67504=42Q3R3F0_1107657505<540@030@272209;53C2210]60@40G:1G118@4B37?299h4652B;53E@2K09:717M031990:8<1999@;0=17:;32FF;32FF;32FF;32FF;31859389;:692;32F;32FF;32FF;8@@?2876:3H2A8N<0I88782A2:B0431253G5001<53106106H1261@6G9629B4048E458025V17I1:0AE1?7<7a:41<1;0J3208:7<50800394@6181;168201<14F3299C6?495926>2=6>E?2BP435130334:6866M382944Q171>7D<9230?1:832FF<@18AABOS120620<;;I8B11213965365I>K^R0M5I7304619=21670764656B6I6I6099=076@?174@@02;4::41B23017B4261943@81@605P032;R3B46<36:183630409^;F8?346250202E0715228:C4<3@34X90081:V5992>802;02>=L1F3B60;32F;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C102588P2@32:;60707608243;<5:H95883ER088380]36J0075221189F15@1176:4443g:G295L8020406;9NS<51;<8HF1>6A20<:71F?109154=31C411=1BI2F61303A=53189M654=;03519=;8151CQ6<900@457F2:3B5Q2<051H0;G;1B@42004218201238611220710003B40B7J2526U5C38634<24C2K612814<;3672;235103GG118371C:?22A510104@<1D587AA6708;9>8DB47J27083;32FF;32FF;32FF;3;32FF612631460D57;2d;32FF;32FF;32FF;003L27]55;9A3L6:90528;32FF;32FF;32FF;32F;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C10250;32F>5B6156N2@9@1>6;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<111<35553@25207I68072522A261>404;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C1025E554001?:4@<1D587AA6708;914<0;32FF;32F;32FF;32F;32FF;32FF;32FF;32FF;3076@?17415;32FF;32FF;32FF;790<1:62526351;32FF@:6242731;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C1025E554001?:4@<1D587AA67;32FF;32FF;32FF;32FF;32FF;F;32;32F;32FF;32FF;32F17>88P2@32:;60707608243;<5:H95883ER088380]36J0075221189F15@1176:4443g:G295L8020406;9NS<51;<8HF1>6A20<:71F?109154=31C411=1BI2F61303A=53189M654=;03519=;8151CQ6<900@457F2:3B5Q2<051H0;G;1B@42004218201238611220710003B40B7J2526U5C38634<24C2K612814<;3672;235103GG118371C:?22A510104@<1F;3185790<1:6258121>0;32F;32FF;32FF;32FF;;8150=51426;6L068;6>89546;32FF;32CC80?;32FF;32FF;32;32FF;32FF;1:4=<10B3:3O753224260?9:C2193@;32FF;32FF3278AB;32FF;32FF;E:58>2;720441529?>90D<=;129090=0<8@50G0M<649YEA252;3809D700JJ1>173E23C7<=;E1V6D>0717550F110FC7<1<6:7;M96G4E0<321@65290752;196>2T;?1@79>8>70349?F0337T2203;050038D29<623?1C;2U?E932?282C43C54666@?1030<30M031D986:5>01Q073:K04413565256M424>6131=1@13614=H3180:914A946;32B0:F20@D;32FF;32FF75E=?A111<03;:<9<321<6:8113056402H363696:B3677J:0092;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C102515;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C102536183D131:836=602;32FF;32FF;32FF;32FF;7B0D432FF;32FF1?25K56915?46;:N0O119@<64716;3A4=<90130=417@2MB7G15495610>F67M=2?:3772533:<0@43;32FF;32FF;32FF;32FF;32FF;32608?35>0673;:69210<>15;32FF;3<0>3121?;32FF;32F;32FF;32F;32FF;32<5?2;32FF;32FF;3>C5124662C41?1A64;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C10258<;32FF;32FF;32FF;3;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;3215937159;32FF;32FF;32FF;3;32FF;32FF;32FF;3;32FF;32FF;32FF;324:31A81>134167121>0;32FF;32FF;A;32FF;32FF;32FF;32G>8<;32FF;32FF;32FF;3;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258276G4:9:1636E=042103610I@81:H349=56350>303692503623A2Z054151;:?1:30KC6201:=53309@90;32FF;32FF;32FF9415847A16?:1?7914;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C1025E554001?:4@<1D587AA6708;9>8D01<0M?1551A5402I03>11430715FEC1825A>?264=3011M<=C=35M531=547B031504334A05;;=6A00?85;32FF;32FF;32FF;32FF;32FF;32F;32FF;32FF;3210?7321180?033;20671I8155@403322434546<_36069622877<5<3O6@5?C;01^80>23633233G:=3=568;>1?226:G:G2?C5?=40C;:<204?476>803B941;A420@150<364<43L4>6322<6?5EF<094940>44>M<424120164362K40812H:7=231B420824:56:A3D8828C6<010QG6B6H77?32O07:R83FO3?;C;7:E3><=40475211:5006098<@1<7627E:0>235@8072M:4210C3124>:8@T=4T1>5E=57;5@0362>?376?5<3931833a?332FF;32EO113552;Q06@4@59?;62:<520031131<=6876640>D50249;6151;756AF:14:2113253B?0C02O3K11C224;;S00243?2347d8<:10H46T=212<511?624H3C025445220587AA6708;9>8D01<2FF;;32FF;2216J74<0083C02;206G0=2<93;51:0CVE55200257430G>9920C0:7005>1373<;194>=4H952=2=:9?4U:B112>KCI0E0>:10=5204A1517002=3:0M11?H;0A;195H0O1:5G1>05951C453002500326007?7;1;7?6:48:A5@260176084693008=26FG81>B1F9145P037J3950175302807V4682>338<38@651026BGG2445E12G7401;M850=41?26;6L364B8<336?0744166081O3\8I>6682T16;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C1025E554001?:4@<1D587AA6708;9>8D01<0M?1551;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C1025LC?K31<5046091091J070A2J4A03505163;722062121M54154741D0:94184122;Y:00284190;:I5?DE4472711;<9866?4;<8;H;39530A11K92@3?05:6524H391:46O880173O<2?3:4@86193544807179J28511>105:<5B3D3003<=A6A;_5340?@C07552C10;@;79245O870331=3J214;F:62G917760F01522C0EF1>1?75E611H133B63530<;32FF;32FF;32FF;3A<33:BO4;01144620T>>;13125??3264@310G60345?:<2B32203TC4<7018<5<10959EAL396846<25P2C1OBO123L110:73F>?89132855NM07LE9446J3A4722A7734898N38<5844?65N=221810836543K030;1:089<2;322P39:11;=0C;1076539^0<75?24404826F162C9I92<235286J<3144782A5055:30>Y3GB3400<3475217I12=35P019<74:0117?=375<485=2;32FF;31H9000>21;32FF2<1K240;32FF;32FF;;32FF;3>2222;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;321G4D73?U?8167T7;32FF;32AA67;32FF;32FF;3;32FF;32FF;32>915725;583540862422E85@00213929;34=G65A81G?5912450@104320:B4C41909=298B641P6I0;0;91C901>2>4@359417;2;1K0Y;>1`72111576626;LF@@>751H753E100=C0:481604054091063;>817D124130K5572:::31320=9678=1560<241=07MH:0<13A@C;5=1G3D;07=1;1308BA2545727\@11<1G09F0031@30?;4H05760H81130AP2=:270568@3IA2N9:65=6544;654=1G;270?12=14:043B304:700=?723L68270>>64>B;1709934mS561=11:A322:2614377042A;605DE6GD7521<81;:F41:4m?6>3E3<92>1=0;55XB2=9<00134G244Q>300@7770260A008531B4C521E20069A25C4@1=1CAB739:A55B405F29222=:k1098>A054544C31V4H98061EN805851?4:=0=61254402377841;:300435;438901:>361803D:6640D<:I1=@00462;629>;H700;AKD1720143K9B43100F0972121:2?2Q285C0018634S:<31J07721750E1822775E24=31195Q04;5;2QY21?30?450;991;1>412646:500N<3>;D0=40793;53B25>504O912H363696:B3677J:0092;32FF;32FF;32FF;32@8<192@D36<66<9210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C1025026227D046M501;32FF;32FF;32FF0081712M14;32FF;5353;:0;5F?5211274>?0:3?;32FF;32FF;32@7]21:5I1B332A05471:53328413:5;:>475;0@A111j4>7D67357>39818C6=:C41;74360E2355907290211960=093V:4850>708M2;9G317129@1202<73Y14E57156?1103;:@1557B020E9:397412H288?83K:A=21:2T020320;>=61;:75<7>?4B8<846305:2205218929@A456:<4428<630;2AE2202310103<241926311410N707JC26395I0310:=8628903?46C92F0C3?1186@N58=02982I210<233:570152320=C?62859:387586X3=:=;03>9121;C5424H@TL5>>1537O0D4D:145K125??326A3A550;32FF;32FF;32FF;3255B405F29222=:k10985841T419170J37324;32FF;32FF;32F06703112E2K51315510346=L=2==C5N1FE1790034L70402221P>7B=1?1Z9?12gH174E040041C62::3<01843590E5?2955=7D5;B8L05@5K<542DR12102D1?18a79=264E24H5348:040@A130294GG@3:072?1;328V52816652<;8B3;3OI70[1?3BD649311:>?E55F1T0311[68F48B433084B81003036740:0913A81B33L4A;3052834C;?522R>00:6BA708e24B==:42AN30?80F3:R412340210900892E108=4339F10929E38]B90929784;37<@3=50<6:126<=A54281<11U3824611L3206\FF;32FF;32FD:1:;3N>20:4@MR4>h0>7F663:@47503J0=3@5<;68AA39226;5A51321111H=4;:5@0@1;238>6<43C915B543?604:137601I200@02221218;746D45450325A4S@00A641[8;5GF7408D;R11:@f5560292;32FFJD;617400>;1C3J;31859389;:69;32FF;32FF;32F389;:6;32FF;32FF;358G273N414C0;22A8<320971\040:1A21:K2;498672FF;32FF;321593136CD>011>91=1<37;5@0362>?376?5<3931833a?332FF;32EO113552;Q06@4@59?;:5>01Q073:K04413565256M424>6131=1@13614=H31314656254B37C3H8P0040;32FF;AF;32FF;32FF;32F;32FF;32FF;32FF;;<30E3=14656254B37C3H8P0040;32FF;AF;32FF;32FF;32F;32FF;32FF;32FF;;<30E3=14656254B37C3H8P0040;32FF;AF;32FF;32FF;32F;32FF;32FF;32FF;;<30E3=14656254B37C3H8P0040;32FF;AF;32FF;32FF;32F;32FF;32FF;32FF;;<30E3=14656254B37C3H8P0040;32FF;AF;32FF;32FF;32F;32FF;32FF;32FF;;<30E3=14656254B37C3H8P0040;32FF;AF;32FF;32FF;32F;32FF;32FF;32FF;;<30E3=14656254B37C3H8P0040;32FF;AF;32FF;32FF;32F;32FF;32FF;32FF;;<30E3=14656254B37C3H8P0040;32FF;AF;32FF;32FF;32F;32FF;32FF;32FF;;<30E3=14656254B37C3H8P0040;32FF;AF;32FF;32FF;32F;32FF;32FF3G7304HF606<;32FF;:3>:9203=54B83;31859416H6ER@F4;32FF;32FF;32FF;;32FF;32FF;32F;32F;32FF;32FF;32;32FF;32FF;32FF;?T12E81k21=87=22?906@707:@`;32FF;32FF;32FF;32F6>30@4>6@4513<>2FF;;32FF;324000<304A656?:14A22227;052611?01:2137480083B:3>Q560>40;I>53:9757203760J24<;66?0:B91>1C10>;?08>102AJ86@4057190553<@09A36E3971D3E4L1I54MK106352F^3098A336I033;6;=1=10;84101O422F5077743Q0T51617:943=22002?1F427:005010A4:GP>>3851532@@38110394H:8:2Bd12831D?3808>40<3=J=E2>;W0<52=022835024115A02>51514160=050623K62014152J12C1;Q53C831M00;4?0013H00E>41:8:;7C40@144;F62529>2G9B335152D@0223KE1V2444A19M03J5923=;36:166450F719933C3D66H880HG04>5:50?8501KA883=Y007A5E24723556A1I2:1250?5F324H=00082111090810A95:>242<13<6012:40B3H0@:85510027C7D:895>:44=536063265930;390=?713:234;425515F0>574274050<120601@024L373221J7R555A;=86@TD72;0FL05274>102C778;C5434E9=381370353325F121030B3:04A8YL794021>084210961:22>1G280;8>;D;32FF;32FF;3143H020@949I2=2?481544<6>00;25>12648=6@356>4189D66<4><01H=S<04<01:3324I=>3?22T049O00805432<73031305K?2308793;17074@L95652100611?20V540705;8:5E420645R121913:3D101022@M32T13<15H22<437;30>:8:;3KDL18434;=7C6<4A65229732:3;32FF;3807D67357>39818C6=:C41;74360E2355907290211960=093V:4850>708M2;9G317129@1202<73Y14E57156?1103;:@1557B020E9:397412H288?83K:A=21:2T020320;>=61;:75<7>?4B8<846305:2205218929@A456:<4428<630;2AE2202310103<241926311410N707JC26395I0310:=8628903?46C92F0C3?1186@N58=02982I210<233:570152320=C?62859:387586X3=:=;03>9121;C5424H@TL5>>1537O0D4D:145K125??326A3A550;32FF;32FF;32FF;3255B405F29222=:k10985841T419170J37324;32FF;32FF;32F06703112E2K51315510346=L=2==C5N1FE1790034L70402221P>7B=1?1Z9?12gH17;32FF;32FF;32FF;3:3623D37B==K651025:03=70:610167380681F;;=9:;32F;32FF;32FF;32F;32FF;32FF;32FF;32FF;316@E04P06>642350190=725;32FF;3215937079:=811935;32FF;32FF;32FF;32FF;32FF;B1F9140:0?900A::6<26;K44<==16001605:5c2462;1297;0D5241^41=688595>G1B7?1=4F9>47;@;002738E06092437065<=0649D68\N16380921DE70H2413034::1<21A3IH3RG9=4;32F;32FF;::14:60;396844:93415@707:@<;=03>1>4843432B1H082:010C>4E63F1@719127667=48;?F15NC2=:19@105;32FF;32FF;32FF;3;32FF;32FF;32FF;32FF;;32FF;Q04;5;2QY;32FF;32FF32F;32E3012611;32;32FF;32FF;32;32FF;32FF:=10;K2281113::=5>B;8=4603;000A4216063D2192612704A75223C9:A056C>4K;215B;;?0006=B:CL233311P28;524G1086483E0C4BP:82I;;=399108>92072L9>61870B17246:484880D:A98354:308<2F=3L3722G56=2F38@A43=16A6K:AN6015274PP90:2BB06@0001?:9C<1D82=3912@7FJ<37U73C@50;;32FF;32FF;32F082;0:11072317I667O27C16M;7<11A8A51150148107LE9446J3A4722A71B32M:2=5?@:928610<32411;32FF;32FF;3433146G028C42D021<811:02=34400843669?105=;32FF;6<;:>34>1167;32FF;32FJ53;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C102515;32F>4OK8222PB<5000<1:6258121>0;32FF30?4FF;32FF;32;32FF;32FFFF;32FF;32FF;3185910;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C102588P2@32:;60707608243;<5:H95883ER088380]36J0075221189F15@1176:4443g:G295L8020406;9NS<51;<8HF1>6A20<:71F?109154=31C411=1BI2F61303A=53189M654=;03519=;8151CQ6<900@457F2:3B5Q2<051H0;G;1B@42004218201238611220710003B40B7J2526U5C38634<24C2K612814<;3672;235103GG118371C:?22A510104@<1D587AA6708;9>8DB47J27083;32FF;32FF;32FF;3;32FF61263088380]36;322P39:1124303;H1:10=949A?44334L79054=0BH;32FF;32FF;326156N2@9@1102515;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C10259YQ@IK34;50300C2M1@29A2185>14E7861=B8;RH5313EJ10<15G196?59C5O376166;@53080103133?25>018R1A112058466?542151>=01B32OR1142E9G30911413=@208086:3056=]6Y55217214222F143:M711392937540200333:0533>51106=B1=@4>358K15:70>80211<34319155=453A402FB?9>98B5h419D0?3162211=109L0A1R>13F0B11=8547@I11082<40O1:57=930@B0:12171@>38C17M@215B34<07009;643F8P4[515A464O6J13:?A017A6C3@32SB<2L25A61<81:7K>3>:636041[4L8A??;0;32FF;32FF;32FF00EU363;5A308:0F441N38<5844?65N=22181083;32FF;32FF;32FF;35434O3=?25235;R504?H6160955G04?73<0104105M41:1<32611G3011>224024677G>5<=1=:82491>@>425312DG02B0571>26P>51785>93<04330582008?5>192079904<1;A1:3:HH00598?=<;1010<=>0:00050:8465C?30R4B11583G29;@C940:45><72F4802=324=;289801302?443D61256H2603SO35b:6<0121?48031620;D=?65Q6W3:01B3?01523CJI84871146111430417555<44;>C9?31185317=0BB058809>9761H8?704:6:2199;8SC1I23I1156L055G561B3<:9882286913A<0143<;62C34d00;2>064?2R4D;0@2F30=1332FF;0:7;?10@616413@4825A>?264=3011M0A45118C6E:6B340AHB587AA6708;32FF;32FF;3;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C102506162063[0492:49090IC25970192B5;3B220=5=M;15:5A51c0452K5F12>61?1504;0:]661=05Bb5901>0>9:J02810H:04=4<:K2?2420;50@732>839853J<38:@F371;<372<0712;182111166>405C=A62=354:L67348221A1R0G15H2070F941186<624477;3:4782G51405=00:;3128652090206G:60=?49>45361345@45=06A724;@:22966A11=;3H@000?;205W63>9;2865A:3244=5A225A<47536517E2=:4;2520554432;32FF;32FF;32FF;31052D;6L@23I;9414802I1E0:15>1>T08:E600F;02?5>1HH287>341601I4=5M00K4321445?JF515<<>1067<>309;81HWT5426?:L4600470=03APBN214E?78F;A9I4161C<22FF;32FF;32FF;32FF00>9:J02565256M424>6131=181=6442B2469043?45Q<A85D;04?89324<=A0Q02I8=57:46A1311517M;501H=19600M>9947=];0B0G6394353;<=210?>721;606@>211I2[203080E=8147012506092\02FL4A2204>0GS5;A372N903C12231:D6B01K358101224<<60>K498011210E8F:3<54<:4639:N1F8G36C>740941;?0?85J33B2B@D8331j3:206H?@190U4@0LA208;0963=_:1I01J1547L261554L;6;0413K>M2I>;<976008>3434K179411IC6483=B585<8C5117:4111496;0?:@0277:2;251:D6881<7447;5132>4J7;33B0000117:A832=D72C247]96<73700053833872413450=607:1230>011817G30372904860=57;2F<51J25>832871<2A=014?3243c2:617205<>1SV77QD>4@54@654C25;714=502:@45143726:225:29<00021700C06127736J11B=0g25307G0344A=1J;0629=I<480300<61H>20386F;318G913<4:218>4618<133368<4=@0?<96802:967600200991744;100<4D13346>30?=127;30713010202G402?:?791=01730>2=62511021590H270121=B8[8641053@02=700::4044;2170705H506919162B1C3@294E?B5?I96534=@374<06596:17862485C?1B325CEA2OF34M61B8AN2:3:?2]N01?1>0H;32FF;32FD:1:B080632;32FF;32FF;618?;32FF;31859389;371596;19C102534>1167;32FF;32FJ53;32FF;32FF;32FF;32FF;31859389;:69210<>15:022;32FF;37?1A75:2147:6@?73678A50<4>69082574J90KB86339;21<0373C35;510=31:043522IH315:14D22178C62485:12?597B:I427251280T13441=2312634DB2;=<21_843>224J038C=0:>5>0052C70AB2<2UD7157430?3<2=762032F;32FF;32FF0FF;3>C5124662C4571>2F5H;32FF;32FF;32FF;32FF;32FF;311;3515;32F;32FF;32FF;32FF;32FF;3EG025:2;N29A=093V6<24?00<4883536=3973053;32FF;32FF;B8[8641053@023000B86;42243>06162063[0492:49090IC25B0?=656=>01P6520;2H1M03;32FF;32FF;32FF;3;32F?:C30100210@048>52:5267>34J2596:569H561C05E164@030977G46314:A2<2@011>:0=7D26C03>5619140=01<8@0003PU25C41D8;2972@054151;:?2=666>68D0C73011>:0=7D26C03>E2>;W0<52=02283;32FF;3624929K5?;0050;0413K>M2I>;<976008>3434K179411IC6483=B585<8C5117:4111496;0?:@0277:2;251:D6881<7447;5132>4J7;33B0000117:A832=D72C247]96<73700053833872413450=607:1230>011817G30372904860=57;2F<51J25>832871<2A=014?3243c2:617205<>1SV77QD>4@54@654C25;714=502:@45143726:225:29<00021700C06127736J11B=0g25307G0344A=1J;0629=I<480300<61H>20386F;318G913<4:218>4618<133368<4=@0?<96802:967600200991744;100<4D13346>30?=127;30713010202G402?:?791=01730>2=62511021590H270121=B8[8641053@02=700::4044;2170705H506919162B1C3@294E?B5?14<0;32FF;32F;;32FF;32F07:@`;32FF;32FF;0R661_06;32FF;32;6L068;6>895F;32FF;0:7?G65;32FF;32FF;32FF;32132VA@1:4<49176:4443g:G295L8020406;9NS<51;<8HF1>6A20<:71F?109154=31C411=1BI2F61303A=53189M654=;03519=;8151CQ6<900@457F2:3B5Q2<051H0;G;1B@42004218201238611220710003B40B7J2526U5C38634<24C2K612814<;3672;235103GG118371C:?22A510104@<1D587AA6708;9>8DB47FF;32FF;3215=0=61254402377841;:300435;438901:>361803D:6640D<:I1=@00462;65800<61H>20386F;97?512:1@P76616P84G03<65123FF;32;32FF;324304A<341567<2FF;3;316<0;8114118E0;8H@1B?2865:12?5947078:33GK:6O:570152320=C?62859:387586X3=:=;03>9121;C5424H@TL5>>1537O0D4D:145K125??326A3A550;32FF;32FF;32FF;3255B405F29222=:k10985841T419170J37324;32FF;32FF;32F06703112E2K51315510346=L=2==C5N1FE1790034L70402221P>7B=1?1Z9?12gH17;32FF;32FF;32FF;3:3623D37B==K6510252070:4901:M7202<21=32406A4I:14><15583:BG94=315E517<39B2H>IA032F3:2:476;32FF;32DE70H29B7:<:77253;07>>44;629=I<480300<61H>20386F;318G913<4:218>4618<133368<4=@0?<96802:967600200991744;100<4D13346>30?=127;30713010202G402?:?791=01730>2=62511021590H270121=B8[8641053@02=700::4044;2170705H506919162B1C3@294E?B5?I96534=@374<06596:17862485C?1B325CEA2OF34M61B8AN2:;4?0013H00E>41:8:;7C40@1A6A;_541A5F;32FF;B3D86=A5:30>A:180=04A1010683706062:005536C436?G650A<33:BO4;0117?=375<485=2V5673A2522A225K97?512:1@P76616P842340@03177K01V2<30B1?13>E554001?:4@<1D587AA6708;9>8D01<0M?1551A5402I03>11430715FEC1825A>?264=3011M<=C=35M531=547B031504334A05;;=6A00?85;32FF;32FF;32FF;32FF;32FF;32F;32FF;32FF;3210?7321181M0A45118C6E:6B340AHB587AA6708;32FF;32FF;3;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C10252222;32FF;3231?C8>671I81;32FF;32FF;32FF;3D=612<6527131F:H572;32F0D=?40<502E260N022<2455241313@02:B68223303112E2K51315510346;32F3623D37B==K6510258U3;780;734A?039079;:69210<>;512S39k8:67:96?HJ611:106052084:8>233>13221CF411820068058<;417?70=7<8041411211B3K15M3<816531C?D1CB632703D727337M001:835;54649451099=73:98;101:72;0200H]=D3;02963B01918;1172794I77127@874<31:156ZB<;06:A0122=2B=624==::?2180B;113I63751FD7L1028123@22BE<0GB393392741192?50BBm;62254141RE:H3=D1461143562O2>95451G45=1;5=;L21183366006<6Q=08;10=?591G6791B2307<4706Lc4=1>6F@=MI006H;19453B7005;8G9219=E338PI75L80200U0977;08844=E47?3?G913<4:218>40G500Y10549F31<24<0?6;32F<2590H27017401703989;52C;=5@0334;0502FF;323F2:>@;1H49=9M220;323C20OM80A18;:@3L3701;:F44<0;32FF;32F;;32FF;32F07:@`;32FF;32FF;0R661_06;32FF;32;6L068;6>895F;32FF;0:7?G65;32FF;32FF;32FF;32132VA@1:4<49176:4443g:G295L8020406;9NS<51;<8HF1>6A20<:71F?109154=31C411=1BI2F61303A=53189M654=;03519=;8151CQ6<900@457F2:3B5Q2<051H0;G;1B@42004218201238611220710003B40B7J2526U5C38634<24C2K612814<;3672;375<485=2V5>=9240;2A225K97?512:11DE70H2413034::12FF;31859389;:6;32FF;32FF;32FF;32F;32FF;32FF;3NF5H;32FF;32FF;32FF;32F;32FF;32FF;32FF;32F;32FF;32FF;32FF;32FFL4=;347;32FF;32FF;32FF;32FF;32FF;F;32;32F;32FF;32FF;32F17>88P2@32:;60707608243;<5:H95883ER088380]36J0075221189F15@1176:4443g:G295L8020406;9NS<51;<8HF1>6A20<:71F?109154=31C411=1BI2F61303A=53189M654=;03519=;8151CQ6<900@457F2:3B5Q2<051H0;G;1B@42004218201238611220710003B40B7J2526U5C38634<24C2K612814<;3672;230323GG118371C:?22A510104@<1D587AA6708;9>8DB47J27083;32FF;32FF;32FF;3;32FF61263088380]36;322P39:1124303;H1:10=949A?44334L79054=0BH;32FF;32FF;326156N2@9@1102515;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C10259YQ@IK34;50300C2M1@29A2185>14E7861=B8;RH5313EJ10<15G196?59C5O376166;@53080103133?25>0131=547B031504334A011104O7:0F6>330C2P041C62::3<01843590E5?2955=7D5;B8L05@5K<542DR12102D1?18a79=264E24H5348:040@A130294GG@3:072?1;328V52816652<;8B3;3OI70[1?3BD649311:>?E55F1T0311[68F48B433084B81003036740:0913A81B33L4A;3052834C;?522R>00:6BA708e24B==:42AN30?80F3:R412340210900892E363B8AN2:3:2FF;32FH;32FF;321;32FF;32FF;>0;32FF;32FF;;32FF;32FF;32159371596;19C102515;32FF;32F25A00630@061=J16;5A51321828:2376S6>17036>02271H0>>681X200@H0J5>1596730;=2:@<:112O;;67513611>760?5F<801@00:;<5:6E=689138E4B3;0565;01:;A@052OU10??@68:3;6:@=J:2@12308=6@4421H565142:D215086G;2440Z@?51856F39K9=3257E451D<<8A;C10933670Z17=R3@;><09e064=242:EV981893756089212@442<:84=3A3758310G2001:3C4;1H00;64F2=5112<1160=151007B:^55279;334<<032103C311<7P1;A4=7=82842338139127;3228DD1;@219040@0?P2<2=6>E?2BP435130334:6866M382944Q171>7D<9230?1:832FF<@18AABOS120620<;;I8B11213965365I>K^R0M5I7304619=21670764656B6I6I6099=076@?1740;32FF;389;:6921:557;02811A5I3096;<:132?;009>808>1<66462503511V02B137?14666732238611220710023?1C;2U?E932?2822FF;0:7;?105=;32FF;32FF;32FF;32FF;31859389;:6;32FF;32FF;32FF;32F;32FF;32FF;3NF5H;32FF;32FF;32FF;32F;32FF;32FF;32FF;32F;32E3012611;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0@=15@==7020:69210<5K569156104:8=250?567L1U082345809:47BG43602?L:>95=U0@112C75430;E212271<@202>7;8D0ED8841240=3294634BD59^1==0023111?84;005611128D5307290O;40:1<5@94V6@[:5=9BR180412802117142O400370602:7004@B10;;<=0CB8769769A?1H2[5:=6?62178>042:3454JF7586842:>2<900A3KX4JA6?1320640G?87::73B7004>6;0E0=0218313?0>7>296691145233<7;440J3:7B0K173<8773?1<120=01372?421140:2262:480940404A1;1A46C2770609=511125121;50P8345\27392N5115>932FF;32FF;32F17>88P2@32:;60707608243;<5:H95883ER088380]36J0075221189F15@1176:4443g:G295L8020406;9NS<51;<8HF1>6A20<:71F?109154=31C411=1BI2F61303A=53189M654=;03519=;8151CQ6<900@457F2:3B5Q2<051H0;G;1B@42004218201238611220710003B40B7J2526U5C38634<24C2K612814<;3672;230323GG118371C:?22A510104@<1D587AA6708;9>8DB47J27083;32FF;32FF;32FF;3;32FF61263088380]36;322P39:1124303;H1:10=949A?44334L79054=0BH;32FF;32FF;326156N2@9@1102530527736J11B=0g2530421=322:2=5?D59@@<=40475211:5006098<@1<7627E:0>235@8072M:4210C3124>:8@T=4T1>5E=57;5@0362>?376?5<3931833a?332FF;32EO113552;Q06@4@59?;62:<520031131<=6876640>D50249;6151;756AF:14:2113253B?0C02O3K11C224;;S00243?2347d8<:10H46T=212<511?624H3C025445220587AA6708;9>8D01<2FF;;32FF;2216J74<0083C02;206G0=2<93;51:0CVE55200257430G>9920C0:7005>1373<;194>=4H952=2=:9?4U:B112>KCI0E0>:10=5204A1517002=3:0M11?H;0A;195H0O1:5G1>IB0652104U55=L02M81A?09253@=5@F<:61160=@1E777:G<0982265044920;<8:4?F330]8674732G31=63]1D9015D73792:>332F;32FF4101O4210<>15;32F;32FF;32FF;32FF;32FF;3EG025:2;N29A=093V6<24?00<4883536=3973053;32FF;32FF;B8[8641053@023000B86;42243>06162063[0492:49090IC25B0?=656=>01P6520;2H1M03;32FF;32FF;32FF;3;32F?:C30100210@048>52:5267>34J2596:569H561C05E164@030977G46314:A2<2@0;0:]661=05Bb59413C3FF2E75>D1038484201_K43;8I@9186072940940E3<5D07H1=?20:71>5?:5[13F<=10=>1C255229732:3;32FF;3807D67357>39818C6=:C41;74360E2355907290211960=093V:4850>708M2;9G317129@1202<73Y14E57156?1103;:@1557B020E9:397412H288?83K:A=21:2T020320;>=61;:75<7>?4B8<846305:2205218929@A456:<4428<630;2AE2202310103<231267CFL0597GG@38<3Q?46160C0M453?770K4163653G1@23004643043<2;68203B57444<4K1C>QZ302J<97103>F25@87093011208062037B=0>L2:5>6;9J2032H:5:2K5;;72195<173860R90<391;876=3056823@;0632FFP15439::25182EO11355>34J2596:F;32FF;32F=:k10:5006098<@1<76279=I<480300<61H>20090242299a60F130782A29M6542:1@P76616P<511?624H373F>?891328554<011A5012;541=15@=9;:6;32FF;3225312DG02B0571>26P>]36J007522118G;1B@420042182012:F2;a@5570NN800943933026U5C383NF5H;32FF;32FF;3:4450>19=@;0E0=0218313?0>7>296691145233<7;440J3:7B0K173<8773?1<120=01372?421140:2262:480940404A1;1A46C2770609=511125121;50P8345\27392N5115>932FF;32FF;32F17>88P2@32:;60707608243;<5:H95883ER088380]36J0075221189F15@1176:4443g:G295L8020406;9NS<512M623C75D6?9F103F7F3X=0>472E60:19802<30:1B8AN2:3:C55FFJD;617<:3:282A5055:30>Y3GB;57K855g30:572FF;32FF;32FF;3213NC2=:19@105;3Q018=?32@9745;>LC?59F;0:7;?10@616413@2J4;1=:=?62F635206;6<9061>78421<1;:72:3Hd372=99;0I967516J213208041411211B3K15M3<816531C?D1CB632703D727337M001:835;54649451099=73:98;101:72;0200H]=D3;02963B01918;1172794I77127@874<31:156ZB<;06:A0122=2B=624==::?2180B;113I63751FD7L1028123@22BE<0GB393392741192?50BBm;62254141RE:H3=D1461143562O2>95451G45=1;5=;L21183366006<6Q=08;10=?591G6791B2307<4706Lc4=81X200@H0J5>15967:1@P76616P847;?105=;32FF;32FF;32FF;32FF;7A0<1:6258121>0;32FF;06266D0E0>:10=6;19C10255T42<:081:921368>0A3O22ACH0EU363;503505163;722062124@17014M61B8AN2:3:?2]N0101<340K0;21328E529>2GAI233C025445220587A1I30=F5;3B220=5=M;15::A0122=2B=624==::?2180B;113I63751FD7L1028123@22BE<0GB393392741192?50BBm;62254141RE:H3=D1461143562O2>95451G45=1;5=;L21183366006<6Q=08;10=?591G6791B2307<4706Lc4=1>6FC5424H@TL50062L526110985841T4132FF;32FF;318597;32FF;32F3063224:61=2645:?26;@<43>2<6:G7:B7C40211126:33026O:220UZdC:0]a7=0777G1B7?1=4F9>47;@;002738E06092437065<=0649D68\N16380921DE70H2413034::1<21A3IH3RG9=4;32F;32FF;::14:60;396844:93415@707:@<;=03>1>4843432B1H082:010C>4E63F1@719127667=48;?F15NC2=:19@105;32FF;32FF;32FF;3;32FF;32FF;32F24H=00082=42CG23;L:39601O4210<>15;32F;32FF;32FF;32FF;32FF;3E;:AA7;32FF;3;:6;32FF;32FF;37=A<:6007608243;<5:@4D0:4780040;9H10D1202<21=356824100P037T3><34319155=453A4025<7>?4B8<846300346:8;209A45;4E4757024802:3817>4>K0P=1210J6?2<72:314?0@75379C9@5<0166B55=U61C095I228J363I3BH;:]<@2R23:21207=5411A3<<1@6319022FK0?1145:248;2;4=6H4A380J31921B;@04@6417106@?9D125A:X510:17362:;1:41162267459M06703287?8E0000?C071596;19C?1<0<;044300503I003AP:745;>LC?59F203764HIF=<6G66495350232B473676002009917;756AF:14:21132FF2487;3=25299=3<41:8:;7C40@1A6A;_541A5F;32FF;B3D86=A5:30>A:180=04A1010683706062:00005611128101:72;0200H]=D=7?9430977G46314:A2<2@190144800038:1859386L1F4420:<5B6=>2=3<=A6A;_5340?6062:0?38120X15266433W5D1512F027492351G2@40167PIDD38804B5>263366D71D;5=A02?7G;9289>:?0710122@08K50406C76577L01?5;6611;:484148212276A2?1N27E2010299<2438265=32335;007=11651H641025JDF2>29E1182T4150>:4071D6>3>0A1@10938>1?8A>2606I?=70145C1;21?0;M1:[06814C65>;7;3=76?7121;792828105717<41F=k0101439082>11132C23:83?261517>32194C?A8070=B47125C3802C2A65023033286;32FF;32FF;;11:43;YD062L24H00286@05:2205218929@A457M:6>=@=5<7>?4B8<846302FF;32FF;310938>1?594@13<7328B?421<1;:AA7;32FF;32FF;32FF;32FF;31859389;:69210<>15;32FF;32FF;32FF;790<1:62526351;32FF@:6242731;31859389;:69210<>15;32FF;32FF;32FF;790<1:6258121>0;32FF;32FF;;32FF;32FF;32159371596;19C1025Q5;82@525458@24=0832>16;;0S@6605347=;7411508021T6>J1703<:10E8263P;0810784067;?1G235??A02H04855C39;2747>9<<664;251C011:4A75020>:;26B45>54417<<:21<1;;3I@?=21?7080011?6415@>170675032G90F31728==5710G30I4=0;137809L11I131G0JE1;2:D5B19P860B0>J420:15879588D310]4:32:62076=208107:865911=9245<;0T08M?44B=2D320>210<133134::0:>0604181956104:8=25733232420@87JA8320;3K?48:01732=706:B0;213>070F<5539171O2>2024E24520=00K=0G1GA3;5R0K350433?9160E68683=0A8:0;A03;D11G620N=9?4B9870246G83Bi95340443?OK5201=4D1013?213C33>2R<:112835@2=102037:3924;4<2I3737611N>590A57<:720=80E0A548005=C7<164?53X566@101A>445J542079I930:3E>10G8<<3110L1<;2132309;81HWT54EF1>1?75E611H133B63530<;32FF;32FF;32FF;3A<33:BO4;01144620T>>;13125??3264@310G60345?:<2B32203TC4<7018<5<10959EAL396846<25P2C1OBO123L110:73F>?89132855NM07LE9446J3A4722A7734898N38<5844?65N=221810836543K030;1:089<2;322P39:11;=0C;1076539^0<75?24404826F162C9I92<235286J<31441596;116?3025C79@3971023C04533F;32FF;3212F?:C30100RF76<37>8=386F;318752626:2020Q178A??;02[0?0141;A420@17061?1504;0:]661=05Bb5901>0>9:J02810H:04;32FF;32FF;35879U?8167T7;32FF;32AH373F>?89:H3407:>N263A5242V16<4938A61OHB12021?1822SA3C34D553:03962F24=?4033<0O8G3?C;R@3E2I36P:T472E083:001216503606479@@708M2;9G31@2<6353241P2@32:2124662C41?1A646811;7P796b>84:46=538<0?16572;9>NPBD24105158232;<3DYC3035=087:555_30811763E1F272<0:<@6412221531C8063=75H297=8346=5122752<202375943869157;7521@1A4@44195:?10V3A2058@D8T;7W65:]1IF6=;22783060@33F=209;Y4U758Z32?44Q=23666819A0;5@:C892<15:8741A0;CC54@2C?68>33016:E:1B<2700=59=?684E522:12D098@223>742R:701Y72@69;2823;4Q579;4206:58384081B<<7\=7>1929;6J@;728929S2H6A0;O7031;Q:73\D1A\92E0:632>=0@02;0H31@5625C5F;32FF;32FF;32FC;`<2E332;622897;8525945=:8O5021A453816978<6054C9C54666@?1030<30M031D986:5>01Q073:K04413565256M424>6131=1@13614=H3180:914A946;32B0:F20@D;32FF;32FF75E=?A111<03;:<9<321<6:8113056402H363696:B3677J:0092F;32FF;32FF;32FF45118C6E:6B340AH84750G2?>8E5705804?;270122794I77127@?C8>671I81;32FF;3238D29<623?2FF;7=322:2=5?D59@2:038=424:0@C28;5802159389;:69210<>1563I3BH;F5H;32FF;0502185691H10Q00S:13432HA56:K28070=B47125C3802C2A51M9>91141=9645;1445><72F48089D66<4><01H=S<04<0;7<11A8A51150148107LE9446J3A46<0121?48031620;D;32FF;32FF;318593966A;_541A5F;32FF;B3D86=A5:30>A:180=04A1010683706062:00005611128101:72;0200H]=D=7?9430977G46314:A2<2@190144800038:1859386L1F4420:<5B6=>2=3<=A6A;_5340?6062:0?38120X15266433W5D1512F027492351G2@40167PIDD38804B5>263366D77>88P2@32:;607076859389;:69210<>;32F802:1D17V5@G2783060@33F=232FF;32FF;32FF;7905A15=3@3;2D907@17014M61B8AN2:3:?233B2B@D8331j3:206H?@190U4@0LA208;0963=_:1I01J1547L261554L;6;0413K>M2I>;<976008>3434K179411IC6483=B585<8C5117:4111496;0?:@0277:2;251:D6881<7447;5132>4J7;33B0000117:A832=D72C247]96<73700053833872413450=607:1230>011817G30372904860=57;2F<51J25>832871<2A=014?3243c2:617205<>1SV77QD>4@54@654C25;714=502:@45143726:225:29<00021700C06127736J11B=0g25307G0344A=1J;0629=I<480300<61H>20386F;318G913<4:218>4618<133368<4=@0?<96802:967600200991744;100<4D13346>30?=127;30713010202G402?:?791=01730>2=62511021590H2701103;:2@;44=5M00K432148IF150J0A713320271D0?34200=@340<;2=424:0@26A3A550;3280LB:>86<6389C24342<11?070;062>4711::<2W?1<2:1650FH011330568864I72;6>J9;56233666:0906B10=47=812453?B8440M:91604<1:205:::204386>:074A8>4985I34D7@00805:4?>;08D39J0?73GM93CGK704D;9I7=308C553670B?53G;31531AA73412A012B;125E55607Hm6:31?372C45?3LA46;8:474J7407623:551?6E3?3>7N3<00576>09H1;60225S8175298:110A1S361:?1215221>68?23225:30547130<56G368CC6029009>;?604=254J2B30<:4C4C510203034><39120?111E7001<231=6405`525?720>2866I601043062N8C:=3975Z54B<0;5674;:>2D92G181423Ka03826;74068:493?6?13O84317G62333:6A34:93>800@>05621311A=@0364F4;0542;0;355B=0:4N617714I5A7<6;1430k7EQ7301>B0_;;0597973830?3258=02982I210<233:570152320=C?62859:387586X3=:=;03>9121;C5424H@TL5>>1537O0D4D:145K125??326A3A550;32FF;32FF;32FF;3255B405F29222=:k10985841T4191FF;32FF;3NF5H;32FF;08?;53344>930802159389;:6921045143F;32FF;32FF;32F0431253G500130H00286@1H<;1B6:180=04A1010683706062:005536C436?G650A<33:BO4;0117?=375<485=2V5673A2522A225K97?512:1@P76616P842340@03177K01V2<30B1?13>E554001?:4@<1D587AA6708;9>8D01<0M?1551A5402I03>11430715FEC1825A>?264=3011M<=C=35M531=547B031504334A05;;=6A00?85;32FF;32FF;32FF;32FF;32FF;32F;32FF;32FF;3210?73211861F812=12511C653;937D15B20D2413K24BE0:10041A4;41I8=01303F6L0=>7?KC;4H025U41;11K16A;70F52F720425:8627:714<:I0001@1B92056202@906>1:16J::25@;13:2?0127:P65424A55=M0CDC37:3A0696B0<41053=99900;@533<050941A5715^1P8K000201A4;7:\157B078C774386<4IE221F893111M84H532541E3H65818L01H1076061;A71732126101R098232Q8750665A878;D954F1619@<32K4W01<2<1>80E20970224:<2045391<81:4151107>00535@0=:52G14W13=94418??>3:10=5730N>05012W3B:46:72=30:0224FD65E574b;=ED:65;C04231@Q;A1981>@2@:035D5FD>;N476174823<>:324]36J0075221189F15@1176:4443g:G295L8020406;9NS<512M623C75D6?9F103F7F3X=0>472E60:19802<30:1B8AN2:3:C55FFJD;617<:3:282A5055:30>Y3GB;57K855g30:572FF;32FF;32FF;3213NC2=:19@110@53<37LO0D4D:143><34319155=45201<14F3299C6?49:M23S:68424C=5D2723>>9;B@<<715S540<603241GhC:620>52B011901>2B>;36009D6?9F103F>@16L2?>629FF;32F97?G65;32FF;23B;5205:5c32FF;32FF0=;19:3336956741I1:1<=f1>755:4134G4<21;0067><:I51E0EBN12O26=2I08O9;B116=77?0022@?1:1:CJ7<6=71>=2=77M37J14SC1I23I1156L0524404826132V>230025003262;34F;32FF;32FF;32FF;32FF;F;32;32F;32FF;32FF;32F17>88P2@32:;60707608243;<5:H95883ER088380]36J0075221189F15@1176:4443g:G295L8020406;9NS<51;<8HF1>6A20<:71F?109154=31C411=1BI2F61303A=53189M654=;03519=;8151CQ6<900@457F2:3BI5<599908=4339F10929E38]45<@@28?@D6<250P;1@71515>3262>100572<6>19173:003811502:5181;02832>025P2070A4?46J=0J535]B4P62>N0392401GF222574<5:000446442423=14J:0@0B659444A87;<3304@8980E:B00:D2CK0320R?J?6F06>A38DK297E4Q04:72:;62K90;184;875458840B0333G5H:I3:5147>64391542A62I62:78W0722N@095010<1771510=WT07012204210610391KD14172?6863C48D3>:12381A43151B47:D<32;5572410C?15GEA2B032B48P4932J12;>213:A21777031C05950G4H:4G162>:20600=11602;23273G531KC=J692685Q5244=J5J17NI0008<4=8268040703:4NCB118;020829>05=54;1S82A440241=5644493:47:D55>:7:0H30007122033J30A3A514071D068531326;KF08159414601022:57D0156;J8;3;D28228462C5@1:1=:210;283412FB1C236_<12416>686@0@512564705N3A0H0<7?63603D9166213081541;712B6219<6B5:353:166?=;4T2?11O<5311;:0=0>0017M3<;:?15C6731>01?H78;=?5>1>20G2621@6063G543CA4249>0?6Dc94140417<5943;0H3C173:375320C0857097=43\BM;11?:=B34;222034<020085913A0;1005@94AF599F5DX86182G<1:@5631\<62795<1411A=4>5C94P13<1>612@J3:><4979194B0=A:7B:2;0160725L7;:355355;UL70F4551371L211238SH<33=72;5481D63=27:3C04092@P62C3800L0:0:7232012U57144272BD:3574032450::=;1:=212BZE360272760;A65;2?Y@456C0F03532=4507=1<82H=L71G3@8:3h54=452031?734J1:4210H5119]@1301741>A:7::402A:410<>1SV77QD>4893243>06162063[0492:49090IC25970192B5;3B220=5=M;15:5A51c0452K5F12>61?1504;0:]661=05Bb5901>0>9:J02810H:04=4<:K2?2420;50@732>839853J<38:@137>D19720470709<08H0J2_702A6;@7=96;19C1025?4;A7C690600C11370=2231600J==27<3020027270F10053751<<:;1605742<346?4?81036615:134<:78I:103NP32210533436;650:846?2S0E=CA31517?33A2:JN02;2=32BC170600DA611LE48610903A103>B04671110;38@E256F4?6=1>B3?01:10>6010>5:=78<;0@011792622C0037A3BG364A2>9358063;><30EQ24<<08<5E46806S08@0B3472G23=1186CP93j?17D81726KL20<=0530=2:5103D20@322HJJ9662073>826?>0355162:8F0>B<44:05E:<200EX04302404F<1>612@J3:><4979194B0=A:7B:2;0160725L7;:355355;UL70F4551371L211238SH<33=72;5481D63=27:3C04092@P62C3800L0:0:127;3071301327542N256083B2<46J;J>@24111020?174I2I25:0O52451B107192051:10130T0754251M10>90C10H8Y0:1;0:0J>2:84662O96L914524I500>316509;75@A1K0\GE69@D2262526U5C386114C0?8?B1E1141<38?45302P2@32YQ@IK34;50300C2M1@;32FF;32FF;32F7480083B:3>Q560>45A07@322E>2005536C436?G625307G034B;53E@2K09:;2278306005E:<200EX04VA@1:45T847;?10505L4607098H43>2D1<9932FF;;<30E3=14656254B37C3H8P0040;32FF;AF;32FF;32FF;32F;32FF;32FF;32FF;;<30E3=14656254B37C3H8P0040;32FF;AF;32FF;32FF;32F;32FF;32FF;32FF;;<30E3=14656254B37C3H8P0040;32FF;AF;32FF;32FF;32F;32FF;32FF;32FF;;<30E3=14656254B37C3H8P0040;32FF;AF;32FF;32FF;32F;32FF;32FF;32FF;;<30E3=14656254B37C3H8P0040;32FF;AF;32FF;32=307<40>10H8Y0:1;0:0J>R?>1216>5909E>=745=01OG67003703161D274744712MC159E708?I4300860@2757?2B10720510071940e<:0KM155=7142BE27<1826J331dE09?Q0B31033>242<1470709<08H0J2496;0?:@0278>1?=515229784;37<@38C=0:>5>0052C700@36928190701B91?45Q<A8216762450307550;37621953975@8=277:1>>05H13312930:133k=0332324=>19;4374162K1602533710181C3P3212\8:O4054101331:823542H536D860641191111=3393G703303537;F9954U:0237=;4A10665I3B9>D19287A:93FM?40;13E?061053262>@0A85;B748;8150080>44;4D=97@0003601717=0;007B501:6=K:;0>=14712232F87;17A2835;:9021041G6L2217Q3=26<2056567@27597:<1<3227133341:84:20=59:B0014FA92351<;43K6;C5193642717P77U6042C475>;=A3@?2N;13=L0A1<:003902;:47JA0<0333T295660=0;0=03659250212:2?E10I1:4714200500>120253864:52QS@34<4090@15;32??2;G22>2=00;@58:110860123861E212271<8F:01B;658=01161@29E4D068531326;KF08Z9?12gH174E048641053@02=700::4010<>15;32FF;32F87F282Q07>1563I3BH;F5H;32FF;0502185691H10Q00S:13432HA56:K28070=B47125C3802C2A51M9>91141=9645;1445><72F48089D66<4><01H=S<04<0;7<11A8A51150148107LE9446J3A46<0121?48031620;D;32FF;32FF;318593966A;_541A5F;32FF;B3D86=A5:30>A:180=04A1010683706062:00005611128101:72;0200H]=D=7?9430977G46314:A2<2@190144800038:1859386L1F4420:<5B6=>2=3<=A6A;_5340?6062:0?38120X15266433W5D1512F027492351G2@40167PIDD38804B5>263366D77>88P2@32:;607076859389;:69210<>;32F802:1D17V5@G2783060@33F=232L79054=0BH;25L7;:355355;UL33?c0452F3623D37B==K651025FF;3NF5H;32FF;F?109154=31C41C6:1;<:31B453;B1211>22144;609013335787;;25143B3a124>6D75411033?5010281C034480276DA=M903E795577F820<<21A=:17A9571A90X09004742?5>9=8MI3@1@@D1176022420B;>244K507076010115328::6669413631?7I14912E773221261
\ No newline at end of file

From d0a3f2569755deb1fa8dad7b73d5c7bc7c46980a Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Thu, 28 Feb 2019 01:52:01 -0800
Subject: [PATCH 078/178] change return type to ULL

---
 doc/zstd_manual.html             | 10 +++++-----
 lib/decompress/zstd_decompress.c |  4 ++--
 lib/zstd.h                       | 10 +++++-----
 tests/fuzzer.c                   |  9 +--------
 4 files changed, 13 insertions(+), 20 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index a11a9589229..426c6cf2436 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -127,11 +127,11 @@ 
zstd 1.3.8 Manual

 


 
 
Helper functions
#define ZSTD_COMPRESSBOUND(srcSize)   ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
-size_t      ZSTD_compressBound(size_t srcSize);                    /*!< maximum compressed size in worst case single-pass scenario */
-size_t      ZSTD_decompressBound(const void* src, size_t srcSice); /*!< maximum decompressed size of the compressed source */
-unsigned    ZSTD_isError(size_t code);                             /*!< tells if a `size_t` function result is an error code */
-const char* ZSTD_getErrorName(size_t code);                        /*!< provides readable string from an error code */
-int         ZSTD_maxCLevel(void);                                  /*!< maximum compression level available */
+size_t             ZSTD_compressBound(size_t srcSize);                    /*!< maximum compressed size in worst case single-pass scenario */
+unsigned long long ZSTD_decompressBound(const void* src, size_t srcSice); /*!< maximum decompressed size of the compressed source */
+unsigned           ZSTD_isError(size_t code);                             /*!< tells if a `size_t` function result is an error code */
+const char*        ZSTD_getErrorName(size_t code);                        /*!< provides readable string from an error code */
+int                ZSTD_maxCLevel(void);                                  /*!< maximum compression level available */
 


 
Explicit context

 
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index a1f656ea4c3..6d56b08c845 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -503,9 +503,9 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
  *  `srcSize` must be at least as large as the frame contained
  *  @return : maximum decompressed size of the compressed source
  */
-size_t ZSTD_decompressBound(const void* src, size_t srcSize)
+unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize)
 {
-    size_t totalDstSize = 0;
+    unsigned long long totalDstSize = 0;
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
     if (ZSTD_isLegacy(src, srcSize))
         return ERROR(version_unsupported);
diff --git a/lib/zstd.h b/lib/zstd.h
index 31341101773..4d764680e98 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -148,11 +148,11 @@ ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t
 
 /*======  Helper functions  ======*/
 #define ZSTD_COMPRESSBOUND(srcSize)   ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
-ZSTDLIB_API size_t      ZSTD_compressBound(size_t srcSize);                    /*!< maximum compressed size in worst case single-pass scenario */
-ZSTDLIB_API size_t      ZSTD_decompressBound(const void* src, size_t srcSice); /*!< maximum decompressed size of the compressed source */
-ZSTDLIB_API unsigned    ZSTD_isError(size_t code);                             /*!< tells if a `size_t` function result is an error code */
-ZSTDLIB_API const char* ZSTD_getErrorName(size_t code);                        /*!< provides readable string from an error code */
-ZSTDLIB_API int         ZSTD_maxCLevel(void);                                  /*!< maximum compression level available */
+ZSTDLIB_API size_t             ZSTD_compressBound(size_t srcSize);                    /*!< maximum compressed size in worst case single-pass scenario */
+ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSice); /*!< maximum decompressed size of the compressed source */
+ZSTDLIB_API unsigned           ZSTD_isError(size_t code);                             /*!< tells if a `size_t` function result is an error code */
+ZSTDLIB_API const char*        ZSTD_getErrorName(size_t code);                        /*!< provides readable string from an error code */
+ZSTDLIB_API int                ZSTD_maxCLevel(void);                                  /*!< maximum compression level available */
 
 
 /***************************************
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 00e7f5442a8..c64fff88390 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -378,14 +378,7 @@ static int basicUnitTests(U32 seed, double compressibility)
 
     DISPLAYLEVEL(3, "test%3i : tight ZSTD_decompressBound test : ", testNb++);
     {
-        size_t bound = ZSTD_decompressBound(compressedBuffer, cSize);
-        if (bound != CNBuffSize) goto _output_error;
-    }
-    DISPLAYLEVEL(3, "OK \n");
-
-    DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test missing Frame_Content_Size : ", testNb++);
-    {
-        size_t bound = ZSTD_decompressBound(compressedBuffer, cSize);
+        unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize);
         if (bound != CNBuffSize) goto _output_error;
     }
     DISPLAYLEVEL(3, "OK \n");

From 515c506b4c7dabb83ea8e4b2edfa709cdf647279 Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Thu, 28 Feb 2019 02:10:17 -0800
Subject: [PATCH 079/178] switch frameBound type to ULL

---
 lib/decompress/zstd_decompress.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 6d56b08c845..d2916d7ef22 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -529,7 +529,7 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize)
         }
 
         {   size_t bound;
-            size_t frameBound;
+            unsigned long long frameBound;
             size_t frameSrcSize;
             unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
             if (ret == ZSTD_CONTENTSIZE_ERROR) return ret;

From dce9a09772887af5a4a0e64564b15308a21c3576 Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Thu, 28 Feb 2019 03:01:21 -0800
Subject: [PATCH 080/178] initialize local vars in decompressBound

---
 lib/decompress/zstd_decompress.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index d2916d7ef22..b7eccdfe899 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -528,9 +528,9 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize)
             continue;
         }
 
-        {   size_t bound;
-            unsigned long long frameBound;
-            size_t frameSrcSize;
+        {   size_t bound = 0;
+            unsigned long long frameBound = 0;
+            size_t frameSrcSize = 0;
             unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
             if (ret == ZSTD_CONTENTSIZE_ERROR) return ret;
 

From 8930c3c79ba725311c72b7c7f33b630118f7f9cb Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Thu, 28 Feb 2019 22:55:18 -0800
Subject: [PATCH 081/178] implement API-level changes

---
 doc/zstd_manual.html             |   9 +-
 lib/decompress/zstd_decompress.c | 146 ++++++++++++++++---------------
 lib/zstd.h                       |  24 +++--
 tests/fuzzer.c                   |   8 +-
 4 files changed, 104 insertions(+), 83 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 426c6cf2436..c7962e7de0c 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -127,11 +127,10 @@ 
zstd 1.3.8 Manual

 


 
 
Helper functions
#define ZSTD_COMPRESSBOUND(srcSize)   ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
-size_t             ZSTD_compressBound(size_t srcSize);                    /*!< maximum compressed size in worst case single-pass scenario */
-unsigned long long ZSTD_decompressBound(const void* src, size_t srcSice); /*!< maximum decompressed size of the compressed source */
-unsigned           ZSTD_isError(size_t code);                             /*!< tells if a `size_t` function result is an error code */
-const char*        ZSTD_getErrorName(size_t code);                        /*!< provides readable string from an error code */
-int                ZSTD_maxCLevel(void);                                  /*!< maximum compression level available */
+size_t      ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
+unsigned    ZSTD_isError(size_t code);          /*!< tells if a `size_t` function result is an error code */
+const char* ZSTD_getErrorName(size_t code);     /*!< provides readable string from an error code */
+int         ZSTD_maxCLevel(void);               /*!< maximum compression level available */
 


 
Explicit context

 
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index b7eccdfe899..4b9402d6957 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -433,40 +433,73 @@ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t he
     return 0;
 }
 
+/**
+ * Contains the compressed frame size and an upper-bound for the decompressed frame size.
+ * Note: before using `compressedSize` or `decompressedBound`,
+ *       you must check the field for errors using ZSTD_isError().
+ */
+typedef struct {
+    size_t compressedSize;
+    size_t decompressedBound;
+} ZSTD_frameSizeInfo;
 
-static size_t ZSTD_findFrameCompressedSize_internal(const void *src, size_t srcSize, size_t* bound)
+static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize)
 {
+    ZSTD_frameSizeInfo frameSizeInfo;
+    memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo));
+
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
-    if (ZSTD_isLegacy(src, srcSize))
-        return ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
+    if (ZSTD_isLegacy(src, srcSize)) {
+        frameSizeInfo.compressedSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
+        frameSizeInfo.decompressedBound = ERROR(version_unsupported);
+        return frameSizeInfo;
+    }
 #endif
-    if ( (srcSize >= ZSTD_SKIPPABLEHEADERSIZE)
-      && (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START ) {
-        return readSkippableFrameSize(src, srcSize);
+
+    if ((srcSize >= ZSTD_SKIPPABLEHEADERSIZE)
+        && (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
+        frameSizeInfo.compressedSize = readSkippableFrameSize(src, srcSize);
+        return frameSizeInfo;
     } else {
         const BYTE* ip = (const BYTE*)src;
         const BYTE* const ipstart = ip;
         size_t remainingSize = srcSize;
-        ZSTD_frameHeader zfh;
         unsigned nbBlocks = 0;
+        ZSTD_frameHeader zfh;
 
         /* Extract Frame Header */
-        {   size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize);
-            if (ZSTD_isError(ret)) return ret;
-            RETURN_ERROR_IF(ret > 0, srcSize_wrong);
+        {
+            size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize);
+            if (ZSTD_isError(ret)) {
+                frameSizeInfo.compressedSize = ret;
+                frameSizeInfo.decompressedBound = ret;
+                return frameSizeInfo;
+            }
+            if (ret > 0) {
+                frameSizeInfo.compressedSize = ERROR(srcSize_wrong);
+                frameSizeInfo.decompressedBound = ERROR(srcSize_wrong);
+                return frameSizeInfo;
+            }
         }
 
         ip += zfh.headerSize;
         remainingSize -= zfh.headerSize;
 
-        /* Loop on each block */
+        /* Iterate over each block */
         while (1) {
             blockProperties_t blockProperties;
             size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
-            if (ZSTD_isError(cBlockSize)) return cBlockSize;
+            if (ZSTD_isError(cBlockSize)) {
+                frameSizeInfo.compressedSize = cBlockSize;
+                frameSizeInfo.decompressedBound = cBlockSize;
+                return frameSizeInfo;
+            }
 
-            RETURN_ERROR_IF(ZSTD_blockHeaderSize + cBlockSize > remainingSize,
-                            srcSize_wrong);
+            if (ZSTD_blockHeaderSize + cBlockSize > remainingSize) {
+                frameSizeInfo.compressedSize = ERROR(srcSize_wrong);
+                frameSizeInfo.decompressedBound = ERROR(srcSize_wrong);
+                return frameSizeInfo;
+            }
 
             ip += ZSTD_blockHeaderSize + cBlockSize;
             remainingSize -= ZSTD_blockHeaderSize + cBlockSize;
@@ -475,14 +508,21 @@ static size_t ZSTD_findFrameCompressedSize_internal(const void *src, size_t srcS
             if (blockProperties.lastBlock) break;
         }
 
-        if (zfh.checksumFlag) {   /* Final frame content checksum */
-            RETURN_ERROR_IF(remainingSize < 4, srcSize_wrong);
+        /* Final frame content checksum */
+        if (zfh.checksumFlag) {
+            if (remainingSize < 4) {
+                frameSizeInfo.compressedSize = ERROR(srcSize_wrong);
+                frameSizeInfo.decompressedBound = ERROR(srcSize_wrong);
+                return frameSizeInfo;
+            }
             ip += 4;
         }
 
-        if (bound != NULL) *bound = (nbBlocks * zfh.blockSizeMax); /* set to block-based bound */
-
-        return ip - ipstart;
+        frameSizeInfo.compressedSize = ip - ipstart;
+        frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN)
+                                        ? zfh.frameContentSize
+                                        : nbBlocks * zfh.blockSizeMax;
+        return frameSizeInfo;
     }
 }
 
@@ -493,7 +533,8 @@ static size_t ZSTD_findFrameCompressedSize_internal(const void *src, size_t srcS
  *  @return : the compressed size of the frame starting at `src` */
 size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
 {
-    return ZSTD_findFrameCompressedSize_internal(src, srcSize, NULL);
+    ZSTD_frameSizeInfo frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize);
+    return frameSizeInfo.compressedSize;
 }
 
 
@@ -502,56 +543,23 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
  *  `src` must point to the start of a ZSTD frame or a skippeable frame
  *  `srcSize` must be at least as large as the frame contained
  *  @return : maximum decompressed size of the compressed source
+ *            or an error code which can be tested with ZSTD_isError()
  */
-unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize)
-{
-    unsigned long long totalDstSize = 0;
-#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
-    if (ZSTD_isLegacy(src, srcSize))
-        return ERROR(version_unsupported);
-#endif
-
-    /* Loop over each frame */
-    while (srcSize >= ZSTD_FRAMEHEADERSIZE_PREFIX) {
-        U32 const magicNumber = MEM_readLE32(src);
-
-        if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
-            size_t const skippableSize = readSkippableFrameSize(src, srcSize);
-            if (ZSTD_isError(skippableSize))
-                return skippableSize;
-            if (srcSize < skippableSize) {
-                return ZSTD_CONTENTSIZE_ERROR;
-            }
-
-            src = (const BYTE *)src + skippableSize;
-            srcSize -= skippableSize;
-            continue;
-        }
-
-        {   size_t bound = 0;
-            unsigned long long frameBound = 0;
-            size_t frameSrcSize = 0;
-            unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
-            if (ret == ZSTD_CONTENTSIZE_ERROR) return ret;
-
-            frameSrcSize = ZSTD_findFrameCompressedSize_internal(src, srcSize, &bound);
-            if (ZSTD_isError(frameSrcSize)) {
-                return ZSTD_CONTENTSIZE_ERROR;
-            }
-
-            frameBound = (ret == ZSTD_CONTENTSIZE_UNKNOWN) ? bound : ret;
-            /* check for overflow */
-            if (totalDstSize + frameBound < totalDstSize) return ZSTD_CONTENTSIZE_ERROR;
-            totalDstSize += frameBound;
-
-            src = (const BYTE *)src + frameSrcSize;
-            srcSize -= frameSrcSize;
-        }
-    }  /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */
-
-    if (srcSize) return ZSTD_CONTENTSIZE_ERROR;
-
-    return totalDstSize;
+size_t ZSTD_decompressBound(const void* src, size_t srcSize)
+{
+    size_t bound = 0;
+    /* Iterate over each frame */
+    while (srcSize > 0) {
+        ZSTD_frameSizeInfo frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize);
+        size_t compressedSize = frameSizeInfo.compressedSize;
+        size_t decompressedBound = frameSizeInfo.decompressedBound;
+        FORWARD_IF_ERROR(compressedSize);
+        FORWARD_IF_ERROR(decompressedBound);
+        src = (const BYTE*)src + compressedSize;
+        srcSize -= compressedSize;
+        bound += decompressedBound;
+    }
+    return bound;
 }
 
 
diff --git a/lib/zstd.h b/lib/zstd.h
index 4d764680e98..6d457cfdc6c 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -148,11 +148,10 @@ ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t
 
 /*======  Helper functions  ======*/
 #define ZSTD_COMPRESSBOUND(srcSize)   ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
-ZSTDLIB_API size_t             ZSTD_compressBound(size_t srcSize);                    /*!< maximum compressed size in worst case single-pass scenario */
-ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSice); /*!< maximum decompressed size of the compressed source */
-ZSTDLIB_API unsigned           ZSTD_isError(size_t code);                             /*!< tells if a `size_t` function result is an error code */
-ZSTDLIB_API const char*        ZSTD_getErrorName(size_t code);                        /*!< provides readable string from an error code */
-ZSTDLIB_API int                ZSTD_maxCLevel(void);                                  /*!< maximum compression level available */
+ZSTDLIB_API size_t      ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
+ZSTDLIB_API unsigned    ZSTD_isError(size_t code);          /*!< tells if a `size_t` function result is an error code */
+ZSTDLIB_API const char* ZSTD_getErrorName(size_t code);     /*!< provides readable string from an error code */
+ZSTDLIB_API int         ZSTD_maxCLevel(void);               /*!< maximum compression level available */
 
 
 /***************************************
@@ -1104,6 +1103,21 @@ typedef enum {
  *            however it does mean that all frame data must be present and valid. */
 ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
 
+/** ZSTD_decompressBound() :
+ *  currently incompatible with legacy mode
+ *  `src` must point to the start of a ZSTD frame or a skippeable frame
+ *  `srcSize` must be at least as large as the frame contained
+ *  @return : maximum decompressed size of the compressed source
+ *            or an error code which can be tested with ZSTD_isError()
+ *
+ *  note 1  : the bound is exact when Frame_Content_Size field is available in EVERY frame of `src`.
+ *  note 2  : when Frame_Content_Size isn't provided, the upper-bound for that frame is calculated by:
+ *                             upper-bound = min(128 KB, Window_Size)
+ *  note 3  : we always use Frame_Content_Size to bound the decompressed frame size if it's present.
+ **           the above formula is only used when Frame_Content_Size is missing.
+ */
+ZSTDLIB_API size_t ZSTD_decompressBound(const void* src, size_t srcSice);
+
 /*! ZSTD_frameHeaderSize() :
  *  srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX.
  * @return : size of the Frame Header,
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index c64fff88390..2ac1f0f4732 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -378,8 +378,8 @@ static int basicUnitTests(U32 seed, double compressibility)
 
     DISPLAYLEVEL(3, "test%3i : tight ZSTD_decompressBound test : ", testNb++);
     {
-        unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize);
-        if (bound != CNBuffSize) goto _output_error;
+        size_t rSize = ZSTD_decompressBound(compressedBuffer, cSize);
+        if (rSize != CNBuffSize) goto _output_error;
     }
     DISPLAYLEVEL(3, "OK \n");
 
@@ -909,8 +909,8 @@ static int basicUnitTests(U32 seed, double compressibility)
     DISPLAYLEVEL(3, "OK \n");
 
     DISPLAYLEVEL(3, "test%3i : get tight decompressed bound of multiple frames : ", testNb++);
-    {   unsigned long long const r = ZSTD_decompressBound(compressedBuffer, cSize);
-        if (r != CNBuffSize / 2) goto _output_error; }
+    {   size_t const rSize = ZSTD_decompressBound(compressedBuffer, cSize);
+        if (rSize != CNBuffSize / 2) goto _output_error; }
     DISPLAYLEVEL(3, "OK \n");
 
     DISPLAYLEVEL(3, "test%3i : decompress multiple frames : ", testNb++);

From 03026c3b1d34e7220b42cbb33855dd213939ac5d Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Fri, 1 Mar 2019 00:03:50 -0800
Subject: [PATCH 082/178] change compressedBound to ULL

---
 lib/decompress/zstd_decompress.c | 31 ++++++++++++++++---------------
 lib/zstd.h                       | 16 ++++++++--------
 tests/fuzzer.c                   |  6 +++---
 3 files changed, 27 insertions(+), 26 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 4b9402d6957..630291f08ae 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -435,12 +435,13 @@ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t he
 
 /**
  * Contains the compressed frame size and an upper-bound for the decompressed frame size.
- * Note: before using `compressedSize` or `decompressedBound`,
- *       you must check the field for errors using ZSTD_isError().
+ * Note: before using `compressedSize` you must check for errors using ZSTD_isError().
+ *       similarly, before using `decompressedBound`, you must check for errors using:
+ *          `decompressedBound` != ZSTD_CONTENTSIZE_UNKNOWN
  */
 typedef struct {
     size_t compressedSize;
-    size_t decompressedBound;
+    unsigned long long decompressedBound;
 } ZSTD_frameSizeInfo;
 
 static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize)
@@ -451,7 +452,7 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
     if (ZSTD_isLegacy(src, srcSize)) {
         frameSizeInfo.compressedSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
-        frameSizeInfo.decompressedBound = ERROR(version_unsupported);
+        frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
         return frameSizeInfo;
     }
 #endif
@@ -477,7 +478,7 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize
             }
             if (ret > 0) {
                 frameSizeInfo.compressedSize = ERROR(srcSize_wrong);
-                frameSizeInfo.decompressedBound = ERROR(srcSize_wrong);
+                frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
                 return frameSizeInfo;
             }
         }
@@ -491,13 +492,13 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize
             size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
             if (ZSTD_isError(cBlockSize)) {
                 frameSizeInfo.compressedSize = cBlockSize;
-                frameSizeInfo.decompressedBound = cBlockSize;
+                frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
                 return frameSizeInfo;
             }
 
             if (ZSTD_blockHeaderSize + cBlockSize > remainingSize) {
                 frameSizeInfo.compressedSize = ERROR(srcSize_wrong);
-                frameSizeInfo.decompressedBound = ERROR(srcSize_wrong);
+                frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
                 return frameSizeInfo;
             }
 
@@ -512,7 +513,7 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize
         if (zfh.checksumFlag) {
             if (remainingSize < 4) {
                 frameSizeInfo.compressedSize = ERROR(srcSize_wrong);
-                frameSizeInfo.decompressedBound = ERROR(srcSize_wrong);
+                frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
                 return frameSizeInfo;
             }
             ip += 4;
@@ -542,19 +543,19 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
  *  currently incompatible with legacy mode
  *  `src` must point to the start of a ZSTD frame or a skippeable frame
  *  `srcSize` must be at least as large as the frame contained
- *  @return : maximum decompressed size of the compressed source
- *            or an error code which can be tested with ZSTD_isError()
+ *  @return : the maximum decompressed size of the compressed source
  */
-size_t ZSTD_decompressBound(const void* src, size_t srcSize)
+unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize)
 {
-    size_t bound = 0;
+    unsigned long long bound = 0;
     /* Iterate over each frame */
     while (srcSize > 0) {
         ZSTD_frameSizeInfo frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize);
         size_t compressedSize = frameSizeInfo.compressedSize;
-        size_t decompressedBound = frameSizeInfo.decompressedBound;
-        FORWARD_IF_ERROR(compressedSize);
-        FORWARD_IF_ERROR(decompressedBound);
+        unsigned long long decompressedBound = frameSizeInfo.decompressedBound;
+        if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) {
+            return ZSTD_CONTENTSIZE_ERROR;
+        }
         src = (const BYTE*)src + compressedSize;
         srcSize -= compressedSize;
         bound += decompressedBound;
diff --git a/lib/zstd.h b/lib/zstd.h
index 6d457cfdc6c..e49a4048a06 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -1107,16 +1107,16 @@ ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t
  *  currently incompatible with legacy mode
  *  `src` must point to the start of a ZSTD frame or a skippeable frame
  *  `srcSize` must be at least as large as the frame contained
- *  @return : maximum decompressed size of the compressed source
- *            or an error code which can be tested with ZSTD_isError()
+ *  @return : - the maximum decompressed size of the compressed source
+ *            - if an error occured: ZSTD_CONTENTSIZE_ERROR
  *
- *  note 1  : the bound is exact when Frame_Content_Size field is available in EVERY frame of `src`.
- *  note 2  : when Frame_Content_Size isn't provided, the upper-bound for that frame is calculated by:
- *                             upper-bound = min(128 KB, Window_Size)
- *  note 3  : we always use Frame_Content_Size to bound the decompressed frame size if it's present.
- **           the above formula is only used when Frame_Content_Size is missing.
+ *  note 1  : an error can occur if `src` points to a legacy frame or an invalid/incorrectly formatted frame.
+ *  note 2  : the bound is exact when Frame_Content_Size field is available in EVERY frame of `src`.
+ *  note 3  : when Frame_Content_Size isn't provided, the upper-bound for that frame is calculated by:
+ *              upper-bound = min(128 KB, Window_Size)
+ *  note 4  : we always use Frame_Content_Size to bound the decompressed frame size if it's present.
  */
-ZSTDLIB_API size_t ZSTD_decompressBound(const void* src, size_t srcSice);
+ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSice);
 
 /*! ZSTD_frameHeaderSize() :
  *  srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX.
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 2ac1f0f4732..9efa232a90d 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -378,7 +378,7 @@ static int basicUnitTests(U32 seed, double compressibility)
 
     DISPLAYLEVEL(3, "test%3i : tight ZSTD_decompressBound test : ", testNb++);
     {
-        size_t rSize = ZSTD_decompressBound(compressedBuffer, cSize);
+        unsigned long long rSize = ZSTD_decompressBound(compressedBuffer, cSize);
         if (rSize != CNBuffSize) goto _output_error;
     }
     DISPLAYLEVEL(3, "OK \n");
@@ -909,8 +909,8 @@ static int basicUnitTests(U32 seed, double compressibility)
     DISPLAYLEVEL(3, "OK \n");
 
     DISPLAYLEVEL(3, "test%3i : get tight decompressed bound of multiple frames : ", testNb++);
-    {   size_t const rSize = ZSTD_decompressBound(compressedBuffer, cSize);
-        if (rSize != CNBuffSize / 2) goto _output_error; }
+    {   unsigned long long const r = ZSTD_decompressBound(compressedBuffer, cSize);
+        if (r != CNBuffSize / 2) goto _output_error; }
     DISPLAYLEVEL(3, "OK \n");
 
     DISPLAYLEVEL(3, "test%3i : decompress multiple frames : ", testNb++);

From 44ae395b3e5241d3edf320a0ac4063ecb311aee4 Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Fri, 1 Mar 2019 00:05:59 -0800
Subject: [PATCH 083/178] change nbBlocks to size_t for consistency

---
 lib/decompress/zstd_decompress.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 630291f08ae..b2f093fedc9 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -465,7 +465,7 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize
         const BYTE* ip = (const BYTE*)src;
         const BYTE* const ipstart = ip;
         size_t remainingSize = srcSize;
-        unsigned nbBlocks = 0;
+        size_t nbBlocks = 0;
         ZSTD_frameHeader zfh;
 
         /* Extract Frame Header */

From 2bb5eec7115d06a36d671302aa65b51864c8607f Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Fri, 1 Mar 2019 00:12:16 -0800
Subject: [PATCH 084/178] update missing error case to CONTENTSIZE_ERROR

---
 lib/decompress/zstd_decompress.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index b2f093fedc9..28f88cde8d8 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -473,7 +473,7 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize
             size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize);
             if (ZSTD_isError(ret)) {
                 frameSizeInfo.compressedSize = ret;
-                frameSizeInfo.decompressedBound = ret;
+                frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
                 return frameSizeInfo;
             }
             if (ret > 0) {

From 1e08c49f757955db19e199892f4de34bf227b2a9 Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Fri, 1 Mar 2019 18:29:35 -0800
Subject: [PATCH 085/178] add stylistic changes

---
 lib/decompress/zstd_decompress.c | 58 +++++++++++++-------------------
 lib/zstd.h                       | 10 +++---
 2 files changed, 29 insertions(+), 39 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 28f88cde8d8..254e2923dc1 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -444,17 +444,22 @@ typedef struct {
     unsigned long long decompressedBound;
 } ZSTD_frameSizeInfo;
 
+static ZSTD_frameSizeInfo ZSTD_errorFrameSizeInfo(size_t ret)
+{
+    ZSTD_frameSizeInfo frameSizeInfo;
+    frameSizeInfo.compressedSize = ret;
+    frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_UNKNOWN;
+    return frameSizeInfo;
+}
+
 static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize)
 {
     ZSTD_frameSizeInfo frameSizeInfo;
     memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo));
 
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
-    if (ZSTD_isLegacy(src, srcSize)) {
-        frameSizeInfo.compressedSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
-        frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
-        return frameSizeInfo;
-    }
+    if (ZSTD_isLegacy(src, srcSize))
+        return ZSTD_errorFrameSizeInfo(ZSTD_findFrameCompressedSizeLegacy(src, srcSize));
 #endif
 
     if ((srcSize >= ZSTD_SKIPPABLEHEADERSIZE)
@@ -471,16 +476,10 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize
         /* Extract Frame Header */
         {
             size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize);
-            if (ZSTD_isError(ret)) {
-                frameSizeInfo.compressedSize = ret;
-                frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
-                return frameSizeInfo;
-            }
-            if (ret > 0) {
-                frameSizeInfo.compressedSize = ERROR(srcSize_wrong);
-                frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
-                return frameSizeInfo;
-            }
+            if (ZSTD_isError(ret))
+                return ZSTD_errorFrameSizeInfo(ret);
+            if (ret > 0)
+                return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong));
         }
 
         ip += zfh.headerSize;
@@ -490,17 +489,11 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize
         while (1) {
             blockProperties_t blockProperties;
             size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
-            if (ZSTD_isError(cBlockSize)) {
-                frameSizeInfo.compressedSize = cBlockSize;
-                frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
-                return frameSizeInfo;
-            }
+            if (ZSTD_isError(cBlockSize))
+                return ZSTD_errorFrameSizeInfo(cBlockSize);
 
-            if (ZSTD_blockHeaderSize + cBlockSize > remainingSize) {
-                frameSizeInfo.compressedSize = ERROR(srcSize_wrong);
-                frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
-                return frameSizeInfo;
-            }
+            if (ZSTD_blockHeaderSize + cBlockSize > remainingSize)
+                return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong));
 
             ip += ZSTD_blockHeaderSize + cBlockSize;
             remainingSize -= ZSTD_blockHeaderSize + cBlockSize;
@@ -511,11 +504,8 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize
 
         /* Final frame content checksum */
         if (zfh.checksumFlag) {
-            if (remainingSize < 4) {
-                frameSizeInfo.compressedSize = ERROR(srcSize_wrong);
-                frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
-                return frameSizeInfo;
-            }
+            if (remainingSize < 4)
+                return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong));
             ip += 4;
         }
 
@@ -534,7 +524,7 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize
  *  @return : the compressed size of the frame starting at `src` */
 size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
 {
-    ZSTD_frameSizeInfo frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize);
+    ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize);
     return frameSizeInfo.compressedSize;
 }
 
@@ -550,9 +540,9 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize)
     unsigned long long bound = 0;
     /* Iterate over each frame */
     while (srcSize > 0) {
-        ZSTD_frameSizeInfo frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize);
-        size_t compressedSize = frameSizeInfo.compressedSize;
-        unsigned long long decompressedBound = frameSizeInfo.decompressedBound;
+        ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize);
+        size_t const compressedSize = frameSizeInfo.compressedSize;
+        unsigned long long const decompressedBound = frameSizeInfo.decompressedBound;
         if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) {
             return ZSTD_CONTENTSIZE_ERROR;
         }
diff --git a/lib/zstd.h b/lib/zstd.h
index e49a4048a06..182ff376c7a 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -1104,16 +1104,16 @@ typedef enum {
 ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
 
 /** ZSTD_decompressBound() :
- *  currently incompatible with legacy mode
- *  `src` must point to the start of a ZSTD frame or a skippeable frame
- *  `srcSize` must be at least as large as the frame contained
+ *  `src` should point the start of a series of ZSTD encoded and/or skippable frames
+ *  `srcSize` must be the _exact_ size of this series
+ *       (i.e. there should be a frame boundary exactly at `srcSize` bytes after `src`)
  *  @return : - the maximum decompressed size of the compressed source
  *            - if an error occured: ZSTD_CONTENTSIZE_ERROR
  *
  *  note 1  : an error can occur if `src` points to a legacy frame or an invalid/incorrectly formatted frame.
- *  note 2  : the bound is exact when Frame_Content_Size field is available in EVERY frame of `src`.
+ *  note 2  : the bound is exact when Frame_Content_Size field is available in _every_ frame of `src`.
  *  note 3  : when Frame_Content_Size isn't provided, the upper-bound for that frame is calculated by:
- *              upper-bound = min(128 KB, Window_Size)
+ *              upper-bound = # blocks * min(128 KB, Window_Size)
  *  note 4  : we always use Frame_Content_Size to bound the decompressed frame size if it's present.
  */
 ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSice);

From 3da3dc2f45e712bcd293a0376aa1dc8479d2324e Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Fri, 1 Mar 2019 21:27:30 -0800
Subject: [PATCH 086/178] add missing size content test

---
 tests/fuzzer.c | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 9efa232a90d..a61df2180f9 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -438,6 +438,28 @@ static int basicUnitTests(U32 seed, double compressibility)
       if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
     DISPLAYLEVEL(3, "OK \n");
 
+    DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with content size missing : ", testNb++);
+    {   /* create compressed buffer with content size missing */
+        ZSTD_CCtx* cctx = ZSTD_createCCtx();
+        CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) );
+        CHECKPLUS(r, ZSTD_compress2(cctx,
+                            compressedBuffer, compressedBufferSize,
+                            CNBuffer, CNBuffSize),
+                  cSize=r );
+        ZSTD_freeCCtx(cctx);
+    }
+    {   /* ensure frame content size is missing */
+        ZSTD_frameHeader zfh;
+        size_t ret = ZSTD_getFrameHeader(&zfh, compressedBuffer, compressedBufferSize);
+        if (ret != 0) goto _output_error;
+        if (zfh.frameContentSize !=  ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
+    }
+    {   /* ensure CNBuffSize <= decompressBound */
+        unsigned long long bound = ZSTD_decompressBound(compressedBuffer, compressedBufferSize);
+        if (CNBuffSize > bound) goto _output_error;
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
     DISPLAYLEVEL(3, "test%3d : check CCtx size after compressing empty input : ", testNb++);
     {   ZSTD_CCtx* const cctx = ZSTD_createCCtx();
         size_t const r = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 19);

From 95dfd4814363e44db29e539f73d10dcdff0a31ee Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Fri, 1 Mar 2019 23:11:15 -0800
Subject: [PATCH 087/178] update formatting

---
 lib/decompress/zstd_decompress.c |  6 ++----
 tests/fuzzer.c                   | 15 +++++++--------
 2 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 254e2923dc1..268ba386a45 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -474,8 +474,7 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize
         ZSTD_frameHeader zfh;
 
         /* Extract Frame Header */
-        {
-            size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize);
+        {   size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize);
             if (ZSTD_isError(ret))
                 return ZSTD_errorFrameSizeInfo(ret);
             if (ret > 0)
@@ -543,9 +542,8 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize)
         ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize);
         size_t const compressedSize = frameSizeInfo.compressedSize;
         unsigned long long const decompressedBound = frameSizeInfo.decompressedBound;
-        if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) {
+        if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR)
             return ZSTD_CONTENTSIZE_ERROR;
-        }
         src = (const BYTE*)src + compressedSize;
         srcSize -= compressedSize;
         bound += decompressedBound;
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index a61df2180f9..37531379516 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -378,8 +378,8 @@ static int basicUnitTests(U32 seed, double compressibility)
 
     DISPLAYLEVEL(3, "test%3i : tight ZSTD_decompressBound test : ", testNb++);
     {
-        unsigned long long rSize = ZSTD_decompressBound(compressedBuffer, cSize);
-        if (rSize != CNBuffSize) goto _output_error;
+        unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize);
+        if (bound != CNBuffSize) goto _output_error;
     }
     DISPLAYLEVEL(3, "OK \n");
 
@@ -450,12 +450,11 @@ static int basicUnitTests(U32 seed, double compressibility)
     }
     {   /* ensure frame content size is missing */
         ZSTD_frameHeader zfh;
-        size_t ret = ZSTD_getFrameHeader(&zfh, compressedBuffer, compressedBufferSize);
-        if (ret != 0) goto _output_error;
-        if (zfh.frameContentSize !=  ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
+        size_t const ret = ZSTD_getFrameHeader(&zfh, compressedBuffer, compressedBufferSize);
+        if (ret != 0 || zfh.frameContentSize !=  ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
     }
     {   /* ensure CNBuffSize <= decompressBound */
-        unsigned long long bound = ZSTD_decompressBound(compressedBuffer, compressedBufferSize);
+        unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, compressedBufferSize);
         if (CNBuffSize > bound) goto _output_error;
     }
     DISPLAYLEVEL(3, "OK \n");
@@ -931,8 +930,8 @@ static int basicUnitTests(U32 seed, double compressibility)
     DISPLAYLEVEL(3, "OK \n");
 
     DISPLAYLEVEL(3, "test%3i : get tight decompressed bound of multiple frames : ", testNb++);
-    {   unsigned long long const r = ZSTD_decompressBound(compressedBuffer, cSize);
-        if (r != CNBuffSize / 2) goto _output_error; }
+    {   unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, cSize);
+        if (bound != CNBuffSize / 2) goto _output_error; }
     DISPLAYLEVEL(3, "OK \n");
 
     DISPLAYLEVEL(3, "test%3i : decompress multiple frames : ", testNb++);

From 9ad3f31d33389f3985bbf15a0954f57a1bfb826c Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Sat, 2 Mar 2019 17:56:10 -0800
Subject: [PATCH 088/178] update documentation for decompressBound

---
 lib/zstd.h | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/lib/zstd.h b/lib/zstd.h
index 182ff376c7a..a29c9669a11 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -1081,9 +1081,9 @@ typedef enum {
 ***************************************/
 
 /*! ZSTD_findDecompressedSize() :
- *  `src` should point the start of a series of ZSTD encoded and/or skippable frames
+ *  `src` should point to the start of a series of ZSTD encoded and/or skippable frames
  *  `srcSize` must be the _exact_ size of this series
- *       (i.e. there should be a frame boundary exactly at `srcSize` bytes after `src`)
+ *       (i.e. there should be a frame boundary at `src + srcSize`)
  *  @return : - decompressed size of all data in all successive frames
  *            - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
  *            - if an error occurred: ZSTD_CONTENTSIZE_ERROR
@@ -1104,17 +1104,17 @@ typedef enum {
 ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
 
 /** ZSTD_decompressBound() :
- *  `src` should point the start of a series of ZSTD encoded and/or skippable frames
+ *  `src` should point to the start of a series of ZSTD encoded and/or skippable frames
  *  `srcSize` must be the _exact_ size of this series
- *       (i.e. there should be a frame boundary exactly at `srcSize` bytes after `src`)
- *  @return : - the maximum decompressed size of the compressed source
+ *       (i.e. there should be a frame boundary at `src + srcSize`)
+ *  @return : - upper-bound for the decompressed size of all data in all successive frames
  *            - if an error occured: ZSTD_CONTENTSIZE_ERROR
  *
- *  note 1  : an error can occur if `src` points to a legacy frame or an invalid/incorrectly formatted frame.
- *  note 2  : the bound is exact when Frame_Content_Size field is available in _every_ frame of `src`.
- *  note 3  : when Frame_Content_Size isn't provided, the upper-bound for that frame is calculated by:
+ *  note 1  : an error can occur if `src` contains a legacy frame or an invalid/incorrectly formatted frame.
+ *  note 2  : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`.
+ *            in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value.
+ *  note 3  : when the decompressed size field isn't available, the upper-bound for that frame is calculated by:
  *              upper-bound = # blocks * min(128 KB, Window_Size)
- *  note 4  : we always use Frame_Content_Size to bound the decompressed frame size if it's present.
  */
 ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSice);
 

From 79827a179f2b659928487aeb9cf802d675aa7840 Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Wed, 13 Mar 2019 01:23:07 -0700
Subject: [PATCH 089/178] Fix incorrectly assigned value in
 ZSTD_errorFrameSizeInfo

As documented in `zstd.h`, ZSTD_decompressBound returns `ZSTD_CONTENTSIZE_ERROR`
if an error occurs (not `ZSTD_CONTENTSIZE_UNKNOWN`). This is consistent with
the error checking made in ZSTD_decompressBound, particularly line 545.
---
 lib/decompress/zstd_decompress.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 268ba386a45..9974dbe3b08 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -437,7 +437,7 @@ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t he
  * Contains the compressed frame size and an upper-bound for the decompressed frame size.
  * Note: before using `compressedSize` you must check for errors using ZSTD_isError().
  *       similarly, before using `decompressedBound`, you must check for errors using:
- *          `decompressedBound` != ZSTD_CONTENTSIZE_UNKNOWN
+ *          `decompressedBound` != ZSTD_CONTENTSIZE_ERROR
  */
 typedef struct {
     size_t compressedSize;
@@ -448,7 +448,7 @@ static ZSTD_frameSizeInfo ZSTD_errorFrameSizeInfo(size_t ret)
 {
     ZSTD_frameSizeInfo frameSizeInfo;
     frameSizeInfo.compressedSize = ret;
-    frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_UNKNOWN;
+    frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
     return frameSizeInfo;
 }
 

From 18d3a97d434a49b40a0462ce9c14b7fde9aea3d8 Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Wed, 13 Mar 2019 01:43:40 -0700
Subject: [PATCH 090/178] Add unit test to validate the error case

---
 tests/fuzzer.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 37531379516..b96fb4acd5d 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -383,6 +383,13 @@ static int basicUnitTests(U32 seed, double compressibility)
     }
     DISPLAYLEVEL(3, "OK \n");
 
+    DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with invalid srcSize : ", testNb++);
+    {
+        unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize - 1);
+        if (bound != ZSTD_CONTENTSIZE_ERROR) goto _output_error;
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
     DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize);
     { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
       if (r != CNBuffSize) goto _output_error; }

From 0594e8135b87e603b7640e58a2b59cbac5b38893 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Wed, 13 Mar 2019 14:54:31 -0700
Subject: [PATCH 091/178] [libzstd] Free local cdict when referencing cdict

We no longer care about the `cdictLocal` after calling
`ZSTD_CCtx_refCDict()`, so we should free it to save some
memory.
---
 lib/compress/zstd_compress.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 3d4091455a7..8de58b70964 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -839,6 +839,9 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s
 size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
 {
     RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong);
+    /* Free the existing local cdict (if any) to save memory. */
+    FORWARD_IF_ERROR( ZSTD_freeCDict(cctx->cdictLocal) );
+    cctx->cdictLocal = NULL;
     cctx->cdict = cdict;
     memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict));  /* exclusive */
     return 0;

From 787b76904a82998a7bdd95a40f4e442c98ae6461 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Wed, 13 Mar 2019 15:23:24 -0700
Subject: [PATCH 092/178] [libzstd] Allow compression parameters to be set with
 a cdict

The order you set parameters in the advanced API is not supposed to matter.
However, once you call `ZSTD_CCtx_refCDict()` the compression parameters
cannot be changed. Remove that restriction, and document what parameters
are used when using a CDict.

If the CCtx is in dictionary mode, then the CDict's parameters are used.
If the CCtx is not in dictionary mode, then its requested parameters are
used.
---
 lib/compress/zstd_compress.c | 20 ++++----------------
 lib/zstd.h                   |  9 +++++++--
 tests/fuzzer.c               | 11 +++++++++++
 3 files changed, 22 insertions(+), 18 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 8de58b70964..987c7b90aa6 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -450,11 +450,12 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
 
     switch(param)
     {
-    case ZSTD_c_compressionLevel:
-        RETURN_ERROR_IF(cctx->cdict, stage_wrong,
-                        "compression level is configured in cdict");
+    case ZSTD_c_nbWorkers:
+        RETURN_ERROR_IF((value!=0) && cctx->staticSize, parameter_unsupported,
+                        "MT not compatible with static alloc");
         break;
 
+    case ZSTD_c_compressionLevel:
     case ZSTD_c_windowLog:
     case ZSTD_c_hashLog:
     case ZSTD_c_chainLog:
@@ -462,20 +463,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
     case ZSTD_c_minMatch:
     case ZSTD_c_targetLength:
     case ZSTD_c_strategy:
-        RETURN_ERROR_IF(cctx->cdict, stage_wrong,
-                        "cparams are configured in cdict");
-        break;
-
-    case ZSTD_c_nbWorkers:
-        RETURN_ERROR_IF((value!=0) && cctx->staticSize, parameter_unsupported,
-                        "MT not compatible with static alloc");
-        break;
-
     case ZSTD_c_ldmHashRateLog:
-        RETURN_ERROR_IF(cctx->cdict, stage_wrong,
-                        "LDM hash rate log is configured in cdict");
-        break;
-
     case ZSTD_c_format:
     case ZSTD_c_contentSizeFlag:
     case ZSTD_c_checksumFlag:
diff --git a/lib/zstd.h b/lib/zstd.h
index a29c9669a11..4373d6dfd36 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -503,7 +503,10 @@ typedef enum { ZSTD_fast=1,
 
 typedef enum {
 
-    /* compression parameters */
+    /* compression parameters
+     * Note: When compressing with a ZSTD_CDict these parameters are superseded
+     * by the parameters used to construct the ZSTD_CDict. See ZSTD_CCtx_refCDict()
+     * for more info (superseded-by-cdict). */
     ZSTD_c_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table
                               * Default level is ZSTD_CLEVEL_DEFAULT==3.
                               * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
@@ -705,7 +708,9 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s
 /*! ZSTD_CCtx_refCDict() :
  *  Reference a prepared dictionary, to be used for all next compressed frames.
  *  Note that compression parameters are enforced from within CDict,
- *  and supercede any compression parameter previously set within CCtx.
+ *  and supersede any compression parameter previously set within CCtx.
+ *  The parameters ignored are labled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
+ *  The ignored parameters will be used again if the CCtx is returned to no-dictionary mode.
  *  The dictionary will remain valid for future compressed frames using same CCtx.
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
  *  Special : Referencing a NULL CDict means "return to no-dictionary mode".
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 37531379516..feb16c8061d 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1294,6 +1294,17 @@ static int basicUnitTests(U32 seed, double compressibility)
         }
         DISPLAYLEVEL(3, "OK \n");
 
+        DISPLAYLEVEL(3, "test%3i : ZSTD_CCtx_refCDict() then set parameters : ", testNb++);
+        {   ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 1);
+            CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
+            CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 ));
+            CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
+            CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
+            CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 ));
+            ZSTD_freeCDict(cdict);
+        }
+        DISPLAYLEVEL(3, "OK \n");
+
         DISPLAYLEVEL(3, "test%3i : Dictionary with non-default repcodes : ", testNb++);
         { U32 u; for (u=0; u
Date: Wed, 13 Mar 2019 17:01:23 -0700
Subject: [PATCH 093/178] [zstreamtest] Remove outdated test

---
 tests/zstreamtest.c | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 6d3cbffb111..b099e2870a6 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1930,16 +1930,6 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest,
                     } else {
                         CHECK_Z( ZSTD_CCtx_loadDictionary_byReference(zc, dict, dictSize) );
                     }
-                    if (dict && dictSize) {
-                        /* test that compression parameters are rejected (correctly) after loading a non-NULL dictionary */
-                        if (opaqueAPI) {
-                            size_t const setError = ZSTD_CCtx_setParametersUsingCCtxParams(zc, cctxParams);
-                            CHECK(!ZSTD_isError(setError), "ZSTD_CCtx_setParametersUsingCCtxParams should have failed");
-                        } else {
-                            size_t const setError = ZSTD_CCtx_setParameter(zc, ZSTD_c_windowLog, cParams.windowLog-1);
-                            CHECK(!ZSTD_isError(setError), "ZSTD_CCtx_setParameter should have failed");
-                        }
-                    }
                 } else {
                     CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) );
                 }

From 91ffc8d256b4a21f29b82057868ccb1fcf9ebb12 Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Fri, 15 Mar 2019 03:59:03 -0700
Subject: [PATCH 094/178] Add test to validate patch

---
 tests/legacy.c | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/tests/legacy.c b/tests/legacy.c
index b749567f440..6da71227ffd 100644
--- a/tests/legacy.c
+++ b/tests/legacy.c
@@ -130,12 +130,28 @@ static int testStreamingAPI(void)
     return error_code;
 }
 
+static int testFrameDecoding(void)
+{
+    if (ZSTD_decompressBound(COMPRESSED, COMPRESSED_SIZE) != ZSTD_CONTENTSIZE_ERROR) {
+        DISPLAY("ERROR: ZSTD_decompressBound: Expected to receive ZSTD_CONTENTSIZE_ERROR\n");
+        return 1;
+    }
+    if (ZSTD_findFrameCompressedSize(COMPRESSED) != COMPRESSED_SIZE) {
+        DISPLAY("ERROR: ZSTD_findFrameCompressedSize: Expected to receive %d\n", COMPRESSED_SIZE);
+        return 1;
+    }
+    DISPLAY("Frame Decoding OK\n");
+    return 0;
+}
+
 int main(void)
 {
     {   int const ret = testSimpleAPI();
         if (ret) return ret; }
     {   int const ret = testStreamingAPI();
         if (ret) return ret; }
+    {   int const ret = testFrameDecoding();
+        if (ret) return ret; }
 
     DISPLAY("OK\n");
     return 0;

From 4c0540da1c77dd1710ee1c492f6c8bd562d417cd Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Fri, 15 Mar 2019 05:13:55 -0700
Subject: [PATCH 095/178] Add static linking to legacy tests

---
 tests/legacy.c | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/tests/legacy.c b/tests/legacy.c
index 6da71227ffd..e660b9cc6d3 100644
--- a/tests/legacy.c
+++ b/tests/legacy.c
@@ -16,10 +16,11 @@
 /*===========================================
 *   Dependencies
 *==========================================*/
-#include      /* size_t */
-#include      /* malloc, free */
-#include       /* fprintf */
-#include      /* strlen */
+#include               /* size_t */
+#include               /* malloc, free */
+#include                /* fprintf */
+#include               /* strlen */
+#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_decompressBound */
 #include "zstd.h"
 #include "zstd_errors.h"
 
@@ -136,10 +137,6 @@ static int testFrameDecoding(void)
         DISPLAY("ERROR: ZSTD_decompressBound: Expected to receive ZSTD_CONTENTSIZE_ERROR\n");
         return 1;
     }
-    if (ZSTD_findFrameCompressedSize(COMPRESSED) != COMPRESSED_SIZE) {
-        DISPLAY("ERROR: ZSTD_findFrameCompressedSize: Expected to receive %d\n", COMPRESSED_SIZE);
-        return 1;
-    }
     DISPLAY("Frame Decoding OK\n");
     return 0;
 }

From 60796e76b057c116c18f998505d209c46246381a Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Fri, 15 Mar 2019 16:10:37 -0700
Subject: [PATCH 096/178] Add legacy support to decompressBound

---
 lib/decompress/zstd_decompress.c | 16 ++--------
 lib/legacy/zstd_legacy.h         | 52 +++++++++++++++++++++++++-------
 lib/legacy/zstd_v01.c            | 37 +++++++++++++++++++----
 lib/legacy/zstd_v01.h            | 15 ++++-----
 lib/legacy/zstd_v02.c            | 41 ++++++++++++++++++-------
 lib/legacy/zstd_v02.h            | 15 ++++-----
 lib/legacy/zstd_v03.c            | 42 +++++++++++++++++++-------
 lib/legacy/zstd_v03.h            | 15 ++++-----
 lib/legacy/zstd_v04.c            | 42 +++++++++++++++++++-------
 lib/legacy/zstd_v04.h            | 15 ++++-----
 lib/legacy/zstd_v05.c            | 37 +++++++++++++++++++----
 lib/legacy/zstd_v05.h            | 15 ++++-----
 lib/legacy/zstd_v06.c            | 42 +++++++++++++++++++++-----
 lib/legacy/zstd_v06.h            |  3 +-
 lib/legacy/zstd_v07.c            | 47 ++++++++++++++++++++++++-----
 lib/legacy/zstd_v07.h            |  3 +-
 lib/zstd.h                       | 11 +++++++
 tests/legacy.c                   |  4 +--
 18 files changed, 329 insertions(+), 123 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 9974dbe3b08..a981af0a16e 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -433,17 +433,6 @@ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t he
     return 0;
 }
 
-/**
- * Contains the compressed frame size and an upper-bound for the decompressed frame size.
- * Note: before using `compressedSize` you must check for errors using ZSTD_isError().
- *       similarly, before using `decompressedBound`, you must check for errors using:
- *          `decompressedBound` != ZSTD_CONTENTSIZE_ERROR
- */
-typedef struct {
-    size_t compressedSize;
-    unsigned long long decompressedBound;
-} ZSTD_frameSizeInfo;
-
 static ZSTD_frameSizeInfo ZSTD_errorFrameSizeInfo(size_t ret)
 {
     ZSTD_frameSizeInfo frameSizeInfo;
@@ -458,8 +447,9 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize
     memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo));
 
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
-    if (ZSTD_isLegacy(src, srcSize))
-        return ZSTD_errorFrameSizeInfo(ZSTD_findFrameCompressedSizeLegacy(src, srcSize));
+    if (ZSTD_isLegacy(src, srcSize)) {
+        return ZSTD_findFrameSizeInfoLegacy(src, srcSize);
+    }
 #endif
 
     if ((srcSize >= ZSTD_SKIPPABLEHEADERSIZE)
diff --git a/lib/legacy/zstd_legacy.h b/lib/legacy/zstd_legacy.h
index 5893cb9657e..306a87399e3 100644
--- a/lib/legacy/zstd_legacy.h
+++ b/lib/legacy/zstd_legacy.h
@@ -178,43 +178,73 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
     }
 }
 
-MEM_STATIC size_t ZSTD_findFrameCompressedSizeLegacy(const void *src,
-                                             size_t compressedSize)
+MEM_STATIC void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
 {
-    U32 const version = ZSTD_isLegacy(src, compressedSize);
+    *cSize = ret;
+    *dBound = ZSTD_CONTENTSIZE_ERROR;
+}
+
+MEM_STATIC ZSTD_frameSizeInfo ZSTD_findFrameSizeInfoLegacy(const void *src, size_t srcSize)
+{
+    ZSTD_frameSizeInfo frameSizeInfo;
+    memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo));
+    U32 const version = ZSTD_isLegacy(src, srcSize);
     switch(version)
     {
 #if (ZSTD_LEGACY_SUPPORT <= 1)
         case 1 :
-            return ZSTDv01_findFrameCompressedSize(src, compressedSize);
+            ZSTDv01_findFrameSizeInfoLegacy(src, srcSize,
+                &frameSizeInfo.compressedSize, &frameSizeInfo.decompressedBound);
+            break;
 #endif
 #if (ZSTD_LEGACY_SUPPORT <= 2)
         case 2 :
-            return ZSTDv02_findFrameCompressedSize(src, compressedSize);
+            ZSTDv02_findFrameSizeInfoLegacy(src, srcSize,
+                &frameSizeInfo.compressedSize, &frameSizeInfo.decompressedBound);
+            break;
 #endif
 #if (ZSTD_LEGACY_SUPPORT <= 3)
         case 3 :
-            return ZSTDv03_findFrameCompressedSize(src, compressedSize);
+            ZSTDv03_findFrameSizeInfoLegacy(src, srcSize,
+                &frameSizeInfo.compressedSize, &frameSizeInfo.decompressedBound);
+            break;
 #endif
 #if (ZSTD_LEGACY_SUPPORT <= 4)
         case 4 :
-            return ZSTDv04_findFrameCompressedSize(src, compressedSize);
+            ZSTDv04_findFrameSizeInfoLegacy(src, srcSize,
+                &frameSizeInfo.compressedSize, &frameSizeInfo.decompressedBound);
+            break;
 #endif
 #if (ZSTD_LEGACY_SUPPORT <= 5)
         case 5 :
-            return ZSTDv05_findFrameCompressedSize(src, compressedSize);
+            ZSTDv05_findFrameSizeInfoLegacy(src, srcSize,
+                &frameSizeInfo.compressedSize, &frameSizeInfo.decompressedBound);
+            break;
 #endif
 #if (ZSTD_LEGACY_SUPPORT <= 6)
         case 6 :
-            return ZSTDv06_findFrameCompressedSize(src, compressedSize);
+            ZSTDv06_findFrameSizeInfoLegacy(src, srcSize,
+                &frameSizeInfo.compressedSize, &frameSizeInfo.decompressedBound);
+            break;
 #endif
 #if (ZSTD_LEGACY_SUPPORT <= 7)
         case 7 :
-            return ZSTDv07_findFrameCompressedSize(src, compressedSize);
+            ZSTDv07_findFrameSizeInfoLegacy(src, srcSize,
+                &frameSizeInfo.compressedSize, &frameSizeInfo.decompressedBound);
+            break;
 #endif
         default :
-            return ERROR(prefix_unknown);
+            ZSTD_errorFrameSizeInfoLegacy(&frameSizeInfo.compressedSize,
+                &frameSizeInfo.decompressedBound, ERROR(prefix_unknown));
+            break;
     }
+    return frameSizeInfo;
+}
+
+MEM_STATIC size_t ZSTD_findFrameCompressedSizeLegacy(const void *src, size_t srcSize)
+{
+    ZSTD_frameSizeInfo frameSizeInfo = ZSTD_findFrameSizeInfoLegacy(src, srcSize);
+    return frameSizeInfo.compressedSize;
 }
 
 MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version)
diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c
index c007e7ceb51..1d2d34f18a2 100644
--- a/lib/legacy/zstd_v01.c
+++ b/lib/legacy/zstd_v01.c
@@ -1336,6 +1336,9 @@ static const U32 ZSTD_magicNumber = 0xFD2FB51E;   /* 3rd version : seqNb header
 #define LITERAL_NOENTROPY 63
 #define COMMAND_NOENTROPY 7   /* to remove */
 
+#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
+#define ZSTD_CONTENTSIZE_ERROR   (0ULL - 2)
+
 static const size_t ZSTD_blockHeaderSize = 3;
 static const size_t ZSTD_frameHeaderSize = 4;
 
@@ -1999,36 +2002,58 @@ size_t ZSTDv01_decompress(void* dst, size_t maxDstSize, const void* src, size_t
     return ZSTDv01_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
 }
 
-size_t ZSTDv01_findFrameCompressedSize(const void* src, size_t srcSize)
+static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
+{
+    *cSize = ret;
+    *dBound = ZSTD_CONTENTSIZE_ERROR;
+}
+
+void ZSTDv01_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
+                                     size_t* cSize, unsigned long long* dBound)
 {
     const BYTE* ip = (const BYTE*)src;
     size_t remainingSize = srcSize;
+    size_t nbBlocks = 0;
     U32 magicNumber;
     blockProperties_t blockProperties;
 
     /* Frame Header */
-    if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
+    if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) {
+        ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
+        return;
+    }
     magicNumber = ZSTD_readBE32(src);
-    if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
+    if (magicNumber != ZSTD_magicNumber) {
+        ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
+        return;
+    }
     ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
 
     /* Loop on each block */
     while (1)
     {
         size_t blockSize = ZSTDv01_getcBlockSize(ip, remainingSize, &blockProperties);
-        if (ZSTDv01_isError(blockSize)) return blockSize;
+        if (ZSTDv01_isError(blockSize)) {
+            ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, blockSize);
+            return;
+        }
 
         ip += ZSTD_blockHeaderSize;
         remainingSize -= ZSTD_blockHeaderSize;
-        if (blockSize > remainingSize) return ERROR(srcSize_wrong);
+        if (blockSize > remainingSize) {
+            ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
+            return;
+        }
 
         if (blockSize == 0) break;   /* bt_end */
 
         ip += blockSize;
         remainingSize -= blockSize;
+        nbBlocks++;
     }
 
-    return ip - (const BYTE*)src;
+    *cSize = ip - (const BYTE*)src;
+    *dBound = nbBlocks * BLOCKSIZE;
 }
 
 /*******************************
diff --git a/lib/legacy/zstd_v01.h b/lib/legacy/zstd_v01.h
index 42f0897c7d2..c56a2f0c70a 100644
--- a/lib/legacy/zstd_v01.h
+++ b/lib/legacy/zstd_v01.h
@@ -35,13 +35,14 @@ ZSTDv01_decompress() : decompress ZSTD frames compliant with v0.1.x format
 size_t ZSTDv01_decompress( void* dst, size_t maxOriginalSize,
                      const void* src, size_t compressedSize);
 
-/**
-ZSTDv01_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.1.x format
-    compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
-    return : the number of bytes that would be read to decompress this frame
-             or an errorCode if it fails (which can be tested using ZSTDv01_isError())
-*/
-size_t ZSTDv01_findFrameCompressedSize(const void* src, size_t compressedSize);
+ /**
+ ZSTDv01_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.1.x format
+     compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
+     return : the number of bytes that would be read to decompress this frame
+              or an errorCode if it fails (which can be tested using ZSTDv01_isError())
+ */
+void ZSTDv01_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
+                                     size_t* cSize, unsigned long long* dBound);
 
 /**
 ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error
diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c
index c09ef8cff23..06496101c09 100644
--- a/lib/legacy/zstd_v02.c
+++ b/lib/legacy/zstd_v02.c
@@ -2728,6 +2728,9 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
 #define LITERAL_NOENTROPY 63
 #define COMMAND_NOENTROPY 7   /* to remove */
 
+#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
+#define ZSTD_CONTENTSIZE_ERROR   (0ULL - 2)
+
 static const size_t ZSTD_blockHeaderSize = 3;
 static const size_t ZSTD_frameHeaderSize = 4;
 
@@ -3312,37 +3315,58 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
     return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
 }
 
-static size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
+MEM_STATIC void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
 {
+    *cSize = ret;
+    *dBound = ZSTD_CONTENTSIZE_ERROR;
+}
 
+void ZSTDv02_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
+                                     size_t* cSize, unsigned long long* dBound)
+{
     const BYTE* ip = (const BYTE*)src;
     size_t remainingSize = srcSize;
+    size_t nbBlocks = 0;
     U32 magicNumber;
     blockProperties_t blockProperties;
 
     /* Frame Header */
-    if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
+    if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) {
+        ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
+        return;
+    }
     magicNumber = MEM_readLE32(src);
-    if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
+    if (magicNumber != ZSTD_magicNumber) {
+        ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
+        return;
+    }
     ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
 
     /* Loop on each block */
     while (1)
     {
         size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
-        if (ZSTD_isError(cBlockSize)) return cBlockSize;
+        if (ZSTD_isError(cBlockSize)) {
+            ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
+            return;
+        }
 
         ip += ZSTD_blockHeaderSize;
         remainingSize -= ZSTD_blockHeaderSize;
-        if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
+        if (cBlockSize > remainingSize) {
+            ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
+            return;
+        }
 
         if (cBlockSize == 0) break;   /* bt_end */
 
         ip += cBlockSize;
         remainingSize -= cBlockSize;
+        nbBlocks++;
     }
 
-    return ip - (const BYTE*)src;
+    *cSize = ip - (const BYTE*)src;
+    *dBound = nbBlocks * BLOCKSIZE;
 }
 
 /*******************************
@@ -3458,11 +3482,6 @@ size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
     return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
 }
 
-size_t ZSTDv02_findFrameCompressedSize(const void *src, size_t compressedSize)
-{
-    return ZSTD_findFrameCompressedSize(src, compressedSize);
-}
-
 ZSTDv02_Dctx* ZSTDv02_createDCtx(void)
 {
     return (ZSTDv02_Dctx*)ZSTD_createDCtx();
diff --git a/lib/legacy/zstd_v02.h b/lib/legacy/zstd_v02.h
index 0dde7a63773..5f7cba5246c 100644
--- a/lib/legacy/zstd_v02.h
+++ b/lib/legacy/zstd_v02.h
@@ -35,13 +35,14 @@ ZSTDv02_decompress() : decompress ZSTD frames compliant with v0.2.x format
 size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
                      const void* src, size_t compressedSize);
 
-/**
-ZSTDv02_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.2.x format
-    compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
-    return : the number of bytes that would be read to decompress this frame
-             or an errorCode if it fails (which can be tested using ZSTDv02_isError())
-*/
-size_t ZSTDv02_findFrameCompressedSize(const void* src, size_t compressedSize);
+ /**
+ ZSTDv02_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.2.x format
+     compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
+     return : the number of bytes that would be read to decompress this frame
+              or an errorCode if it fails (which can be tested using ZSTDv02_isError())
+ */
+void ZSTDv02_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
+                                     size_t* cSize, unsigned long long* dBound);
 
 /**
 ZSTDv02_isError() : tells if the result of ZSTDv02_decompress() is an error
diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c
index 0c4cdf6888a..27c1b67bbe7 100644
--- a/lib/legacy/zstd_v03.c
+++ b/lib/legacy/zstd_v03.c
@@ -2369,6 +2369,9 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
 #define LITERAL_NOENTROPY 63
 #define COMMAND_NOENTROPY 7   /* to remove */
 
+#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
+#define ZSTD_CONTENTSIZE_ERROR   (0ULL - 2)
+
 static const size_t ZSTD_blockHeaderSize = 3;
 static const size_t ZSTD_frameHeaderSize = 4;
 
@@ -2953,36 +2956,58 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
     return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
 }
 
-static size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
+MEM_STATIC void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
+{
+    *cSize = ret;
+    *dBound = ZSTD_CONTENTSIZE_ERROR;
+}
+
+void ZSTDv03_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
+                                     size_t* cSize, unsigned long long* dBound)
 {
     const BYTE* ip = (const BYTE*)src;
     size_t remainingSize = srcSize;
+    size_t nbBlocks = 0;
     U32 magicNumber;
     blockProperties_t blockProperties;
 
     /* Frame Header */
-    if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
+    if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) {
+        ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
+        return;
+    }
     magicNumber = MEM_readLE32(src);
-    if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
+    if (magicNumber != ZSTD_magicNumber) {
+        ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
+        return;
+    }
     ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
 
     /* Loop on each block */
     while (1)
     {
         size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
-        if (ZSTD_isError(cBlockSize)) return cBlockSize;
+        if (ZSTD_isError(cBlockSize)) {
+            ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
+            return;
+        }
 
         ip += ZSTD_blockHeaderSize;
         remainingSize -= ZSTD_blockHeaderSize;
-        if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
+        if (cBlockSize > remainingSize) {
+            ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
+            return;
+        }
 
         if (cBlockSize == 0) break;   /* bt_end */
 
         ip += cBlockSize;
         remainingSize -= cBlockSize;
+        nbBlocks++;
     }
 
-    return ip - (const BYTE*)src;
+    *cSize = ip - (const BYTE*)src;
+    *dBound = nbBlocks * BLOCKSIZE;
 }
 
 
@@ -3099,11 +3124,6 @@ size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
     return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
 }
 
-size_t ZSTDv03_findFrameCompressedSize(const void* src, size_t srcSize)
-{
-    return ZSTD_findFrameCompressedSize(src, srcSize);
-}
-
 ZSTDv03_Dctx* ZSTDv03_createDCtx(void)
 {
     return (ZSTDv03_Dctx*)ZSTD_createDCtx();
diff --git a/lib/legacy/zstd_v03.h b/lib/legacy/zstd_v03.h
index b4449e2999e..36fc6119050 100644
--- a/lib/legacy/zstd_v03.h
+++ b/lib/legacy/zstd_v03.h
@@ -35,13 +35,14 @@ ZSTDv03_decompress() : decompress ZSTD frames compliant with v0.3.x format
 size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
                      const void* src, size_t compressedSize);
 
-/**
-ZSTDv03_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.3.x format
-    compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
-    return : the number of bytes that would be read to decompress this frame
-             or an errorCode if it fails (which can be tested using ZSTDv03_isError())
-*/
-size_t ZSTDv03_findFrameCompressedSize(const void* src, size_t compressedSize);
+ /**
+ ZSTDv03_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.3.x format
+     compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
+     return : the number of bytes that would be read to decompress this frame
+              or an errorCode if it fails (which can be tested using ZSTDv03_isError())
+ */
+ void ZSTDv03_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
+                                      size_t* cSize, unsigned long long* dBound);
 
     /**
 ZSTDv03_isError() : tells if the result of ZSTDv03_decompress() is an error
diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c
index d7522025e7f..81994d1f320 100644
--- a/lib/legacy/zstd_v04.c
+++ b/lib/legacy/zstd_v04.c
@@ -373,6 +373,9 @@ static const size_t ZSTD_frameHeaderSize_min = 5;
 #define MIN_SEQUENCES_SIZE (2 /*seqNb*/ + 2 /*dumps*/ + 3 /*seqTables*/ + 1 /*bitStream*/)
 #define MIN_CBLOCK_SIZE (3 /*litCSize*/ + MIN_SEQUENCES_SIZE)
 
+#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
+#define ZSTD_CONTENTSIZE_ERROR   (0ULL - 2)
+
 typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
 
 
@@ -3119,34 +3122,56 @@ static size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx,
     return op-ostart;
 }
 
-static size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
+MEM_STATIC void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
+{
+    *cSize = ret;
+    *dBound = ZSTD_CONTENTSIZE_ERROR;
+}
+
+void ZSTDv04_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
+                                     size_t* cSize, unsigned long long* dBound)
 {
     const BYTE* ip = (const BYTE*)src;
     size_t remainingSize = srcSize;
+    size_t nbBlocks = 0;
     blockProperties_t blockProperties;
 
     /* Frame Header */
-    if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);
-    if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown);
+    if (srcSize < ZSTD_frameHeaderSize_min) {
+        ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
+        return;
+    }
+    if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) {
+        ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
+        return;
+    }
     ip += ZSTD_frameHeaderSize_min; remainingSize -= ZSTD_frameHeaderSize_min;
 
     /* Loop on each block */
     while (1)
     {
         size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
-        if (ZSTD_isError(cBlockSize)) return cBlockSize;
+        if (ZSTD_isError(cBlockSize)) {
+            ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
+            return;
+        }
 
         ip += ZSTD_blockHeaderSize;
         remainingSize -= ZSTD_blockHeaderSize;
-        if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
+        if (cBlockSize > remainingSize) {
+            ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
+            return;
+        }
 
         if (cBlockSize == 0) break;   /* bt_end */
 
         ip += cBlockSize;
         remainingSize -= cBlockSize;
+        nbBlocks++;
     }
 
-    return ip - (const BYTE*)src;
+    *cSize = ip - (const BYTE*)src;
+    *dBound = nbBlocks * BLOCKSIZE;
 }
 
 /* ******************************
@@ -3578,11 +3603,6 @@ size_t ZSTDv04_decompress(void* dst, size_t maxDstSize, const void* src, size_t
 #endif
 }
 
-size_t ZSTDv04_findFrameCompressedSize(const void* src, size_t srcSize)
-{
-    return ZSTD_findFrameCompressedSize(src, srcSize);
-}
-
 size_t ZSTDv04_resetDCtx(ZSTDv04_Dctx* dctx) { return ZSTD_resetDCtx(dctx); }
 
 size_t ZSTDv04_nextSrcSizeToDecompress(ZSTDv04_Dctx* dctx)
diff --git a/lib/legacy/zstd_v04.h b/lib/legacy/zstd_v04.h
index 6391631fc43..1da32c6c935 100644
--- a/lib/legacy/zstd_v04.h
+++ b/lib/legacy/zstd_v04.h
@@ -35,13 +35,14 @@ ZSTDv04_decompress() : decompress ZSTD frames compliant with v0.4.x format
 size_t ZSTDv04_decompress( void* dst, size_t maxOriginalSize,
                      const void* src, size_t compressedSize);
 
-/**
-ZSTDv04_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.4.x format
-    compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
-    return : the number of bytes that would be read to decompress this frame
-             or an errorCode if it fails (which can be tested using ZSTDv04_isError())
-*/
-size_t ZSTDv04_findFrameCompressedSize(const void* src, size_t compressedSize);
+ /**
+ ZSTDv04_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.4.x format
+     compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
+     return : the number of bytes that would be read to decompress this frame
+              or an errorCode if it fails (which can be tested using ZSTDv04_isError())
+ */
+ void ZSTDv04_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
+                                      size_t* cSize, unsigned long long* dBound);
 
 /**
 ZSTDv04_isError() : tells if the result of ZSTDv04_decompress() is an error
diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c
index 96bffc44a1a..ff1095ca96d 100644
--- a/lib/legacy/zstd_v05.c
+++ b/lib/legacy/zstd_v05.c
@@ -491,6 +491,9 @@ static const size_t ZSTDv05_frameHeaderSize_min = 5;
 
 #define WILDCOPY_OVERLENGTH 8
 
+#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
+#define ZSTD_CONTENTSIZE_ERROR   (0ULL - 2)
+
 typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
 
 
@@ -3508,34 +3511,56 @@ size_t ZSTDv05_decompress(void* dst, size_t maxDstSize, const void* src, size_t
 #endif
 }
 
-size_t ZSTDv05_findFrameCompressedSize(const void *src, size_t srcSize)
+MEM_STATIC void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
+{
+    *cSize = ret;
+    *dBound = ZSTD_CONTENTSIZE_ERROR;
+}
+
+void ZSTDv05_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
+                                     size_t* cSize, unsigned long long* dBound)
 {
     const BYTE* ip = (const BYTE*)src;
     size_t remainingSize = srcSize;
+    size_t nbBlocks = 0;
     blockProperties_t blockProperties;
 
     /* Frame Header */
-    if (srcSize < ZSTDv05_frameHeaderSize_min) return ERROR(srcSize_wrong);
-    if (MEM_readLE32(src) != ZSTDv05_MAGICNUMBER) return ERROR(prefix_unknown);
+    if (srcSize < ZSTDv05_frameHeaderSize_min) {
+        ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
+        return;
+    }
+    if (MEM_readLE32(src) != ZSTDv05_MAGICNUMBER) {
+        ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
+        return;
+    }
     ip += ZSTDv05_frameHeaderSize_min; remainingSize -= ZSTDv05_frameHeaderSize_min;
 
     /* Loop on each block */
     while (1)
     {
         size_t cBlockSize = ZSTDv05_getcBlockSize(ip, remainingSize, &blockProperties);
-        if (ZSTDv05_isError(cBlockSize)) return cBlockSize;
+        if (ZSTDv05_isError(cBlockSize)) {
+            ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
+            return;
+        }
 
         ip += ZSTDv05_blockHeaderSize;
         remainingSize -= ZSTDv05_blockHeaderSize;
-        if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
+        if (cBlockSize > remainingSize) {
+            ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
+            return;
+        }
 
         if (cBlockSize == 0) break;   /* bt_end */
 
         ip += cBlockSize;
         remainingSize -= cBlockSize;
+        nbBlocks++;
     }
 
-    return ip - (const BYTE*)src;
+    *cSize = ip - (const BYTE*)src;
+    *dBound = nbBlocks * BLOCKSIZE;
 }
 
 /* ******************************
diff --git a/lib/legacy/zstd_v05.h b/lib/legacy/zstd_v05.h
index b68fd578ee9..3798c509254 100644
--- a/lib/legacy/zstd_v05.h
+++ b/lib/legacy/zstd_v05.h
@@ -33,13 +33,14 @@ extern "C" {
 size_t ZSTDv05_decompress( void* dst, size_t dstCapacity,
                      const void* src, size_t compressedSize);
 
-/**
-ZSTDv05_getFrameSrcSize() : get the source length of a ZSTD frame
-    compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
-    return : the number of bytes that would be read to decompress this frame
-             or an errorCode if it fails (which can be tested using ZSTDv05_isError())
-*/
-size_t ZSTDv05_findFrameCompressedSize(const void* src, size_t compressedSize);
+ /**
+ ZSTDv05_getFrameSrcSize() : get the source length of a ZSTD frame
+     compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
+     return : the number of bytes that would be read to decompress this frame
+              or an errorCode if it fails (which can be tested using ZSTDv05_isError())
+ */
+void ZSTDv05_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
+                                     size_t* cSize, unsigned long long* dBound);
 
 /* *************************************
 *  Helper functions
diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c
index 60d8d6fd9a9..33a2b7dafdc 100644
--- a/lib/legacy/zstd_v06.c
+++ b/lib/legacy/zstd_v06.c
@@ -506,6 +506,9 @@ typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
 #define FSEv06_ENCODING_STATIC  2
 #define FSEv06_ENCODING_DYNAMIC 3
 
+#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
+#define ZSTD_CONTENTSIZE_ERROR   (0ULL - 2)
+
 static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                                       1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12,
                                      13,14,15,16 };
@@ -3654,36 +3657,61 @@ size_t ZSTDv06_decompress(void* dst, size_t dstCapacity, const void* src, size_t
 #endif
 }
 
-size_t ZSTDv06_findFrameCompressedSize(const void* src, size_t srcSize)
+MEM_STATIC void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
+{
+    *cSize = ret;
+    *dBound = ZSTD_CONTENTSIZE_ERROR;
+}
+
+void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
+                                     size_t* cSize, unsigned long long* dBound)
 {
     const BYTE* ip = (const BYTE*)src;
     size_t remainingSize = srcSize;
+    size_t nbBlocks = 0;
     blockProperties_t blockProperties = { bt_compressed, 0 };
 
     /* Frame Header */
     {   size_t const frameHeaderSize = ZSTDv06_frameHeaderSize(src, ZSTDv06_frameHeaderSize_min);
-        if (ZSTDv06_isError(frameHeaderSize)) return frameHeaderSize;
-        if (MEM_readLE32(src) != ZSTDv06_MAGICNUMBER) return ERROR(prefix_unknown);
-        if (srcSize < frameHeaderSize+ZSTDv06_blockHeaderSize) return ERROR(srcSize_wrong);
+        if (ZSTDv06_isError(frameHeaderSize)) {
+            ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, frameHeaderSize);
+            return;
+        }
+        if (MEM_readLE32(src) != ZSTDv06_MAGICNUMBER) {
+            ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
+            return;
+        }
+        if (srcSize < frameHeaderSize+ZSTDv06_blockHeaderSize) {
+            ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
+            return;
+        }
         ip += frameHeaderSize; remainingSize -= frameHeaderSize;
     }
 
     /* Loop on each block */
     while (1) {
         size_t const cBlockSize = ZSTDv06_getcBlockSize(ip, remainingSize, &blockProperties);
-        if (ZSTDv06_isError(cBlockSize)) return cBlockSize;
+        if (ZSTDv06_isError(cBlockSize)) {
+            ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
+            return;
+        }
 
         ip += ZSTDv06_blockHeaderSize;
         remainingSize -= ZSTDv06_blockHeaderSize;
-        if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
+        if (cBlockSize > remainingSize) {
+            ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
+            return;
+        }
 
         if (cBlockSize == 0) break;   /* bt_end */
 
         ip += cBlockSize;
         remainingSize -= cBlockSize;
+        nbBlocks++;
     }
 
-    return ip - (const BYTE*)src;
+    *cSize = ip - (const BYTE*)src;
+    *dBound = nbBlocks * ZSTDv06_BLOCKSIZE_MAX;
 }
 
 /*_******************************
diff --git a/lib/legacy/zstd_v06.h b/lib/legacy/zstd_v06.h
index fb4eb37c89e..d3305e02b64 100644
--- a/lib/legacy/zstd_v06.h
+++ b/lib/legacy/zstd_v06.h
@@ -48,7 +48,8 @@ ZSTDv06_getFrameSrcSize() : get the source length of a ZSTD frame
     return : the number of bytes that would be read to decompress this frame
              or an errorCode if it fails (which can be tested using ZSTDv06_isError())
 */
-size_t ZSTDv06_findFrameCompressedSize(const void* src, size_t compressedSize);
+void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
+                                     size_t* cSize, unsigned long long* dBound);
 
 /* *************************************
 *  Helper functions
diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c
index c7bb7a52981..9ad8b07d3fb 100644
--- a/lib/legacy/zstd_v07.c
+++ b/lib/legacy/zstd_v07.c
@@ -2740,6 +2740,9 @@ typedef enum { lbt_huffman, lbt_repeat, lbt_raw, lbt_rle } litBlockType_t;
 #define FSEv07_ENCODING_STATIC  2
 #define FSEv07_ENCODING_DYNAMIC 3
 
+#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
+#define ZSTD_CONTENTSIZE_ERROR   (0ULL - 2)
+
 static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                                       1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12,
                                      13,14,15,16 };
@@ -3895,19 +3898,39 @@ size_t ZSTDv07_decompress(void* dst, size_t dstCapacity, const void* src, size_t
 #endif
 }
 
-size_t ZSTDv07_findFrameCompressedSize(const void* src, size_t srcSize)
+MEM_STATIC void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
+{
+    *cSize = ret;
+    *dBound = ZSTD_CONTENTSIZE_ERROR;
+}
+
+void ZSTDv07_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
+                                     size_t* cSize, unsigned long long* dBound)
 {
     const BYTE* ip = (const BYTE*)src;
     size_t remainingSize = srcSize;
+    size_t nbBlocks = 0;
 
     /* check */
-    if (srcSize < ZSTDv07_frameHeaderSize_min+ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong);
+    if (srcSize < ZSTDv07_frameHeaderSize_min+ZSTDv07_blockHeaderSize) {
+        ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
+        return;
+    }
 
     /* Frame Header */
     {   size_t const frameHeaderSize = ZSTDv07_frameHeaderSize(src, ZSTDv07_frameHeaderSize_min);
-        if (ZSTDv07_isError(frameHeaderSize)) return frameHeaderSize;
-        if (MEM_readLE32(src) != ZSTDv07_MAGICNUMBER) return ERROR(prefix_unknown);
-        if (srcSize < frameHeaderSize+ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong);
+        if (ZSTDv07_isError(frameHeaderSize)) {
+            ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, frameHeaderSize);
+            return;
+        }
+        if (MEM_readLE32(src) != ZSTDv07_MAGICNUMBER) {
+            ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
+            return;
+        }
+        if (srcSize < frameHeaderSize+ZSTDv07_blockHeaderSize) {
+            ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
+            return;
+        }
         ip += frameHeaderSize; remainingSize -= frameHeaderSize;
     }
 
@@ -3915,20 +3938,28 @@ size_t ZSTDv07_findFrameCompressedSize(const void* src, size_t srcSize)
     while (1) {
         blockProperties_t blockProperties;
         size_t const cBlockSize = ZSTDv07_getcBlockSize(ip, remainingSize, &blockProperties);
-        if (ZSTDv07_isError(cBlockSize)) return cBlockSize;
+        if (ZSTDv07_isError(cBlockSize)) {
+            ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
+            return;
+        }
 
         ip += ZSTDv07_blockHeaderSize;
         remainingSize -= ZSTDv07_blockHeaderSize;
 
         if (blockProperties.blockType == bt_end) break;
 
-        if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
+        if (cBlockSize > remainingSize) {
+            ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
+            return;
+        }
 
         ip += cBlockSize;
         remainingSize -= cBlockSize;
+        nbBlocks++;
     }
 
-    return ip - (const BYTE*)src;
+    *cSize = ip - (const BYTE*)src;
+    *dBound = nbBlocks * ZSTDv07_BLOCKSIZE_ABSOLUTEMAX;
 }
 
 /*_******************************
diff --git a/lib/legacy/zstd_v07.h b/lib/legacy/zstd_v07.h
index 6591cd3014b..8543e0b6a06 100644
--- a/lib/legacy/zstd_v07.h
+++ b/lib/legacy/zstd_v07.h
@@ -55,7 +55,8 @@ ZSTDv07_getFrameSrcSize() : get the source length of a ZSTD frame
     return : the number of bytes that would be read to decompress this frame
              or an errorCode if it fails (which can be tested using ZSTDv07_isError())
 */
-size_t ZSTDv07_findFrameCompressedSize(const void* src, size_t compressedSize);
+void ZSTDv07_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
+                                     size_t* cSize, unsigned long long* dBound);
 
 /*======  Helper functions  ======*/
 ZSTDLIBv07_API unsigned    ZSTDv07_isError(size_t code);          /*!< tells if a `size_t` function result is an error code */
diff --git a/lib/zstd.h b/lib/zstd.h
index 4373d6dfd36..e9e5eec0faf 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -1085,6 +1085,17 @@ typedef enum {
 *  Frame size functions
 ***************************************/
 
+/**
+ * Contains the compressed frame size and an upper-bound for the decompressed frame size.
+ * Note: before using `compressedSize` you must check for errors using ZSTD_isError().
+ *       similarly, before using `decompressedBound`, you must check for errors using:
+ *          `decompressedBound` != ZSTD_CONTENTSIZE_ERROR
+ */
+typedef struct {
+    size_t compressedSize;
+    unsigned long long decompressedBound;
+} ZSTD_frameSizeInfo;
+
 /*! ZSTD_findDecompressedSize() :
  *  `src` should point to the start of a series of ZSTD encoded and/or skippable frames
  *  `srcSize` must be the _exact_ size of this series
diff --git a/tests/legacy.c b/tests/legacy.c
index e660b9cc6d3..20f6e2830bf 100644
--- a/tests/legacy.c
+++ b/tests/legacy.c
@@ -133,8 +133,8 @@ static int testStreamingAPI(void)
 
 static int testFrameDecoding(void)
 {
-    if (ZSTD_decompressBound(COMPRESSED, COMPRESSED_SIZE) != ZSTD_CONTENTSIZE_ERROR) {
-        DISPLAY("ERROR: ZSTD_decompressBound: Expected to receive ZSTD_CONTENTSIZE_ERROR\n");
+    if (strlen(EXPECTED) > ZSTD_decompressBound(COMPRESSED, COMPRESSED_SIZE)) {
+        DISPLAY("ERROR: ZSTD_decompressBound: decompressed bound too small\n");
         return 1;
     }
     DISPLAY("Frame Decoding OK\n");

From 8cd423a659b576c2b293b9e12d12a933fec2aa86 Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Fri, 15 Mar 2019 16:20:34 -0700
Subject: [PATCH 097/178] Reorder declaration in ZSTD_findFrameSizeInfoLegacy

---
 lib/legacy/zstd_legacy.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/legacy/zstd_legacy.h b/lib/legacy/zstd_legacy.h
index 306a87399e3..f0fbb6cebb4 100644
--- a/lib/legacy/zstd_legacy.h
+++ b/lib/legacy/zstd_legacy.h
@@ -186,9 +186,9 @@ MEM_STATIC void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long*
 
 MEM_STATIC ZSTD_frameSizeInfo ZSTD_findFrameSizeInfoLegacy(const void *src, size_t srcSize)
 {
+    U32 const version = ZSTD_isLegacy(src, srcSize);
     ZSTD_frameSizeInfo frameSizeInfo;
     memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo));
-    U32 const version = ZSTD_isLegacy(src, srcSize);
     switch(version)
     {
 #if (ZSTD_LEGACY_SUPPORT <= 1)

From 19b75b6ecb3ecc133c5e2e859b39afa23fbfc244 Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Fri, 15 Mar 2019 18:04:19 -0700
Subject: [PATCH 098/178] Test new ZSTD_findFrameCompressedSize and update
 documentation

---
 lib/common/zstd_internal.h | 11 +++++++++++
 lib/legacy/zstd_legacy.h   | 12 +++---------
 lib/legacy/zstd_v01.h      | 10 ++++++----
 lib/legacy/zstd_v02.h      | 10 ++++++----
 lib/legacy/zstd_v03.h      | 10 ++++++----
 lib/legacy/zstd_v04.h      | 10 ++++++----
 lib/legacy/zstd_v05.h      | 10 ++++++----
 lib/legacy/zstd_v06.h      | 10 ++++++----
 lib/legacy/zstd_v07.h      | 10 ++++++----
 lib/zstd.h                 | 11 -----------
 tests/legacy.c             | 17 +++++++++++++++++
 11 files changed, 73 insertions(+), 48 deletions(-)

diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index c4e2647a24f..f5de27a1998 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -242,6 +242,17 @@ typedef struct {
     U32   longLengthPos;
 } seqStore_t;
 
+/**
+ * Contains the compressed frame size and an upper-bound for the decompressed frame size.
+ * Note: before using `compressedSize` you must check for errors using ZSTD_isError().
+ *       similarly, before using `decompressedBound`, you must check for errors using:
+ *          `decompressedBound` != ZSTD_CONTENTSIZE_ERROR
+ */
+typedef struct {
+    size_t compressedSize;
+    unsigned long long decompressedBound;
+} ZSTD_frameSizeInfo;
+
 const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);   /* compress & dictBuilder */
 void ZSTD_seqToCodes(const seqStore_t* seqStorePtr);   /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */
 
diff --git a/lib/legacy/zstd_legacy.h b/lib/legacy/zstd_legacy.h
index f0fbb6cebb4..966c6a2c510 100644
--- a/lib/legacy/zstd_legacy.h
+++ b/lib/legacy/zstd_legacy.h
@@ -20,7 +20,7 @@ extern "C" {
 ***************************************/
 #include "mem.h"            /* MEM_STATIC */
 #include "error_private.h"  /* ERROR */
-#include "zstd.h"           /* ZSTD_inBuffer, ZSTD_outBuffer */
+#include "zstd_internal.h"  /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTD_frameSizeInfo */
 
 #if !defined (ZSTD_LEGACY_SUPPORT) || (ZSTD_LEGACY_SUPPORT == 0)
 #  undef ZSTD_LEGACY_SUPPORT
@@ -178,12 +178,6 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
     }
 }
 
-MEM_STATIC void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
-{
-    *cSize = ret;
-    *dBound = ZSTD_CONTENTSIZE_ERROR;
-}
-
 MEM_STATIC ZSTD_frameSizeInfo ZSTD_findFrameSizeInfoLegacy(const void *src, size_t srcSize)
 {
     U32 const version = ZSTD_isLegacy(src, srcSize);
@@ -234,8 +228,8 @@ MEM_STATIC ZSTD_frameSizeInfo ZSTD_findFrameSizeInfoLegacy(const void *src, size
             break;
 #endif
         default :
-            ZSTD_errorFrameSizeInfoLegacy(&frameSizeInfo.compressedSize,
-                &frameSizeInfo.decompressedBound, ERROR(prefix_unknown));
+            frameSizeInfo.compressedSize = ERROR(prefix_unknown);
+            frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
             break;
     }
     return frameSizeInfo;
diff --git a/lib/legacy/zstd_v01.h b/lib/legacy/zstd_v01.h
index c56a2f0c70a..bcc3545fddb 100644
--- a/lib/legacy/zstd_v01.h
+++ b/lib/legacy/zstd_v01.h
@@ -36,10 +36,12 @@ size_t ZSTDv01_decompress( void* dst, size_t maxOriginalSize,
                      const void* src, size_t compressedSize);
 
  /**
- ZSTDv01_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.1.x format
-     compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
-     return : the number of bytes that would be read to decompress this frame
-              or an errorCode if it fails (which can be tested using ZSTDv01_isError())
+ ZSTDv01_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.1.x format
+     srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
+     cSize (output parameter) : the number of bytes that would be read to decompress this frame
+                                or an errorCode if it fails (which can be tested using ZSTDv01_isError())
+     dBound (output paramter) : an upper-bound for the decompressed size of the data in the frame
+                                or ZSTD_CONTENTSIZE_ERROR if an error occurs
  */
 void ZSTDv01_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
                                      size_t* cSize, unsigned long long* dBound);
diff --git a/lib/legacy/zstd_v02.h b/lib/legacy/zstd_v02.h
index 5f7cba5246c..5e6911f8a70 100644
--- a/lib/legacy/zstd_v02.h
+++ b/lib/legacy/zstd_v02.h
@@ -36,10 +36,12 @@ size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
                      const void* src, size_t compressedSize);
 
  /**
- ZSTDv02_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.2.x format
-     compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
-     return : the number of bytes that would be read to decompress this frame
-              or an errorCode if it fails (which can be tested using ZSTDv02_isError())
+ ZSTDv02_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.2.x format
+     srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
+     cSize (output parameter) : the number of bytes that would be read to decompress this frame
+                                or an errorCode if it fails (which can be tested using ZSTDv01_isError())
+     dBound (output paramter) : an upper-bound for the decompressed size of the data in the frame
+                                or ZSTD_CONTENTSIZE_ERROR if an error occurs
  */
 void ZSTDv02_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
                                      size_t* cSize, unsigned long long* dBound);
diff --git a/lib/legacy/zstd_v03.h b/lib/legacy/zstd_v03.h
index 36fc6119050..6bde907f00c 100644
--- a/lib/legacy/zstd_v03.h
+++ b/lib/legacy/zstd_v03.h
@@ -36,10 +36,12 @@ size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
                      const void* src, size_t compressedSize);
 
  /**
- ZSTDv03_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.3.x format
-     compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
-     return : the number of bytes that would be read to decompress this frame
-              or an errorCode if it fails (which can be tested using ZSTDv03_isError())
+ ZSTDv03_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.3.x format
+     srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
+     cSize (output parameter) : the number of bytes that would be read to decompress this frame
+                                or an errorCode if it fails (which can be tested using ZSTDv01_isError())
+     dBound (output paramter) : an upper-bound for the decompressed size of the data in the frame
+                                or ZSTD_CONTENTSIZE_ERROR if an error occurs
  */
  void ZSTDv03_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
                                       size_t* cSize, unsigned long long* dBound);
diff --git a/lib/legacy/zstd_v04.h b/lib/legacy/zstd_v04.h
index 1da32c6c935..11b3c00b3a2 100644
--- a/lib/legacy/zstd_v04.h
+++ b/lib/legacy/zstd_v04.h
@@ -36,10 +36,12 @@ size_t ZSTDv04_decompress( void* dst, size_t maxOriginalSize,
                      const void* src, size_t compressedSize);
 
  /**
- ZSTDv04_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.4.x format
-     compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
-     return : the number of bytes that would be read to decompress this frame
-              or an errorCode if it fails (which can be tested using ZSTDv04_isError())
+ ZSTDv04_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.4.x format
+     srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
+     cSize (output parameter) : the number of bytes that would be read to decompress this frame
+                                or an errorCode if it fails (which can be tested using ZSTDv01_isError())
+     dBound (output paramter) : an upper-bound for the decompressed size of the data in the frame
+                                or ZSTD_CONTENTSIZE_ERROR if an error occurs
  */
  void ZSTDv04_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
                                       size_t* cSize, unsigned long long* dBound);
diff --git a/lib/legacy/zstd_v05.h b/lib/legacy/zstd_v05.h
index 3798c509254..2d95084b6e9 100644
--- a/lib/legacy/zstd_v05.h
+++ b/lib/legacy/zstd_v05.h
@@ -34,10 +34,12 @@ size_t ZSTDv05_decompress( void* dst, size_t dstCapacity,
                      const void* src, size_t compressedSize);
 
  /**
- ZSTDv05_getFrameSrcSize() : get the source length of a ZSTD frame
-     compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
-     return : the number of bytes that would be read to decompress this frame
-              or an errorCode if it fails (which can be tested using ZSTDv05_isError())
+ ZSTDv05_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.5.x format
+     srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
+     cSize (output parameter) : the number of bytes that would be read to decompress this frame
+                                or an errorCode if it fails (which can be tested using ZSTDv01_isError())
+     dBound (output paramter) : an upper-bound for the decompressed size of the data in the frame
+                                or ZSTD_CONTENTSIZE_ERROR if an error occurs
  */
 void ZSTDv05_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
                                      size_t* cSize, unsigned long long* dBound);
diff --git a/lib/legacy/zstd_v06.h b/lib/legacy/zstd_v06.h
index d3305e02b64..71d3b6af118 100644
--- a/lib/legacy/zstd_v06.h
+++ b/lib/legacy/zstd_v06.h
@@ -43,10 +43,12 @@ ZSTDLIBv06_API size_t ZSTDv06_decompress( void* dst, size_t dstCapacity,
                                     const void* src, size_t compressedSize);
 
 /**
-ZSTDv06_getFrameSrcSize() : get the source length of a ZSTD frame
-    compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
-    return : the number of bytes that would be read to decompress this frame
-             or an errorCode if it fails (which can be tested using ZSTDv06_isError())
+ZSTDv06_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.6.x format
+    srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
+    cSize (output parameter) : the number of bytes that would be read to decompress this frame
+                               or an errorCode if it fails (which can be tested using ZSTDv01_isError())
+    dBound (output paramter) : an upper-bound for the decompressed size of the data in the frame
+                               or ZSTD_CONTENTSIZE_ERROR if an error occurs
 */
 void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
                                      size_t* cSize, unsigned long long* dBound);
diff --git a/lib/legacy/zstd_v07.h b/lib/legacy/zstd_v07.h
index 8543e0b6a06..912ff7c5e11 100644
--- a/lib/legacy/zstd_v07.h
+++ b/lib/legacy/zstd_v07.h
@@ -50,10 +50,12 @@ ZSTDLIBv07_API size_t ZSTDv07_decompress( void* dst, size_t dstCapacity,
                                     const void* src, size_t compressedSize);
 
 /**
-ZSTDv07_getFrameSrcSize() : get the source length of a ZSTD frame
-    compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
-    return : the number of bytes that would be read to decompress this frame
-             or an errorCode if it fails (which can be tested using ZSTDv07_isError())
+ZSTDv07_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.7.x format
+    srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
+    cSize (output parameter) : the number of bytes that would be read to decompress this frame
+                               or an errorCode if it fails (which can be tested using ZSTDv01_isError())
+    dBound (output paramter) : an upper-bound for the decompressed size of the data in the frame
+                               or ZSTD_CONTENTSIZE_ERROR if an error occurs
 */
 void ZSTDv07_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
                                      size_t* cSize, unsigned long long* dBound);
diff --git a/lib/zstd.h b/lib/zstd.h
index e9e5eec0faf..4373d6dfd36 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -1085,17 +1085,6 @@ typedef enum {
 *  Frame size functions
 ***************************************/
 
-/**
- * Contains the compressed frame size and an upper-bound for the decompressed frame size.
- * Note: before using `compressedSize` you must check for errors using ZSTD_isError().
- *       similarly, before using `decompressedBound`, you must check for errors using:
- *          `decompressedBound` != ZSTD_CONTENTSIZE_ERROR
- */
-typedef struct {
-    size_t compressedSize;
-    unsigned long long decompressedBound;
-} ZSTD_frameSizeInfo;
-
 /*! ZSTD_findDecompressedSize() :
  *  `src` should point to the start of a series of ZSTD encoded and/or skippable frames
  *  `srcSize` must be the _exact_ size of this series
diff --git a/tests/legacy.c b/tests/legacy.c
index 20f6e2830bf..eb329203833 100644
--- a/tests/legacy.c
+++ b/tests/legacy.c
@@ -137,6 +137,23 @@ static int testFrameDecoding(void)
         DISPLAY("ERROR: ZSTD_decompressBound: decompressed bound too small\n");
         return 1;
     }
+    {   const char* ip = COMPRESSED;
+        size_t remainingSize = COMPRESSED_SIZE;
+        while (1) {
+            size_t frameSize = ZSTD_findFrameCompressedSize(ip, remainingSize);
+            if (ZSTD_isError(frameSize)) {
+                DISPLAY("ERROR: ZSTD_findFrameCompressedSize: %s\n", ZSTD_getErrorName(frameSize));
+                return 1;
+            }
+            if (frameSize > remainingSize) {
+                DISPLAY("ERROR: ZSTD_findFrameCompressedSize: expected frameSize to align with src buffer");
+                return 1;
+            }
+            ip += frameSize;
+            remainingSize -= frameSize;
+            if (remainingSize == 0) break;
+        }
+    }
     DISPLAY("Frame Decoding OK\n");
     return 0;
 }

From 0033bb4785debbd9fc916cab34aad54cb105f4c2 Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Sun, 17 Mar 2019 17:41:27 -0700
Subject: [PATCH 099/178] Update documentation for ZSTD_frameSizeInfo

---
 lib/common/zstd_internal.h | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index f5de27a1998..31f756ab581 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -244,14 +244,14 @@ typedef struct {
 
 /**
  * Contains the compressed frame size and an upper-bound for the decompressed frame size.
- * Note: before using `compressedSize` you must check for errors using ZSTD_isError().
- *       similarly, before using `decompressedBound`, you must check for errors using:
- *          `decompressedBound` != ZSTD_CONTENTSIZE_ERROR
+ * Note: before using `compressedSize`, check for errors using ZSTD_isError().
+ *       similarly, before using `decompressedBound`, check for errors using:
+ *          `decompressedBound != ZSTD_CONTENTSIZE_ERROR`
  */
 typedef struct {
     size_t compressedSize;
     unsigned long long decompressedBound;
-} ZSTD_frameSizeInfo;
+} ZSTD_frameSizeInfo;   /* decompress & legacy */
 
 const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);   /* compress & dictBuilder */
 void ZSTD_seqToCodes(const seqStore_t* seqStorePtr);   /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */

From 20aa1b455c2546fcb26a798622c0387d6a4e3810 Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Sun, 17 Mar 2019 19:35:43 -0700
Subject: [PATCH 100/178] Stylistic changes

---
 lib/decompress/zstd_decompress.c |  3 +--
 lib/legacy/zstd_legacy.h         | 21 ++++++++++++++-------
 lib/legacy/zstd_v01.c            |  4 +---
 lib/legacy/zstd_v02.c            |  4 +---
 lib/legacy/zstd_v03.c            |  4 +---
 lib/legacy/zstd_v04.c            |  4 +---
 lib/legacy/zstd_v05.c            |  4 +---
 lib/legacy/zstd_v06.c            |  4 +---
 lib/legacy/zstd_v07.c            |  4 +---
 9 files changed, 22 insertions(+), 30 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index a981af0a16e..7870bae5082 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -447,9 +447,8 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize
     memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo));
 
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
-    if (ZSTD_isLegacy(src, srcSize)) {
+    if (ZSTD_isLegacy(src, srcSize))
         return ZSTD_findFrameSizeInfoLegacy(src, srcSize);
-    }
 #endif
 
     if ((srcSize >= ZSTD_SKIPPABLEHEADERSIZE)
diff --git a/lib/legacy/zstd_legacy.h b/lib/legacy/zstd_legacy.h
index 966c6a2c510..4eed6afa464 100644
--- a/lib/legacy/zstd_legacy.h
+++ b/lib/legacy/zstd_legacy.h
@@ -188,43 +188,50 @@ MEM_STATIC ZSTD_frameSizeInfo ZSTD_findFrameSizeInfoLegacy(const void *src, size
 #if (ZSTD_LEGACY_SUPPORT <= 1)
         case 1 :
             ZSTDv01_findFrameSizeInfoLegacy(src, srcSize,
-                &frameSizeInfo.compressedSize, &frameSizeInfo.decompressedBound);
+                &frameSizeInfo.compressedSize,
+                &frameSizeInfo.decompressedBound);
             break;
 #endif
 #if (ZSTD_LEGACY_SUPPORT <= 2)
         case 2 :
             ZSTDv02_findFrameSizeInfoLegacy(src, srcSize,
-                &frameSizeInfo.compressedSize, &frameSizeInfo.decompressedBound);
+                &frameSizeInfo.compressedSize,
+                &frameSizeInfo.decompressedBound);
             break;
 #endif
 #if (ZSTD_LEGACY_SUPPORT <= 3)
         case 3 :
             ZSTDv03_findFrameSizeInfoLegacy(src, srcSize,
-                &frameSizeInfo.compressedSize, &frameSizeInfo.decompressedBound);
+                &frameSizeInfo.compressedSize,
+                &frameSizeInfo.decompressedBound);
             break;
 #endif
 #if (ZSTD_LEGACY_SUPPORT <= 4)
         case 4 :
             ZSTDv04_findFrameSizeInfoLegacy(src, srcSize,
-                &frameSizeInfo.compressedSize, &frameSizeInfo.decompressedBound);
+                &frameSizeInfo.compressedSize,
+                &frameSizeInfo.decompressedBound);
             break;
 #endif
 #if (ZSTD_LEGACY_SUPPORT <= 5)
         case 5 :
             ZSTDv05_findFrameSizeInfoLegacy(src, srcSize,
-                &frameSizeInfo.compressedSize, &frameSizeInfo.decompressedBound);
+                &frameSizeInfo.compressedSize,
+                &frameSizeInfo.decompressedBound);
             break;
 #endif
 #if (ZSTD_LEGACY_SUPPORT <= 6)
         case 6 :
             ZSTDv06_findFrameSizeInfoLegacy(src, srcSize,
-                &frameSizeInfo.compressedSize, &frameSizeInfo.decompressedBound);
+                &frameSizeInfo.compressedSize,
+                &frameSizeInfo.decompressedBound);
             break;
 #endif
 #if (ZSTD_LEGACY_SUPPORT <= 7)
         case 7 :
             ZSTDv07_findFrameSizeInfoLegacy(src, srcSize,
-                &frameSizeInfo.compressedSize, &frameSizeInfo.decompressedBound);
+                &frameSizeInfo.compressedSize,
+                &frameSizeInfo.decompressedBound);
             break;
 #endif
         default :
diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c
index 1d2d34f18a2..0f77533fcf2 100644
--- a/lib/legacy/zstd_v01.c
+++ b/lib/legacy/zstd_v01.c
@@ -1336,7 +1336,6 @@ static const U32 ZSTD_magicNumber = 0xFD2FB51E;   /* 3rd version : seqNb header
 #define LITERAL_NOENTROPY 63
 #define COMMAND_NOENTROPY 7   /* to remove */
 
-#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
 #define ZSTD_CONTENTSIZE_ERROR   (0ULL - 2)
 
 static const size_t ZSTD_blockHeaderSize = 3;
@@ -2008,8 +2007,7 @@ static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBo
     *dBound = ZSTD_CONTENTSIZE_ERROR;
 }
 
-void ZSTDv01_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
-                                     size_t* cSize, unsigned long long* dBound)
+void ZSTDv01_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
 {
     const BYTE* ip = (const BYTE*)src;
     size_t remainingSize = srcSize;
diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c
index 06496101c09..1d7c206496b 100644
--- a/lib/legacy/zstd_v02.c
+++ b/lib/legacy/zstd_v02.c
@@ -2728,7 +2728,6 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
 #define LITERAL_NOENTROPY 63
 #define COMMAND_NOENTROPY 7   /* to remove */
 
-#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
 #define ZSTD_CONTENTSIZE_ERROR   (0ULL - 2)
 
 static const size_t ZSTD_blockHeaderSize = 3;
@@ -3321,8 +3320,7 @@ MEM_STATIC void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long*
     *dBound = ZSTD_CONTENTSIZE_ERROR;
 }
 
-void ZSTDv02_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
-                                     size_t* cSize, unsigned long long* dBound)
+void ZSTDv02_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
 {
     const BYTE* ip = (const BYTE*)src;
     size_t remainingSize = srcSize;
diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c
index 27c1b67bbe7..2aac45d187f 100644
--- a/lib/legacy/zstd_v03.c
+++ b/lib/legacy/zstd_v03.c
@@ -2369,7 +2369,6 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
 #define LITERAL_NOENTROPY 63
 #define COMMAND_NOENTROPY 7   /* to remove */
 
-#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
 #define ZSTD_CONTENTSIZE_ERROR   (0ULL - 2)
 
 static const size_t ZSTD_blockHeaderSize = 3;
@@ -2962,8 +2961,7 @@ MEM_STATIC void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long*
     *dBound = ZSTD_CONTENTSIZE_ERROR;
 }
 
-void ZSTDv03_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
-                                     size_t* cSize, unsigned long long* dBound)
+void ZSTDv03_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
 {
     const BYTE* ip = (const BYTE*)src;
     size_t remainingSize = srcSize;
diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c
index 81994d1f320..22de6b94da6 100644
--- a/lib/legacy/zstd_v04.c
+++ b/lib/legacy/zstd_v04.c
@@ -373,7 +373,6 @@ static const size_t ZSTD_frameHeaderSize_min = 5;
 #define MIN_SEQUENCES_SIZE (2 /*seqNb*/ + 2 /*dumps*/ + 3 /*seqTables*/ + 1 /*bitStream*/)
 #define MIN_CBLOCK_SIZE (3 /*litCSize*/ + MIN_SEQUENCES_SIZE)
 
-#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
 #define ZSTD_CONTENTSIZE_ERROR   (0ULL - 2)
 
 typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
@@ -3128,8 +3127,7 @@ MEM_STATIC void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long*
     *dBound = ZSTD_CONTENTSIZE_ERROR;
 }
 
-void ZSTDv04_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
-                                     size_t* cSize, unsigned long long* dBound)
+void ZSTDv04_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
 {
     const BYTE* ip = (const BYTE*)src;
     size_t remainingSize = srcSize;
diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c
index ff1095ca96d..469466b14c8 100644
--- a/lib/legacy/zstd_v05.c
+++ b/lib/legacy/zstd_v05.c
@@ -491,7 +491,6 @@ static const size_t ZSTDv05_frameHeaderSize_min = 5;
 
 #define WILDCOPY_OVERLENGTH 8
 
-#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
 #define ZSTD_CONTENTSIZE_ERROR   (0ULL - 2)
 
 typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
@@ -3517,8 +3516,7 @@ MEM_STATIC void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long*
     *dBound = ZSTD_CONTENTSIZE_ERROR;
 }
 
-void ZSTDv05_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
-                                     size_t* cSize, unsigned long long* dBound)
+void ZSTDv05_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
 {
     const BYTE* ip = (const BYTE*)src;
     size_t remainingSize = srcSize;
diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c
index 33a2b7dafdc..2daea0e9148 100644
--- a/lib/legacy/zstd_v06.c
+++ b/lib/legacy/zstd_v06.c
@@ -506,7 +506,6 @@ typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
 #define FSEv06_ENCODING_STATIC  2
 #define FSEv06_ENCODING_DYNAMIC 3
 
-#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
 #define ZSTD_CONTENTSIZE_ERROR   (0ULL - 2)
 
 static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -3663,8 +3662,7 @@ MEM_STATIC void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long*
     *dBound = ZSTD_CONTENTSIZE_ERROR;
 }
 
-void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
-                                     size_t* cSize, unsigned long long* dBound)
+void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
 {
     const BYTE* ip = (const BYTE*)src;
     size_t remainingSize = srcSize;
diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c
index 9ad8b07d3fb..89fe7d5aae5 100644
--- a/lib/legacy/zstd_v07.c
+++ b/lib/legacy/zstd_v07.c
@@ -2740,7 +2740,6 @@ typedef enum { lbt_huffman, lbt_repeat, lbt_raw, lbt_rle } litBlockType_t;
 #define FSEv07_ENCODING_STATIC  2
 #define FSEv07_ENCODING_DYNAMIC 3
 
-#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
 #define ZSTD_CONTENTSIZE_ERROR   (0ULL - 2)
 
 static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -3904,8 +3903,7 @@ MEM_STATIC void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long*
     *dBound = ZSTD_CONTENTSIZE_ERROR;
 }
 
-void ZSTDv07_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
-                                     size_t* cSize, unsigned long long* dBound)
+void ZSTDv07_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
 {
     const BYTE* ip = (const BYTE*)src;
     size_t remainingSize = srcSize;

From 0a3fa6f9095299488c1ebf39b081d1cae67b56eb Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Mon, 18 Mar 2019 20:33:15 -0700
Subject: [PATCH 101/178] Add legacy mode in documentation

---
 lib/decompress/zstd_decompress.c | 2 +-
 lib/legacy/zstd_v01.c            | 2 ++
 lib/legacy/zstd_v01.h            | 2 ++
 lib/legacy/zstd_v02.c            | 4 +++-
 lib/legacy/zstd_v02.h            | 2 ++
 lib/legacy/zstd_v03.c            | 2 ++
 lib/legacy/zstd_v03.h            | 3 +++
 lib/legacy/zstd_v04.c            | 4 +++-
 lib/legacy/zstd_v04.h            | 2 ++
 lib/legacy/zstd_v05.c            | 4 +++-
 lib/legacy/zstd_v05.h            | 3 +++
 lib/legacy/zstd_v06.c            | 4 +++-
 lib/legacy/zstd_v06.h            | 3 +++
 lib/legacy/zstd_v07.c            | 4 +++-
 lib/legacy/zstd_v07.h            | 3 +++
 lib/zstd.h                       | 2 +-
 16 files changed, 39 insertions(+), 7 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 7870bae5082..142923fe4bc 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -518,7 +518,7 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
 
 
 /** ZSTD_decompressBound() :
- *  currently incompatible with legacy mode
+ *  compatible with legacy mode
  *  `src` must point to the start of a ZSTD frame or a skippeable frame
  *  `srcSize` must be at least as large as the frame contained
  *  @return : the maximum decompressed size of the compressed source
diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c
index 0f77533fcf2..bb0f4b59321 100644
--- a/lib/legacy/zstd_v01.c
+++ b/lib/legacy/zstd_v01.c
@@ -2001,6 +2001,8 @@ size_t ZSTDv01_decompress(void* dst, size_t maxDstSize, const void* src, size_t
     return ZSTDv01_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
 }
 
+/* ZSTD_errorFrameSizeInfoLegacy() :
+   assumes `cSize` and `dBound` are _not_ NULL */
 static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
 {
     *cSize = ret;
diff --git a/lib/legacy/zstd_v01.h b/lib/legacy/zstd_v01.h
index bcc3545fddb..8329429a322 100644
--- a/lib/legacy/zstd_v01.h
+++ b/lib/legacy/zstd_v01.h
@@ -42,6 +42,8 @@ size_t ZSTDv01_decompress( void* dst, size_t maxOriginalSize,
                                 or an errorCode if it fails (which can be tested using ZSTDv01_isError())
      dBound (output paramter) : an upper-bound for the decompressed size of the data in the frame
                                 or ZSTD_CONTENTSIZE_ERROR if an error occurs
+
+     note : assumes `cSize` and `dBound` are _not_ NULL.
  */
 void ZSTDv01_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
                                      size_t* cSize, unsigned long long* dBound);
diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c
index 1d7c206496b..594835943f4 100644
--- a/lib/legacy/zstd_v02.c
+++ b/lib/legacy/zstd_v02.c
@@ -3314,7 +3314,9 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
     return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
 }
 
-MEM_STATIC void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
+/* ZSTD_errorFrameSizeInfoLegacy() :
+   assumes `cSize` and `dBound` are _not_ NULL */
+static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
 {
     *cSize = ret;
     *dBound = ZSTD_CONTENTSIZE_ERROR;
diff --git a/lib/legacy/zstd_v02.h b/lib/legacy/zstd_v02.h
index 5e6911f8a70..556d509e33d 100644
--- a/lib/legacy/zstd_v02.h
+++ b/lib/legacy/zstd_v02.h
@@ -42,6 +42,8 @@ size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
                                 or an errorCode if it fails (which can be tested using ZSTDv01_isError())
      dBound (output paramter) : an upper-bound for the decompressed size of the data in the frame
                                 or ZSTD_CONTENTSIZE_ERROR if an error occurs
+
+    note : assumes `cSize` and `dBound` are _not_ NULL.
  */
 void ZSTDv02_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
                                      size_t* cSize, unsigned long long* dBound);
diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c
index 2aac45d187f..b6c60d296c8 100644
--- a/lib/legacy/zstd_v03.c
+++ b/lib/legacy/zstd_v03.c
@@ -2955,6 +2955,8 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
     return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
 }
 
+/* ZSTD_errorFrameSizeInfoLegacy() :
+   assumes `cSize` and `dBound` are _not_ NULL */
 MEM_STATIC void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
 {
     *cSize = ret;
diff --git a/lib/legacy/zstd_v03.h b/lib/legacy/zstd_v03.h
index 6bde907f00c..c2d9f191fc6 100644
--- a/lib/legacy/zstd_v03.h
+++ b/lib/legacy/zstd_v03.h
@@ -42,6 +42,9 @@ size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
                                 or an errorCode if it fails (which can be tested using ZSTDv01_isError())
      dBound (output paramter) : an upper-bound for the decompressed size of the data in the frame
                                 or ZSTD_CONTENTSIZE_ERROR if an error occurs
+
+    note : assumes `cSize` and `dBound` are _not_ NULL.
+
  */
  void ZSTDv03_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
                                       size_t* cSize, unsigned long long* dBound);
diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c
index 22de6b94da6..65dc64dbbab 100644
--- a/lib/legacy/zstd_v04.c
+++ b/lib/legacy/zstd_v04.c
@@ -3121,7 +3121,9 @@ static size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx,
     return op-ostart;
 }
 
-MEM_STATIC void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
+/* ZSTD_errorFrameSizeInfoLegacy() :
+   assumes `cSize` and `dBound` are _not_ NULL */
+static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
 {
     *cSize = ret;
     *dBound = ZSTD_CONTENTSIZE_ERROR;
diff --git a/lib/legacy/zstd_v04.h b/lib/legacy/zstd_v04.h
index 11b3c00b3a2..84a3e7d7012 100644
--- a/lib/legacy/zstd_v04.h
+++ b/lib/legacy/zstd_v04.h
@@ -42,6 +42,8 @@ size_t ZSTDv04_decompress( void* dst, size_t maxOriginalSize,
                                 or an errorCode if it fails (which can be tested using ZSTDv01_isError())
      dBound (output paramter) : an upper-bound for the decompressed size of the data in the frame
                                 or ZSTD_CONTENTSIZE_ERROR if an error occurs
+
+    note : assumes `cSize` and `dBound` are _not_ NULL.
  */
  void ZSTDv04_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
                                       size_t* cSize, unsigned long long* dBound);
diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c
index 469466b14c8..1c39f2f2682 100644
--- a/lib/legacy/zstd_v05.c
+++ b/lib/legacy/zstd_v05.c
@@ -3510,7 +3510,9 @@ size_t ZSTDv05_decompress(void* dst, size_t maxDstSize, const void* src, size_t
 #endif
 }
 
-MEM_STATIC void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
+/* ZSTD_errorFrameSizeInfoLegacy() :
+   assumes `cSize` and `dBound` are _not_ NULL */
+static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
 {
     *cSize = ret;
     *dBound = ZSTD_CONTENTSIZE_ERROR;
diff --git a/lib/legacy/zstd_v05.h b/lib/legacy/zstd_v05.h
index 2d95084b6e9..3b866083a3d 100644
--- a/lib/legacy/zstd_v05.h
+++ b/lib/legacy/zstd_v05.h
@@ -40,6 +40,9 @@ size_t ZSTDv05_decompress( void* dst, size_t dstCapacity,
                                 or an errorCode if it fails (which can be tested using ZSTDv01_isError())
      dBound (output paramter) : an upper-bound for the decompressed size of the data in the frame
                                 or ZSTD_CONTENTSIZE_ERROR if an error occurs
+
+    note : assumes `cSize` and `dBound` are _not_ NULL.
+
  */
 void ZSTDv05_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
                                      size_t* cSize, unsigned long long* dBound);
diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c
index 2daea0e9148..65975ac2933 100644
--- a/lib/legacy/zstd_v06.c
+++ b/lib/legacy/zstd_v06.c
@@ -3656,7 +3656,9 @@ size_t ZSTDv06_decompress(void* dst, size_t dstCapacity, const void* src, size_t
 #endif
 }
 
-MEM_STATIC void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
+/* ZSTD_errorFrameSizeInfoLegacy() :
+   assumes `cSize` and `dBound` are _not_ NULL */
+static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
 {
     *cSize = ret;
     *dBound = ZSTD_CONTENTSIZE_ERROR;
diff --git a/lib/legacy/zstd_v06.h b/lib/legacy/zstd_v06.h
index 71d3b6af118..971429c9b96 100644
--- a/lib/legacy/zstd_v06.h
+++ b/lib/legacy/zstd_v06.h
@@ -49,6 +49,9 @@ ZSTDv06_findFrameSizeInfoLegacy() : get the source length and decompressed bound
                                or an errorCode if it fails (which can be tested using ZSTDv01_isError())
     dBound (output paramter) : an upper-bound for the decompressed size of the data in the frame
                                or ZSTD_CONTENTSIZE_ERROR if an error occurs
+
+    note : assumes `cSize` and `dBound` are _not_ NULL.
+
 */
 void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
                                      size_t* cSize, unsigned long long* dBound);
diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c
index 89fe7d5aae5..443524b3ac2 100644
--- a/lib/legacy/zstd_v07.c
+++ b/lib/legacy/zstd_v07.c
@@ -3897,7 +3897,9 @@ size_t ZSTDv07_decompress(void* dst, size_t dstCapacity, const void* src, size_t
 #endif
 }
 
-MEM_STATIC void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
+/* ZSTD_errorFrameSizeInfoLegacy() :
+   assumes `cSize` and `dBound` are _not_ NULL */
+static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
 {
     *cSize = ret;
     *dBound = ZSTD_CONTENTSIZE_ERROR;
diff --git a/lib/legacy/zstd_v07.h b/lib/legacy/zstd_v07.h
index 912ff7c5e11..c8144bd61bb 100644
--- a/lib/legacy/zstd_v07.h
+++ b/lib/legacy/zstd_v07.h
@@ -56,6 +56,9 @@ ZSTDv07_findFrameSizeInfoLegacy() : get the source length and decompressed bound
                                or an errorCode if it fails (which can be tested using ZSTDv01_isError())
     dBound (output paramter) : an upper-bound for the decompressed size of the data in the frame
                                or ZSTD_CONTENTSIZE_ERROR if an error occurs
+
+    note : assumes `cSize` and `dBound` are _not_ NULL.
+
 */
 void ZSTDv07_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
                                      size_t* cSize, unsigned long long* dBound);
diff --git a/lib/zstd.h b/lib/zstd.h
index 4373d6dfd36..f6332589d6b 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -1115,7 +1115,7 @@ ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t
  *  @return : - upper-bound for the decompressed size of all data in all successive frames
  *            - if an error occured: ZSTD_CONTENTSIZE_ERROR
  *
- *  note 1  : an error can occur if `src` contains a legacy frame or an invalid/incorrectly formatted frame.
+ *  note 1  : an error can occur if `src` contains an invalid or incorrectly formatted frame.
  *  note 2  : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`.
  *            in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value.
  *  note 3  : when the decompressed size field isn't available, the upper-bound for that frame is calculated by:

From 5740eb67696f23cf4277253cd6047fb1e506d470 Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Mon, 18 Mar 2019 21:05:35 -0700
Subject: [PATCH 102/178] Remove extraneous spacing in comments

---
 lib/legacy/zstd_v03.h | 1 -
 lib/legacy/zstd_v05.h | 1 -
 lib/legacy/zstd_v06.h | 1 -
 lib/legacy/zstd_v07.h | 1 -
 4 files changed, 4 deletions(-)

diff --git a/lib/legacy/zstd_v03.h b/lib/legacy/zstd_v03.h
index c2d9f191fc6..3c9ca730e48 100644
--- a/lib/legacy/zstd_v03.h
+++ b/lib/legacy/zstd_v03.h
@@ -44,7 +44,6 @@ size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
                                 or ZSTD_CONTENTSIZE_ERROR if an error occurs
 
     note : assumes `cSize` and `dBound` are _not_ NULL.
-
  */
  void ZSTDv03_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
                                       size_t* cSize, unsigned long long* dBound);
diff --git a/lib/legacy/zstd_v05.h b/lib/legacy/zstd_v05.h
index 3b866083a3d..03b49c79677 100644
--- a/lib/legacy/zstd_v05.h
+++ b/lib/legacy/zstd_v05.h
@@ -42,7 +42,6 @@ size_t ZSTDv05_decompress( void* dst, size_t dstCapacity,
                                 or ZSTD_CONTENTSIZE_ERROR if an error occurs
 
     note : assumes `cSize` and `dBound` are _not_ NULL.
-
  */
 void ZSTDv05_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
                                      size_t* cSize, unsigned long long* dBound);
diff --git a/lib/legacy/zstd_v06.h b/lib/legacy/zstd_v06.h
index 971429c9b96..930633d79e8 100644
--- a/lib/legacy/zstd_v06.h
+++ b/lib/legacy/zstd_v06.h
@@ -51,7 +51,6 @@ ZSTDv06_findFrameSizeInfoLegacy() : get the source length and decompressed bound
                                or ZSTD_CONTENTSIZE_ERROR if an error occurs
 
     note : assumes `cSize` and `dBound` are _not_ NULL.
-
 */
 void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
                                      size_t* cSize, unsigned long long* dBound);
diff --git a/lib/legacy/zstd_v07.h b/lib/legacy/zstd_v07.h
index c8144bd61bb..fc4fecba9fc 100644
--- a/lib/legacy/zstd_v07.h
+++ b/lib/legacy/zstd_v07.h
@@ -58,7 +58,6 @@ ZSTDv07_findFrameSizeInfoLegacy() : get the source length and decompressed bound
                                or ZSTD_CONTENTSIZE_ERROR if an error occurs
 
     note : assumes `cSize` and `dBound` are _not_ NULL.
-
 */
 void ZSTDv07_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
                                      size_t* cSize, unsigned long long* dBound);

From 186ded6d911dd6ae2469b161ffc92b5795470531 Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Tue, 19 Mar 2019 01:44:08 -0700
Subject: [PATCH 103/178] Fix typo in legacy documentation

---
 lib/legacy/zstd_legacy.h | 3 +--
 lib/legacy/zstd_v01.h    | 8 ++++----
 lib/legacy/zstd_v02.h    | 8 ++++----
 lib/legacy/zstd_v03.h    | 8 ++++----
 lib/legacy/zstd_v04.h    | 8 ++++----
 lib/legacy/zstd_v05.h    | 8 ++++----
 lib/legacy/zstd_v06.h    | 8 ++++----
 lib/legacy/zstd_v07.h    | 8 ++++----
 8 files changed, 29 insertions(+), 30 deletions(-)

diff --git a/lib/legacy/zstd_legacy.h b/lib/legacy/zstd_legacy.h
index 4eed6afa464..e5b383ee4c0 100644
--- a/lib/legacy/zstd_legacy.h
+++ b/lib/legacy/zstd_legacy.h
@@ -180,9 +180,8 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
 
 MEM_STATIC ZSTD_frameSizeInfo ZSTD_findFrameSizeInfoLegacy(const void *src, size_t srcSize)
 {
-    U32 const version = ZSTD_isLegacy(src, srcSize);
     ZSTD_frameSizeInfo frameSizeInfo;
-    memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo));
+    U32 const version = ZSTD_isLegacy(src, srcSize);
     switch(version)
     {
 #if (ZSTD_LEGACY_SUPPORT <= 1)
diff --git a/lib/legacy/zstd_v01.h b/lib/legacy/zstd_v01.h
index 8329429a322..245f9dd3146 100644
--- a/lib/legacy/zstd_v01.h
+++ b/lib/legacy/zstd_v01.h
@@ -38,10 +38,10 @@ size_t ZSTDv01_decompress( void* dst, size_t maxOriginalSize,
  /**
  ZSTDv01_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.1.x format
      srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
-     cSize (output parameter) : the number of bytes that would be read to decompress this frame
-                                or an errorCode if it fails (which can be tested using ZSTDv01_isError())
-     dBound (output paramter) : an upper-bound for the decompressed size of the data in the frame
-                                or ZSTD_CONTENTSIZE_ERROR if an error occurs
+     cSize (output parameter)  : the number of bytes that would be read to decompress this frame
+                                 or an error code if it fails (which can be tested using ZSTDv01_isError())
+     dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
+                                 or ZSTD_CONTENTSIZE_ERROR if an error occurs
 
      note : assumes `cSize` and `dBound` are _not_ NULL.
  */
diff --git a/lib/legacy/zstd_v02.h b/lib/legacy/zstd_v02.h
index 556d509e33d..9d7d8d9b5bc 100644
--- a/lib/legacy/zstd_v02.h
+++ b/lib/legacy/zstd_v02.h
@@ -38,10 +38,10 @@ size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
  /**
  ZSTDv02_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.2.x format
      srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
-     cSize (output parameter) : the number of bytes that would be read to decompress this frame
-                                or an errorCode if it fails (which can be tested using ZSTDv01_isError())
-     dBound (output paramter) : an upper-bound for the decompressed size of the data in the frame
-                                or ZSTD_CONTENTSIZE_ERROR if an error occurs
+     cSize (output parameter)  : the number of bytes that would be read to decompress this frame
+                                 or an error code if it fails (which can be tested using ZSTDv01_isError())
+     dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
+                                 or ZSTD_CONTENTSIZE_ERROR if an error occurs
 
     note : assumes `cSize` and `dBound` are _not_ NULL.
  */
diff --git a/lib/legacy/zstd_v03.h b/lib/legacy/zstd_v03.h
index 3c9ca730e48..efd8c2b9241 100644
--- a/lib/legacy/zstd_v03.h
+++ b/lib/legacy/zstd_v03.h
@@ -38,10 +38,10 @@ size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
  /**
  ZSTDv03_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.3.x format
      srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
-     cSize (output parameter) : the number of bytes that would be read to decompress this frame
-                                or an errorCode if it fails (which can be tested using ZSTDv01_isError())
-     dBound (output paramter) : an upper-bound for the decompressed size of the data in the frame
-                                or ZSTD_CONTENTSIZE_ERROR if an error occurs
+     cSize (output parameter)  : the number of bytes that would be read to decompress this frame
+                                 or an error code if it fails (which can be tested using ZSTDv01_isError())
+     dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
+                                 or ZSTD_CONTENTSIZE_ERROR if an error occurs
 
     note : assumes `cSize` and `dBound` are _not_ NULL.
  */
diff --git a/lib/legacy/zstd_v04.h b/lib/legacy/zstd_v04.h
index 84a3e7d7012..bb5f3b7d0b8 100644
--- a/lib/legacy/zstd_v04.h
+++ b/lib/legacy/zstd_v04.h
@@ -38,10 +38,10 @@ size_t ZSTDv04_decompress( void* dst, size_t maxOriginalSize,
  /**
  ZSTDv04_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.4.x format
      srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
-     cSize (output parameter) : the number of bytes that would be read to decompress this frame
-                                or an errorCode if it fails (which can be tested using ZSTDv01_isError())
-     dBound (output paramter) : an upper-bound for the decompressed size of the data in the frame
-                                or ZSTD_CONTENTSIZE_ERROR if an error occurs
+     cSize (output parameter)  : the number of bytes that would be read to decompress this frame
+                                 or an error code if it fails (which can be tested using ZSTDv01_isError())
+     dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
+                                 or ZSTD_CONTENTSIZE_ERROR if an error occurs
 
     note : assumes `cSize` and `dBound` are _not_ NULL.
  */
diff --git a/lib/legacy/zstd_v05.h b/lib/legacy/zstd_v05.h
index 03b49c79677..4a979854b36 100644
--- a/lib/legacy/zstd_v05.h
+++ b/lib/legacy/zstd_v05.h
@@ -36,10 +36,10 @@ size_t ZSTDv05_decompress( void* dst, size_t dstCapacity,
  /**
  ZSTDv05_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.5.x format
      srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
-     cSize (output parameter) : the number of bytes that would be read to decompress this frame
-                                or an errorCode if it fails (which can be tested using ZSTDv01_isError())
-     dBound (output paramter) : an upper-bound for the decompressed size of the data in the frame
-                                or ZSTD_CONTENTSIZE_ERROR if an error occurs
+     cSize (output parameter)  : the number of bytes that would be read to decompress this frame
+                                 or an error code if it fails (which can be tested using ZSTDv01_isError())
+     dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
+                                 or ZSTD_CONTENTSIZE_ERROR if an error occurs
 
     note : assumes `cSize` and `dBound` are _not_ NULL.
  */
diff --git a/lib/legacy/zstd_v06.h b/lib/legacy/zstd_v06.h
index 930633d79e8..07818571dca 100644
--- a/lib/legacy/zstd_v06.h
+++ b/lib/legacy/zstd_v06.h
@@ -45,10 +45,10 @@ ZSTDLIBv06_API size_t ZSTDv06_decompress( void* dst, size_t dstCapacity,
 /**
 ZSTDv06_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.6.x format
     srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
-    cSize (output parameter) : the number of bytes that would be read to decompress this frame
-                               or an errorCode if it fails (which can be tested using ZSTDv01_isError())
-    dBound (output paramter) : an upper-bound for the decompressed size of the data in the frame
-                               or ZSTD_CONTENTSIZE_ERROR if an error occurs
+    cSize (output parameter)  : the number of bytes that would be read to decompress this frame
+                                or an error code if it fails (which can be tested using ZSTDv01_isError())
+    dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
+                                or ZSTD_CONTENTSIZE_ERROR if an error occurs
 
     note : assumes `cSize` and `dBound` are _not_ NULL.
 */
diff --git a/lib/legacy/zstd_v07.h b/lib/legacy/zstd_v07.h
index fc4fecba9fc..a566c1d102a 100644
--- a/lib/legacy/zstd_v07.h
+++ b/lib/legacy/zstd_v07.h
@@ -52,10 +52,10 @@ ZSTDLIBv07_API size_t ZSTDv07_decompress( void* dst, size_t dstCapacity,
 /**
 ZSTDv07_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.7.x format
     srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
-    cSize (output parameter) : the number of bytes that would be read to decompress this frame
-                               or an errorCode if it fails (which can be tested using ZSTDv01_isError())
-    dBound (output paramter) : an upper-bound for the decompressed size of the data in the frame
-                               or ZSTD_CONTENTSIZE_ERROR if an error occurs
+    cSize (output parameter)  : the number of bytes that would be read to decompress this frame
+                                or an error code if it fails (which can be tested using ZSTDv01_isError())
+    dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
+                                or ZSTD_CONTENTSIZE_ERROR if an error occurs
 
     note : assumes `cSize` and `dBound` are _not_ NULL.
 */

From 11e73576bb54dea193f1ba6bc3e29ba5146fd11d Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Wed, 20 Mar 2019 21:20:14 -0700
Subject: [PATCH 104/178] [regression] Add more streaming tests

* Test all of the `ZSTD_initCStream*()` variants.
* Fix a typo in the zstdcli method.
---
 tests/regression/config.c    |   2 +-
 tests/regression/method.c    | 217 ++++++++++++++++++-------
 tests/regression/results.csv | 306 ++++++++++++++++++++++++++++++++++-
 3 files changed, 465 insertions(+), 60 deletions(-)

diff --git a/tests/regression/config.c b/tests/regression/config.c
index 262cb605dc8..b82482f4612 100644
--- a/tests/regression/config.c
+++ b/tests/regression/config.c
@@ -97,7 +97,7 @@ static param_value_t mt_advanced_param_values[] = {
 
 static config_t mt_advanced = {
     .name = "multithreaded with advanced params",
-    .cli_args = "-T2 --no-compressed-literals",
+    .cli_args = "-T2 --no-compress-literals",
     .param_values = PARAM_VALUES(mt_advanced_param_values),
 };
 
diff --git a/tests/regression/method.c b/tests/regression/method.c
index 16701c0b440..1e84021c3ef 100644
--- a/tests/regression/method.c
+++ b/tests/regression/method.c
@@ -432,77 +432,158 @@ static result_t advanced_streaming_compress(
     return result;
 }
 
-static result_t old_streaming_compress(
-    method_state_t* base,
-    config_t const* config) {
-    buffer_state_t* state = container_of(base, buffer_state_t, base);
-
-    if (buffer_state_bad(state, config))
-        return result_error(result_error_system_error);
-
-    int const level = config_get_level(config);
-    if (level == CONFIG_NO_LEVEL)
-        return result_error(result_error_skip);
-
-    ZSTD_CStream* zcs = ZSTD_createCStream();
-    result_t result;
-    if (zcs == NULL) {
-        result = result_error(result_error_compression_error);
-        goto out;
-    }
+static int init_cstream(
+    buffer_state_t* state,
+    ZSTD_CStream* zcs,
+    config_t const* config,
+    int const advanced,
+    ZSTD_CDict** cdict)
+{
     size_t zret;
-    if (config->use_dictionary) {
-        zret = ZSTD_initCStream_usingDict(
-            zcs, state->dictionary.data, state->dictionary.size, level);
+    if (advanced) {
+        ZSTD_parameters const params = config_get_zstd_params(config, 0, 0);
+        ZSTD_CDict* dict = NULL;
+        if (cdict) {
+            *cdict = ZSTD_createCDict_advanced(
+                state->dictionary.data,
+                state->dictionary.size,
+                ZSTD_dlm_byRef,
+                ZSTD_dct_auto,
+                params.cParams,
+                ZSTD_defaultCMem);
+            if (!*cdict) {
+                return 1;
+            }
+            zret = ZSTD_initCStream_usingCDict_advanced(
+                zcs, *cdict, params.fParams, ZSTD_CONTENTSIZE_UNKNOWN);
+        } else {
+            zret = ZSTD_initCStream_advanced(
+                zcs,
+                state->dictionary.data,
+                state->dictionary.size,
+                params,
+                ZSTD_CONTENTSIZE_UNKNOWN);
+        }
     } else {
-        zret = ZSTD_initCStream(zcs, level);
+        int const level = config_get_level(config);
+        if (cdict) {
+            *cdict = ZSTD_createCDict(
+                state->dictionary.data,
+                state->dictionary.size,
+                level);
+            if (!*cdict) {
+                return 1;
+            }
+            zret = ZSTD_initCStream_usingCDict(zcs, *cdict);
+        } else if (config->use_dictionary) {
+            zret = ZSTD_initCStream_usingDict(
+                zcs, state->dictionary.data, state->dictionary.size, level);
+        } else {
+            zret = ZSTD_initCStream(zcs, level);
+        }
     }
     if (ZSTD_isError(zret)) {
-        result = result_error(result_error_compression_error);
-        goto out;
+        return 1;
     }
+    return 0;
+}
 
-    result_data_t data = {.total_size = 0};
-    for (size_t i = 0; i < state->inputs.size; ++i) {
-        data_buffer_t input = state->inputs.buffers[i];
-        zret = ZSTD_resetCStream(
-            zcs,
-            config->no_pledged_src_size ? ZSTD_CONTENTSIZE_UNKNOWN
-                                        : input.size);
-        if (ZSTD_isError(zret)) {
-            result = result_error(result_error_compression_error);
-            goto out;
-        }
+static result_t old_streaming_compress_internal(
+    method_state_t* base,
+    config_t const* config,
+    int const advanced,
+    int const cdict) {
+  buffer_state_t* state = container_of(base, buffer_state_t, base);
+
+  if (buffer_state_bad(state, config))
+    return result_error(result_error_system_error);
+
+
+  ZSTD_CStream* zcs = ZSTD_createCStream();
+  ZSTD_CDict* cd = NULL;
+  result_t result;
+  if (zcs == NULL) {
+    result = result_error(result_error_compression_error);
+    goto out;
+  }
+  if (init_cstream(state, zcs, config, advanced, cdict ? &cd : NULL)) {
+      result = result_error(result_error_compression_error);
+      goto out;
+  }
+
+  result_data_t data = {.total_size = 0};
+  for (size_t i = 0; i < state->inputs.size; ++i) {
+    data_buffer_t input = state->inputs.buffers[i];
+    size_t zret = ZSTD_resetCStream(
+        zcs,
+        config->no_pledged_src_size ? ZSTD_CONTENTSIZE_UNKNOWN : input.size);
+    if (ZSTD_isError(zret)) {
+      result = result_error(result_error_compression_error);
+      goto out;
+    }
 
-        while (input.size > 0) {
-            ZSTD_inBuffer in = {input.data, MIN(input.size, 4096)};
-            input.data += in.size;
-            input.size -= in.size;
-            ZSTD_EndDirective const op =
-                input.size > 0 ? ZSTD_e_continue : ZSTD_e_end;
-            zret = 0;
-            while (in.pos < in.size || (op == ZSTD_e_end && zret != 0)) {
-                ZSTD_outBuffer out = {state->compressed.data,
-                                      MIN(state->compressed.capacity, 1024)};
-                if (op == ZSTD_e_continue || in.pos < in.size)
-                    zret = ZSTD_compressStream(zcs, &out, &in);
-                else
-                    zret = ZSTD_endStream(zcs, &out);
-                if (ZSTD_isError(zret)) {
-                    result = result_error(result_error_compression_error);
-                    goto out;
-                }
-                data.total_size += out.pos;
-            }
+    while (input.size > 0) {
+      ZSTD_inBuffer in = {input.data, MIN(input.size, 4096)};
+      input.data += in.size;
+      input.size -= in.size;
+      ZSTD_EndDirective const op =
+          input.size > 0 ? ZSTD_e_continue : ZSTD_e_end;
+      zret = 0;
+      while (in.pos < in.size || (op == ZSTD_e_end && zret != 0)) {
+        ZSTD_outBuffer out = {state->compressed.data,
+                              MIN(state->compressed.capacity, 1024)};
+        if (op == ZSTD_e_continue || in.pos < in.size)
+          zret = ZSTD_compressStream(zcs, &out, &in);
+        else
+          zret = ZSTD_endStream(zcs, &out);
+        if (ZSTD_isError(zret)) {
+          result = result_error(result_error_compression_error);
+          goto out;
         }
+        data.total_size += out.pos;
+      }
     }
+  }
 
-    result = result_data(data);
+  result = result_data(data);
 out:
     ZSTD_freeCStream(zcs);
+    ZSTD_freeCDict(cd);
     return result;
 }
 
+static result_t old_streaming_compress(
+    method_state_t* base,
+    config_t const* config)
+{
+    return old_streaming_compress_internal(
+        base, config, /* advanced */ 0, /* cdict */ 0);
+}
+
+static result_t old_streaming_compress_advanced(
+    method_state_t* base,
+    config_t const* config)
+{
+    return old_streaming_compress_internal(
+        base, config, /* advanced */ 1, /* cdict */ 0);
+}
+
+static result_t old_streaming_compress_cdict(
+    method_state_t* base,
+    config_t const* config)
+{
+    return old_streaming_compress_internal(
+        base, config, /* advanced */ 0, /* cdict */ 1);
+}
+
+static result_t old_streaming_compress_cdict_advanced(
+    method_state_t* base,
+    config_t const* config)
+{
+    return old_streaming_compress_internal(
+        base, config, /* advanced */ 1, /* cdict */ 1);
+}
+
 method_t const simple = {
     .name = "compress simple",
     .create = buffer_state_create,
@@ -545,6 +626,27 @@ method_t const old_streaming = {
     .destroy = buffer_state_destroy,
 };
 
+method_t const old_streaming_advanced = {
+    .name = "old streaming advanced",
+    .create = buffer_state_create,
+    .compress = old_streaming_compress,
+    .destroy = buffer_state_destroy,
+};
+
+method_t const old_streaming_cdict = {
+    .name = "old streaming cdcit",
+    .create = buffer_state_create,
+    .compress = old_streaming_compress,
+    .destroy = buffer_state_destroy,
+};
+
+method_t const old_streaming_advanced_cdict = {
+    .name = "old streaming advanced cdict",
+    .create = buffer_state_create,
+    .compress = old_streaming_compress,
+    .destroy = buffer_state_destroy,
+};
+
 method_t const cli = {
     .name = "zstdcli",
     .create = method_state_create,
@@ -560,6 +662,9 @@ static method_t const* g_methods[] = {
     &advanced_one_pass_small_out,
     &advanced_streaming,
     &old_streaming,
+    &old_streaming_advanced,
+    &old_streaming_cdict,
+    &old_streaming_advanced_cdict,
     NULL,
 };
 
diff --git a/tests/regression/results.csv b/tests/regression/results.csv
index a24553b018f..f5c63932039 100644
--- a/tests/regression/results.csv
+++ b/tests/regression/results.csv
@@ -104,7 +104,7 @@ silesia,                            explicit params,                    zstdcli,
 silesia,                            uncompressed literals,              zstdcli,                            5155472
 silesia,                            uncompressed literals optimal,      zstdcli,                            4325475
 silesia,                            huffman literals,                   zstdcli,                            5341405
-silesia,                            multithreaded with advanced params, zstdcli,                            compression error
+silesia,                            multithreaded with advanced params, zstdcli,                            5155472
 silesia.tar,                        level -5,                           zstdcli,                            7161160
 silesia.tar,                        level -3,                           zstdcli,                            6789865
 silesia.tar,                        level -1,                           zstdcli,                            6196433
@@ -130,7 +130,7 @@ silesia.tar,                        explicit params,                    zstdcli,
 silesia.tar,                        uncompressed literals,              zstdcli,                            5158134
 silesia.tar,                        uncompressed literals optimal,      zstdcli,                            4321098
 silesia.tar,                        huffman literals,                   zstdcli,                            5358479
-silesia.tar,                        multithreaded with advanced params, zstdcli,                            compression error
+silesia.tar,                        multithreaded with advanced params, zstdcli,                            5158134
 github,                             level -5,                           zstdcli,                            234744
 github,                             level -5 with dict,                 zstdcli,                            48718
 github,                             level -3,                           zstdcli,                            222611
@@ -169,7 +169,7 @@ github,                             explicit params,                    zstdcli,
 github,                             uncompressed literals,              zstdcli,                            169004
 github,                             uncompressed literals optimal,      zstdcli,                            158824
 github,                             huffman literals,                   zstdcli,                            145457
-github,                             multithreaded with advanced params, zstdcli,                            compression error
+github,                             multithreaded with advanced params, zstdcli,                            169004
 silesia,                            level -5,                           advanced one pass,                  7152294
 silesia,                            level -3,                           advanced one pass,                  6789969
 silesia,                            level -1,                           advanced one pass,                  6191548
@@ -461,9 +461,17 @@ silesia,                            level 13,                           old stre
 silesia,                            level 16,                           old streaming,                      4377391
 silesia,                            level 19,                           old streaming,                      4293262
 silesia,                            no source size,                     old streaming,                      4862341
+silesia,                            long distance mode,                 old streaming,                      12000408
+silesia,                            multithreaded,                      old streaming,                      12000408
+silesia,                            multithreaded long distance mode,   old streaming,                      12000408
+silesia,                            small window log,                   old streaming,                      12000408
+silesia,                            small hash log,                     old streaming,                      12000408
+silesia,                            small chain log,                    old streaming,                      12000408
+silesia,                            explicit params,                    old streaming,                      12000408
 silesia,                            uncompressed literals,              old streaming,                      4862377
 silesia,                            uncompressed literals optimal,      old streaming,                      4293262
 silesia,                            huffman literals,                   old streaming,                      6191549
+silesia,                            multithreaded with advanced params, old streaming,                      12000408
 silesia.tar,                        level -5,                           old streaming,                      7160440
 silesia.tar,                        level -3,                           old streaming,                      6789026
 silesia.tar,                        level -1,                           old streaming,                      6195465
@@ -479,9 +487,17 @@ silesia.tar,                        level 13,                           old stre
 silesia.tar,                        level 16,                           old streaming,                      4381277
 silesia.tar,                        level 19,                           old streaming,                      4281514
 silesia.tar,                        no source size,                     old streaming,                      4875006
+silesia.tar,                        long distance mode,                 old streaming,                      12022046
+silesia.tar,                        multithreaded,                      old streaming,                      12022046
+silesia.tar,                        multithreaded long distance mode,   old streaming,                      12022046
+silesia.tar,                        small window log,                   old streaming,                      12022046
+silesia.tar,                        small hash log,                     old streaming,                      12022046
+silesia.tar,                        small chain log,                    old streaming,                      12022046
+silesia.tar,                        explicit params,                    old streaming,                      12022046
 silesia.tar,                        uncompressed literals,              old streaming,                      4875010
 silesia.tar,                        uncompressed literals optimal,      old streaming,                      4281514
 silesia.tar,                        huffman literals,                   old streaming,                      6195465
+silesia.tar,                        multithreaded with advanced params, old streaming,                      12022046
 github,                             level -5,                           old streaming,                      232744
 github,                             level -5 with dict,                 old streaming,                      46718
 github,                             level -3,                           old streaming,                      220611
@@ -511,6 +527,290 @@ github,                             level 16 with dict,                 old stre
 github,                             level 19,                           old streaming,                      133717
 github,                             level 19 with dict,                 old streaming,                      37576
 github,                             no source size,                     old streaming,                      141003
+github,                             long distance mode,                 old streaming,                      412933
+github,                             multithreaded,                      old streaming,                      412933
+github,                             multithreaded long distance mode,   old streaming,                      412933
+github,                             small window log,                   old streaming,                      412933
+github,                             small hash log,                     old streaming,                      412933
+github,                             small chain log,                    old streaming,                      412933
+github,                             explicit params,                    old streaming,                      412933
 github,                             uncompressed literals,              old streaming,                      136397
 github,                             uncompressed literals optimal,      old streaming,                      133717
 github,                             huffman literals,                   old streaming,                      176575
+github,                             multithreaded with advanced params, old streaming,                      412933
+silesia,                            level -5,                           old streaming advanced,             7152294
+silesia,                            level -3,                           old streaming advanced,             6789973
+silesia,                            level -1,                           old streaming advanced,             6191549
+silesia,                            level 0,                            old streaming advanced,             4862377
+silesia,                            level 1,                            old streaming advanced,             5318036
+silesia,                            level 3,                            old streaming advanced,             4862377
+silesia,                            level 4,                            old streaming advanced,             4800629
+silesia,                            level 5,                            old streaming advanced,             4710178
+silesia,                            level 6,                            old streaming advanced,             4659996
+silesia,                            level 7,                            old streaming advanced,             4596234
+silesia,                            level 9,                            old streaming advanced,             4543862
+silesia,                            level 13,                           old streaming advanced,             4482073
+silesia,                            level 16,                           old streaming advanced,             4377391
+silesia,                            level 19,                           old streaming advanced,             4293262
+silesia,                            no source size,                     old streaming advanced,             4862341
+silesia,                            long distance mode,                 old streaming advanced,             12000408
+silesia,                            multithreaded,                      old streaming advanced,             12000408
+silesia,                            multithreaded long distance mode,   old streaming advanced,             12000408
+silesia,                            small window log,                   old streaming advanced,             12000408
+silesia,                            small hash log,                     old streaming advanced,             12000408
+silesia,                            small chain log,                    old streaming advanced,             12000408
+silesia,                            explicit params,                    old streaming advanced,             12000408
+silesia,                            uncompressed literals,              old streaming advanced,             4862377
+silesia,                            uncompressed literals optimal,      old streaming advanced,             4293262
+silesia,                            huffman literals,                   old streaming advanced,             6191549
+silesia,                            multithreaded with advanced params, old streaming advanced,             12000408
+silesia.tar,                        level -5,                           old streaming advanced,             7160440
+silesia.tar,                        level -3,                           old streaming advanced,             6789026
+silesia.tar,                        level -1,                           old streaming advanced,             6195465
+silesia.tar,                        level 0,                            old streaming advanced,             4875010
+silesia.tar,                        level 1,                            old streaming advanced,             5339701
+silesia.tar,                        level 3,                            old streaming advanced,             4875010
+silesia.tar,                        level 4,                            old streaming advanced,             4813507
+silesia.tar,                        level 5,                            old streaming advanced,             4722240
+silesia.tar,                        level 6,                            old streaming advanced,             4672203
+silesia.tar,                        level 7,                            old streaming advanced,             4606658
+silesia.tar,                        level 9,                            old streaming advanced,             4554105
+silesia.tar,                        level 13,                           old streaming advanced,             4491703
+silesia.tar,                        level 16,                           old streaming advanced,             4381277
+silesia.tar,                        level 19,                           old streaming advanced,             4281514
+silesia.tar,                        no source size,                     old streaming advanced,             4875006
+silesia.tar,                        long distance mode,                 old streaming advanced,             12022046
+silesia.tar,                        multithreaded,                      old streaming advanced,             12022046
+silesia.tar,                        multithreaded long distance mode,   old streaming advanced,             12022046
+silesia.tar,                        small window log,                   old streaming advanced,             12022046
+silesia.tar,                        small hash log,                     old streaming advanced,             12022046
+silesia.tar,                        small chain log,                    old streaming advanced,             12022046
+silesia.tar,                        explicit params,                    old streaming advanced,             12022046
+silesia.tar,                        uncompressed literals,              old streaming advanced,             4875010
+silesia.tar,                        uncompressed literals optimal,      old streaming advanced,             4281514
+silesia.tar,                        huffman literals,                   old streaming advanced,             6195465
+silesia.tar,                        multithreaded with advanced params, old streaming advanced,             12022046
+github,                             level -5,                           old streaming advanced,             232744
+github,                             level -5 with dict,                 old streaming advanced,             46718
+github,                             level -3,                           old streaming advanced,             220611
+github,                             level -3 with dict,                 old streaming advanced,             45395
+github,                             level -1,                           old streaming advanced,             176575
+github,                             level -1 with dict,                 old streaming advanced,             43170
+github,                             level 0,                            old streaming advanced,             136397
+github,                             level 0 with dict,                  old streaming advanced,             41170
+github,                             level 1,                            old streaming advanced,             143457
+github,                             level 1 with dict,                  old streaming advanced,             41682
+github,                             level 3,                            old streaming advanced,             136397
+github,                             level 3 with dict,                  old streaming advanced,             41170
+github,                             level 4,                            old streaming advanced,             136144
+github,                             level 4 with dict,                  old streaming advanced,             41306
+github,                             level 5,                            old streaming advanced,             135106
+github,                             level 5 with dict,                  old streaming advanced,             38938
+github,                             level 6,                            old streaming advanced,             135108
+github,                             level 6 with dict,                  old streaming advanced,             38632
+github,                             level 7,                            old streaming advanced,             135108
+github,                             level 7 with dict,                  old streaming advanced,             38766
+github,                             level 9,                            old streaming advanced,             135108
+github,                             level 9 with dict,                  old streaming advanced,             39326
+github,                             level 13,                           old streaming advanced,             133717
+github,                             level 13 with dict,                 old streaming advanced,             39716
+github,                             level 16,                           old streaming advanced,             133717
+github,                             level 16 with dict,                 old streaming advanced,             37577
+github,                             level 19,                           old streaming advanced,             133717
+github,                             level 19 with dict,                 old streaming advanced,             37576
+github,                             no source size,                     old streaming advanced,             141003
+github,                             long distance mode,                 old streaming advanced,             412933
+github,                             multithreaded,                      old streaming advanced,             412933
+github,                             multithreaded long distance mode,   old streaming advanced,             412933
+github,                             small window log,                   old streaming advanced,             412933
+github,                             small hash log,                     old streaming advanced,             412933
+github,                             small chain log,                    old streaming advanced,             412933
+github,                             explicit params,                    old streaming advanced,             412933
+github,                             uncompressed literals,              old streaming advanced,             136397
+github,                             uncompressed literals optimal,      old streaming advanced,             133717
+github,                             huffman literals,                   old streaming advanced,             176575
+github,                             multithreaded with advanced params, old streaming advanced,             412933
+silesia,                            level -5,                           old streaming cdcit,                7152294
+silesia,                            level -3,                           old streaming cdcit,                6789973
+silesia,                            level -1,                           old streaming cdcit,                6191549
+silesia,                            level 0,                            old streaming cdcit,                4862377
+silesia,                            level 1,                            old streaming cdcit,                5318036
+silesia,                            level 3,                            old streaming cdcit,                4862377
+silesia,                            level 4,                            old streaming cdcit,                4800629
+silesia,                            level 5,                            old streaming cdcit,                4710178
+silesia,                            level 6,                            old streaming cdcit,                4659996
+silesia,                            level 7,                            old streaming cdcit,                4596234
+silesia,                            level 9,                            old streaming cdcit,                4543862
+silesia,                            level 13,                           old streaming cdcit,                4482073
+silesia,                            level 16,                           old streaming cdcit,                4377391
+silesia,                            level 19,                           old streaming cdcit,                4293262
+silesia,                            no source size,                     old streaming cdcit,                4862341
+silesia,                            long distance mode,                 old streaming cdcit,                12000408
+silesia,                            multithreaded,                      old streaming cdcit,                12000408
+silesia,                            multithreaded long distance mode,   old streaming cdcit,                12000408
+silesia,                            small window log,                   old streaming cdcit,                12000408
+silesia,                            small hash log,                     old streaming cdcit,                12000408
+silesia,                            small chain log,                    old streaming cdcit,                12000408
+silesia,                            explicit params,                    old streaming cdcit,                12000408
+silesia,                            uncompressed literals,              old streaming cdcit,                4862377
+silesia,                            uncompressed literals optimal,      old streaming cdcit,                4293262
+silesia,                            huffman literals,                   old streaming cdcit,                6191549
+silesia,                            multithreaded with advanced params, old streaming cdcit,                12000408
+silesia.tar,                        level -5,                           old streaming cdcit,                7160440
+silesia.tar,                        level -3,                           old streaming cdcit,                6789026
+silesia.tar,                        level -1,                           old streaming cdcit,                6195465
+silesia.tar,                        level 0,                            old streaming cdcit,                4875010
+silesia.tar,                        level 1,                            old streaming cdcit,                5339701
+silesia.tar,                        level 3,                            old streaming cdcit,                4875010
+silesia.tar,                        level 4,                            old streaming cdcit,                4813507
+silesia.tar,                        level 5,                            old streaming cdcit,                4722240
+silesia.tar,                        level 6,                            old streaming cdcit,                4672203
+silesia.tar,                        level 7,                            old streaming cdcit,                4606658
+silesia.tar,                        level 9,                            old streaming cdcit,                4554105
+silesia.tar,                        level 13,                           old streaming cdcit,                4491703
+silesia.tar,                        level 16,                           old streaming cdcit,                4381277
+silesia.tar,                        level 19,                           old streaming cdcit,                4281514
+silesia.tar,                        no source size,                     old streaming cdcit,                4875006
+silesia.tar,                        long distance mode,                 old streaming cdcit,                12022046
+silesia.tar,                        multithreaded,                      old streaming cdcit,                12022046
+silesia.tar,                        multithreaded long distance mode,   old streaming cdcit,                12022046
+silesia.tar,                        small window log,                   old streaming cdcit,                12022046
+silesia.tar,                        small hash log,                     old streaming cdcit,                12022046
+silesia.tar,                        small chain log,                    old streaming cdcit,                12022046
+silesia.tar,                        explicit params,                    old streaming cdcit,                12022046
+silesia.tar,                        uncompressed literals,              old streaming cdcit,                4875010
+silesia.tar,                        uncompressed literals optimal,      old streaming cdcit,                4281514
+silesia.tar,                        huffman literals,                   old streaming cdcit,                6195465
+silesia.tar,                        multithreaded with advanced params, old streaming cdcit,                12022046
+github,                             level -5,                           old streaming cdcit,                232744
+github,                             level -5 with dict,                 old streaming cdcit,                46718
+github,                             level -3,                           old streaming cdcit,                220611
+github,                             level -3 with dict,                 old streaming cdcit,                45395
+github,                             level -1,                           old streaming cdcit,                176575
+github,                             level -1 with dict,                 old streaming cdcit,                43170
+github,                             level 0,                            old streaming cdcit,                136397
+github,                             level 0 with dict,                  old streaming cdcit,                41170
+github,                             level 1,                            old streaming cdcit,                143457
+github,                             level 1 with dict,                  old streaming cdcit,                41682
+github,                             level 3,                            old streaming cdcit,                136397
+github,                             level 3 with dict,                  old streaming cdcit,                41170
+github,                             level 4,                            old streaming cdcit,                136144
+github,                             level 4 with dict,                  old streaming cdcit,                41306
+github,                             level 5,                            old streaming cdcit,                135106
+github,                             level 5 with dict,                  old streaming cdcit,                38938
+github,                             level 6,                            old streaming cdcit,                135108
+github,                             level 6 with dict,                  old streaming cdcit,                38632
+github,                             level 7,                            old streaming cdcit,                135108
+github,                             level 7 with dict,                  old streaming cdcit,                38766
+github,                             level 9,                            old streaming cdcit,                135108
+github,                             level 9 with dict,                  old streaming cdcit,                39326
+github,                             level 13,                           old streaming cdcit,                133717
+github,                             level 13 with dict,                 old streaming cdcit,                39716
+github,                             level 16,                           old streaming cdcit,                133717
+github,                             level 16 with dict,                 old streaming cdcit,                37577
+github,                             level 19,                           old streaming cdcit,                133717
+github,                             level 19 with dict,                 old streaming cdcit,                37576
+github,                             no source size,                     old streaming cdcit,                141003
+github,                             long distance mode,                 old streaming cdcit,                412933
+github,                             multithreaded,                      old streaming cdcit,                412933
+github,                             multithreaded long distance mode,   old streaming cdcit,                412933
+github,                             small window log,                   old streaming cdcit,                412933
+github,                             small hash log,                     old streaming cdcit,                412933
+github,                             small chain log,                    old streaming cdcit,                412933
+github,                             explicit params,                    old streaming cdcit,                412933
+github,                             uncompressed literals,              old streaming cdcit,                136397
+github,                             uncompressed literals optimal,      old streaming cdcit,                133717
+github,                             huffman literals,                   old streaming cdcit,                176575
+github,                             multithreaded with advanced params, old streaming cdcit,                412933
+silesia,                            level -5,                           old streaming advanced cdict,       7152294
+silesia,                            level -3,                           old streaming advanced cdict,       6789973
+silesia,                            level -1,                           old streaming advanced cdict,       6191549
+silesia,                            level 0,                            old streaming advanced cdict,       4862377
+silesia,                            level 1,                            old streaming advanced cdict,       5318036
+silesia,                            level 3,                            old streaming advanced cdict,       4862377
+silesia,                            level 4,                            old streaming advanced cdict,       4800629
+silesia,                            level 5,                            old streaming advanced cdict,       4710178
+silesia,                            level 6,                            old streaming advanced cdict,       4659996
+silesia,                            level 7,                            old streaming advanced cdict,       4596234
+silesia,                            level 9,                            old streaming advanced cdict,       4543862
+silesia,                            level 13,                           old streaming advanced cdict,       4482073
+silesia,                            level 16,                           old streaming advanced cdict,       4377391
+silesia,                            level 19,                           old streaming advanced cdict,       4293262
+silesia,                            no source size,                     old streaming advanced cdict,       4862341
+silesia,                            long distance mode,                 old streaming advanced cdict,       12000408
+silesia,                            multithreaded,                      old streaming advanced cdict,       12000408
+silesia,                            multithreaded long distance mode,   old streaming advanced cdict,       12000408
+silesia,                            small window log,                   old streaming advanced cdict,       12000408
+silesia,                            small hash log,                     old streaming advanced cdict,       12000408
+silesia,                            small chain log,                    old streaming advanced cdict,       12000408
+silesia,                            explicit params,                    old streaming advanced cdict,       12000408
+silesia,                            uncompressed literals,              old streaming advanced cdict,       4862377
+silesia,                            uncompressed literals optimal,      old streaming advanced cdict,       4293262
+silesia,                            huffman literals,                   old streaming advanced cdict,       6191549
+silesia,                            multithreaded with advanced params, old streaming advanced cdict,       12000408
+silesia.tar,                        level -5,                           old streaming advanced cdict,       7160440
+silesia.tar,                        level -3,                           old streaming advanced cdict,       6789026
+silesia.tar,                        level -1,                           old streaming advanced cdict,       6195465
+silesia.tar,                        level 0,                            old streaming advanced cdict,       4875010
+silesia.tar,                        level 1,                            old streaming advanced cdict,       5339701
+silesia.tar,                        level 3,                            old streaming advanced cdict,       4875010
+silesia.tar,                        level 4,                            old streaming advanced cdict,       4813507
+silesia.tar,                        level 5,                            old streaming advanced cdict,       4722240
+silesia.tar,                        level 6,                            old streaming advanced cdict,       4672203
+silesia.tar,                        level 7,                            old streaming advanced cdict,       4606658
+silesia.tar,                        level 9,                            old streaming advanced cdict,       4554105
+silesia.tar,                        level 13,                           old streaming advanced cdict,       4491703
+silesia.tar,                        level 16,                           old streaming advanced cdict,       4381277
+silesia.tar,                        level 19,                           old streaming advanced cdict,       4281514
+silesia.tar,                        no source size,                     old streaming advanced cdict,       4875006
+silesia.tar,                        long distance mode,                 old streaming advanced cdict,       12022046
+silesia.tar,                        multithreaded,                      old streaming advanced cdict,       12022046
+silesia.tar,                        multithreaded long distance mode,   old streaming advanced cdict,       12022046
+silesia.tar,                        small window log,                   old streaming advanced cdict,       12022046
+silesia.tar,                        small hash log,                     old streaming advanced cdict,       12022046
+silesia.tar,                        small chain log,                    old streaming advanced cdict,       12022046
+silesia.tar,                        explicit params,                    old streaming advanced cdict,       12022046
+silesia.tar,                        uncompressed literals,              old streaming advanced cdict,       4875010
+silesia.tar,                        uncompressed literals optimal,      old streaming advanced cdict,       4281514
+silesia.tar,                        huffman literals,                   old streaming advanced cdict,       6195465
+silesia.tar,                        multithreaded with advanced params, old streaming advanced cdict,       12022046
+github,                             level -5,                           old streaming advanced cdict,       232744
+github,                             level -5 with dict,                 old streaming advanced cdict,       46718
+github,                             level -3,                           old streaming advanced cdict,       220611
+github,                             level -3 with dict,                 old streaming advanced cdict,       45395
+github,                             level -1,                           old streaming advanced cdict,       176575
+github,                             level -1 with dict,                 old streaming advanced cdict,       43170
+github,                             level 0,                            old streaming advanced cdict,       136397
+github,                             level 0 with dict,                  old streaming advanced cdict,       41170
+github,                             level 1,                            old streaming advanced cdict,       143457
+github,                             level 1 with dict,                  old streaming advanced cdict,       41682
+github,                             level 3,                            old streaming advanced cdict,       136397
+github,                             level 3 with dict,                  old streaming advanced cdict,       41170
+github,                             level 4,                            old streaming advanced cdict,       136144
+github,                             level 4 with dict,                  old streaming advanced cdict,       41306
+github,                             level 5,                            old streaming advanced cdict,       135106
+github,                             level 5 with dict,                  old streaming advanced cdict,       38938
+github,                             level 6,                            old streaming advanced cdict,       135108
+github,                             level 6 with dict,                  old streaming advanced cdict,       38632
+github,                             level 7,                            old streaming advanced cdict,       135108
+github,                             level 7 with dict,                  old streaming advanced cdict,       38766
+github,                             level 9,                            old streaming advanced cdict,       135108
+github,                             level 9 with dict,                  old streaming advanced cdict,       39326
+github,                             level 13,                           old streaming advanced cdict,       133717
+github,                             level 13 with dict,                 old streaming advanced cdict,       39716
+github,                             level 16,                           old streaming advanced cdict,       133717
+github,                             level 16 with dict,                 old streaming advanced cdict,       37577
+github,                             level 19,                           old streaming advanced cdict,       133717
+github,                             level 19 with dict,                 old streaming advanced cdict,       37576
+github,                             no source size,                     old streaming advanced cdict,       141003
+github,                             long distance mode,                 old streaming advanced cdict,       412933
+github,                             multithreaded,                      old streaming advanced cdict,       412933
+github,                             multithreaded long distance mode,   old streaming advanced cdict,       412933
+github,                             small window log,                   old streaming advanced cdict,       412933
+github,                             small hash log,                     old streaming advanced cdict,       412933
+github,                             small chain log,                    old streaming advanced cdict,       412933
+github,                             explicit params,                    old streaming advanced cdict,       412933
+github,                             uncompressed literals,              old streaming advanced cdict,       136397
+github,                             uncompressed literals optimal,      old streaming advanced cdict,       133717
+github,                             huffman literals,                   old streaming advanced cdict,       176575
+github,                             multithreaded with advanced params, old streaming advanced cdict,       412933

From e55da9e9638ef504f93a0ed448a47d0a8d745657 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Wed, 13 Mar 2019 14:05:18 -0700
Subject: [PATCH 105/178] Wrap the new advanced api completely

---
 lib/compress/zstd_compress.c          | 112 ++++++++++++++------------
 lib/compress/zstd_compress_internal.h |   7 --
 lib/compress/zstdmt_compress.c        |   2 +-
 tests/zstreamtest.c                   |  15 ++--
 4 files changed, 70 insertions(+), 66 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 987c7b90aa6..62dab80033a 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3769,13 +3769,17 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* cctx,
 
 /* ZSTD_resetCStream():
  * pledgedSrcSize == 0 means "unknown" */
-size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
+size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pss)
 {
-    ZSTD_CCtx_params params = zcs->requestedParams;
+    /* temporary : 0 interpreted as "unknown" during transition period.
+     * Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN.
+     * 0 will be interpreted as "empty" in the future.
+     */
+    U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss;
     DEBUGLOG(4, "ZSTD_resetCStream: pledgedSrcSize = %u", (unsigned)pledgedSrcSize);
-    if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
-    params.fParams.contentSizeFlag = 1;
-    return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dct_auto, zcs->cdict, params, pledgedSrcSize);
+    FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) );
+    FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) );
+    return 0;
 }
 
 /*! ZSTD_initCStream_internal() :
@@ -3787,31 +3791,18 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
                     ZSTD_CCtx_params params, unsigned long long pledgedSrcSize)
 {
     DEBUGLOG(4, "ZSTD_initCStream_internal");
-    params.cParams = ZSTD_getCParamsFromCCtxParams(¶ms, pledgedSrcSize, dictSize);
+    FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) );
+    FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) );
     assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
+    zcs->requestedParams = params;
     assert(!((dict) && (cdict)));  /* either dict or cdict, not both */
-
-    if (dict && dictSize >= 8) {
-        DEBUGLOG(4, "loading dictionary of size %u", (unsigned)dictSize);
-        RETURN_ERROR_IF(
-            zcs->staticSize, memory_allocation,
-            "static CCtx: incompatible with internal cdict creation");
-        ZSTD_freeCDict(zcs->cdictLocal);
-        zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
-                                            ZSTD_dlm_byCopy, ZSTD_dct_auto,
-                                            params.cParams, zcs->customMem);
-        zcs->cdict = zcs->cdictLocal;
-        RETURN_ERROR_IF(zcs->cdictLocal == NULL, memory_allocation);
+    if (dict) {
+        FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) );
     } else {
-        if (cdict) {
-            params.cParams = ZSTD_getCParamsFromCDict(cdict);  /* cParams are enforced from cdict; it includes windowLog */
-        }
-        ZSTD_freeCDict(zcs->cdictLocal);
-        zcs->cdictLocal = NULL;
-        zcs->cdict = cdict;
+        /* Dictionary is cleared if !cdict */
+        FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) );
     }
-
-    return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dct_auto, zcs->cdict, params, pledgedSrcSize);
+    return 0;
 }
 
 /* ZSTD_initCStream_usingCDict_advanced() :
@@ -3822,23 +3813,20 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
                                             unsigned long long pledgedSrcSize)
 {
     DEBUGLOG(4, "ZSTD_initCStream_usingCDict_advanced");
-    RETURN_ERROR_IF(!cdict, dictionary_wrong,
-                    "cannot handle NULL cdict (does not know what to do)");
-    {   ZSTD_CCtx_params params = zcs->requestedParams;
-        params.cParams = ZSTD_getCParamsFromCDict(cdict);
-        params.fParams = fParams;
-        return ZSTD_initCStream_internal(zcs,
-                                NULL, 0, cdict,
-                                params, pledgedSrcSize);
-    }
+    FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) );
+    FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) );
+    zcs->requestedParams.fParams = fParams;
+    FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) );
+    return 0;
 }
 
 /* note : cdict must outlive compression session */
 size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
 {
-    ZSTD_frameParameters const fParams = { 0 /* contentSizeFlag */, 0 /* checksum */, 0 /* hideDictID */ };
     DEBUGLOG(4, "ZSTD_initCStream_usingCDict");
-    return ZSTD_initCStream_usingCDict_advanced(zcs, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN);  /* note : will check that cdict != NULL */
+    FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) );
+    FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) );
+    return 0;
 }
 
 
@@ -3848,33 +3836,53 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
  * dict is loaded with default parameters ZSTD_dm_auto and ZSTD_dlm_byCopy. */
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
                                  const void* dict, size_t dictSize,
-                                 ZSTD_parameters params, unsigned long long pledgedSrcSize)
+                                 ZSTD_parameters params, unsigned long long pss)
 {
-    DEBUGLOG(4, "ZSTD_initCStream_advanced: pledgedSrcSize=%u, flag=%u",
-                (unsigned)pledgedSrcSize, params.fParams.contentSizeFlag);
+    /* for compatibility with older programs relying on this behavior.
+     * Users should now specify ZSTD_CONTENTSIZE_UNKNOWN.
+     * This line will be removed in the future.
+     */
+    U64 const pledgedSrcSize = (pss==0 && params.fParams.contentSizeFlag==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss;
+    DEBUGLOG(4, "ZSTD_initCStream_advanced");
+    FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) );
+    FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) );
     FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) );
-    if ((pledgedSrcSize==0) && (params.fParams.contentSizeFlag==0)) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;  /* for compatibility with older programs relying on this behavior. Users should now specify ZSTD_CONTENTSIZE_UNKNOWN. This line will be removed in the future. */
     zcs->requestedParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params);
-    return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL /*cdict*/, zcs->requestedParams, pledgedSrcSize);
+    FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) );
+    return 0;
 }
 
 size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel)
 {
-    ZSTD_CCtxParams_init(&zcs->requestedParams, compressionLevel);
-    return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, zcs->requestedParams, ZSTD_CONTENTSIZE_UNKNOWN);
+    DEBUGLOG(4, "ZSTD_initCStream_usingDict");
+    FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) );
+    FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) );
+    FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) );
+    return 0;
 }
 
 size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pss)
 {
-    U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss;  /* temporary : 0 interpreted as "unknown" during transition period. Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN. `0` will be interpreted as "empty" in the future */
-    ZSTD_CCtxParams_init(&zcs->requestedParams, compressionLevel);
-    return ZSTD_initCStream_internal(zcs, NULL, 0, NULL, zcs->requestedParams, pledgedSrcSize);
+    /* temporary : 0 interpreted as "unknown" during transition period.
+     * Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN.
+     * 0 will be interpreted as "empty" in the future.
+     */
+    U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss;
+    DEBUGLOG(4, "ZSTD_initCStream_srcSize");
+    FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) );
+    FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) );
+    FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) );
+    FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) );
+    return 0;
 }
 
 size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
 {
     DEBUGLOG(4, "ZSTD_initCStream");
-    return ZSTD_initCStream_srcSize(zcs, compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN);
+    FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) );
+    FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) );
+    FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) );
+    return 0;
 }
 
 /*======   Compression   ======*/
@@ -3898,10 +3906,10 @@ static size_t ZSTD_limitCopy(void* dst, size_t dstCapacity,
  *  internal function for all *compressStream*() variants
  *  non-static, because can be called from zstdmt_compress.c
  * @return : hint size for next input */
-size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
-                                   ZSTD_outBuffer* output,
-                                   ZSTD_inBuffer* input,
-                                   ZSTD_EndDirective const flushMode)
+static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
+                                          ZSTD_outBuffer* output,
+                                          ZSTD_inBuffer* input,
+                                          ZSTD_EndDirective const flushMode)
 {
     const char* const istart = (const char*)input->src;
     const char* const iend = istart + input->size;
diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index a828de3b4a6..c90034d4f52 100644
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -808,13 +808,6 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
 
 void ZSTD_resetSeqStore(seqStore_t* ssPtr);
 
-/*! ZSTD_compressStream_generic() :
- *  Private use only. To be called from zstdmt_compress.c in single-thread mode. */
-size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
-                                   ZSTD_outBuffer* output,
-                                   ZSTD_inBuffer* input,
-                                   ZSTD_EndDirective const flushMode);
-
 /*! ZSTD_getCParamsFromCDict() :
  *  as the name implies */
 ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict);
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index dab3a9508ec..7e2c78984eb 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -1967,7 +1967,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
     assert(input->pos  <= input->size);
 
     if (mtctx->singleBlockingThread) {  /* delegate to single-thread (synchronous) */
-        return ZSTD_compressStream_generic(mtctx->cctxPool->cctx[0], output, input, endOp);
+        return ZSTD_compressStream2(mtctx->cctxPool->cctx[0], output, input, endOp);
     }
 
     if ((mtctx->frameEnded) && (endOp==ZSTD_e_continue)) {
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index b099e2870a6..e26375e07b2 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -495,7 +495,7 @@ static int basicUnitTests(U32 seed, double compressibility)
 
     /* _srcSize compression test */
     DISPLAYLEVEL(3, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
-    ZSTD_initCStream_srcSize(zc, 1, CNBufferSize);
+    CHECK_Z( ZSTD_initCStream_srcSize(zc, 1, CNBufferSize) );
     outBuff.dst = (char*)(compressedBuffer);
     outBuff.size = compressedBufferSize;
     outBuff.pos = 0;
@@ -503,11 +503,14 @@ static int basicUnitTests(U32 seed, double compressibility)
     inBuff.size = CNBufferSize;
     inBuff.pos = 0;
     CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
-    if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
-    { size_t const r = ZSTD_endStream(zc, &outBuff);
-      if (r != 0) goto _output_error; }  /* error, or some data not flushed */
-    { unsigned long long origSize = ZSTD_findDecompressedSize(outBuff.dst, outBuff.pos);
-      if ((size_t)origSize != CNBufferSize) goto _output_error; }  /* exact original size must be present */
+    CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
+    {   size_t const r = ZSTD_endStream(zc, &outBuff);
+        CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
+    }
+    {   unsigned long long origSize = ZSTD_findDecompressedSize(outBuff.dst, outBuff.pos);
+        CHECK(origSize == ZSTD_CONTENTSIZE_UNKNOWN, "Unknown!");
+        CHECK((size_t)origSize != CNBufferSize, "Exact original size must be present (got %llu)", origSize);
+    }
     DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
 
     /* wrong _srcSize compression test */

From 20f9ff7e5361916560e95336c89e8dc8564ba230 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 15 Mar 2019 18:03:31 -0700
Subject: [PATCH 106/178] Update documentation to tell how to replace the old
 streaming API with the new one.

---
 lib/zstd.h | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 74 insertions(+), 5 deletions(-)

diff --git a/lib/zstd.h b/lib/zstd.h
index f6332589d6b..c1c0fe744c0 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -347,9 +347,24 @@ ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
 ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
 
 /*===== Streaming compression functions =====*/
+/**
+ * Equivalent to:
+ *
+ *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ *     ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
+ *     ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
+ */
 ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
+/**
+ * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).
+ * NOTE: The return value is different. ZSTD_compressStream() returns a hint for
+ * the next read size (if non-zero and not an error). ZSTD_compressStream2()
+ * returns the number of bytes left to flush (if non-zero and not an error).
+ */
 ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+/** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */
 ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+/** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */
 ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
 
 ZSTDLIB_API size_t ZSTD_CStreamInSize(void);    /**< recommended size for input buffer */
@@ -1542,14 +1557,68 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs (
 ********************************************************************/
 
 /*=====   Advanced Streaming compression functions  =====*/
-ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   /**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, "0" also disables frame content size field. It may be enabled in the future. */
-ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/
+/**! ZSTD_initCStream_srcSize() :
+ * This function is deprecated, and equivalent to:
+ *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ *     ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
+ *     ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
+ *     ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
+ *
+ * pledgedSrcSize must be correct. If it is not known at init time, use
+ * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs,
+ * "0" also disables frame content size field. It may be enabled in the future.
+ */
+ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);
+/**! ZSTD_initCStream_usingDict() :
+ * This function is deprecated, and is equivalent to:
+ *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ *     ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
+ *     ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
+ *
+ * Creates of an internal CDict (incompatible with static CCtx), except if
+ * dict == NULL or dictSize < 8, in which case no dict is used.
+ * Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if
+ * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
+ */
+ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
+/**! ZSTD_initCStream_advanced() :
+ * This function is deprecated, and is approximately equivalent to:
+ *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ *     ZSTD_CCtx_setZstdParams(zcs, params); // Set the zstd params and leave the rest as-is
+ *     ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
+ *     ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
+ *
+ * pledgedSrcSize must be correct. If srcSize is not known at init time, use
+ * value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy.
+ */
 ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
-                                             ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */
-ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);  /**< note : cdict will just be referenced, and must outlive compression session */
-ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize);  /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters. pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. */
+                                             ZSTD_parameters params, unsigned long long pledgedSrcSize);
+/**! ZSTD_initCStream_usingCDict() :
+ * This function is deprecated, and equivalent to:
+ *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ *     ZSTD_CCtx_refCDict(zcs, cdict);
+ *
+ * note : cdict will just be referenced, and must outlive compression session
+ */
+ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
+/**! ZSTD_initCStream_usingCDict_advanced() :
+ * This function is deprecated, and is approximately equivalent to:
+ *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ *     ZSTD_CCtx_setZstdFrameParams(zcs, fParams); // Set the zstd frame params and leave the rest as-is
+ *     ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
+ *     ZSTD_CCtx_refCDict(zcs, cdict);
+ *
+ * same as ZSTD_initCStream_usingCDict(), with control over frame parameters.
+ * pledgedSrcSize must be correct. If srcSize is not known at init time, use
+ * value ZSTD_CONTENTSIZE_UNKNOWN.
+ */
+ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize);
 
 /*! ZSTD_resetCStream() :
+ * This function is deprecated, and is equivalent to:
+ *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ *     ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
+ *
  *  start a new frame, using same parameters from previous frame.
  *  This is typically useful to skip dictionary loading stage, since it will re-use it in-place.
  *  Note that zcs must be init at least once before using ZSTD_resetCStream().

From 6b053b9f6098d5f0430186158fb66dfd1f969d09 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Thu, 21 Mar 2019 15:17:41 -0700
Subject: [PATCH 107/178] [lib] Allow ZSTD_CCtx_loadDictionary() to be called
 before parameters are set

* After loading a dictionary only create the cdict once we've started the
  compression job. This allows the user to pass the dictionary before they
  set other settings, and is in line with the rest of the API.
* Add tests that mix the 3 dictionary loading APIs.
* Add extra tests for `ZSTD_CCtx_loadDictionary()`.
* The first 2 tests added fail before this patch.
* Run the regression test suite.
---
 lib/compress/zstd_compress.c          |  94 ++++++++++++++++-----
 lib/compress/zstd_compress_internal.h |  10 ++-
 tests/fuzzer.c                        | 115 +++++++++++++++++++++++++-
 3 files changed, 198 insertions(+), 21 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 62dab80033a..fe59f522fd2 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -103,12 +103,31 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize)
     return cctx;
 }
 
+/**
+ * Clears and frees all of the dictionaries in the CCtx.
+ */
+static void ZSTD_clearAllDicts(ZSTD_CCtx* cctx)
+{
+    ZSTD_free(cctx->localDict.dictBuffer, cctx->customMem);
+    ZSTD_freeCDict(cctx->localDict.cdict);
+    memset(&cctx->localDict, 0, sizeof(cctx->localDict));
+    memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict));
+    cctx->cdict = NULL;
+}
+
+static size_t ZSTD_sizeof_localDict(ZSTD_localDict dict)
+{
+    size_t const bufferSize = dict.dictBuffer != NULL ? dict.dictSize : 0;
+    size_t const cdictSize = ZSTD_sizeof_CDict(dict.cdict);
+    return bufferSize + cdictSize;
+}
+
 static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx)
 {
     assert(cctx != NULL);
     assert(cctx->staticSize == 0);
     ZSTD_free(cctx->workSpace, cctx->customMem); cctx->workSpace = NULL;
-    ZSTD_freeCDict(cctx->cdictLocal); cctx->cdictLocal = NULL;
+    ZSTD_clearAllDicts(cctx);
 #ifdef ZSTD_MULTITHREAD
     ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL;
 #endif
@@ -140,7 +159,7 @@ size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx)
 {
     if (cctx==NULL) return 0;   /* support sizeof on NULL */
     return sizeof(*cctx) + cctx->workSpaceSize
-           + ZSTD_sizeof_CDict(cctx->cdictLocal)
+           + ZSTD_sizeof_localDict(cctx->localDict)
            + ZSTD_sizeof_mtctx(cctx);
 }
 
@@ -785,6 +804,44 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo
     return 0;
 }
 
+/**
+ * Initializes the local dict using the requested parameters.
+ * NOTE: This does not use the pledged src size, because it may be used for more
+ * than one compression.
+ */
+static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx)
+{
+    ZSTD_localDict* const dl = &cctx->localDict;
+    ZSTD_compressionParameters const cParams = ZSTD_getCParamsFromCCtxParams(
+            &cctx->requestedParams, 0, dl->dictSize);
+    if (dl->dict == NULL) {
+        /* No local dictionary. */
+        assert(dl->dictBuffer == NULL);
+        assert(dl->cdict == NULL);
+        assert(dl->dictSize == 0);
+        return 0;
+    }
+    if (dl->cdict != NULL) {
+        assert(cctx->cdict == dl->cdict);
+        /* Local dictionary already initialized. */
+        return 0;
+    }
+    assert(dl->dictSize > 0);
+    assert(cctx->cdict == NULL);
+    assert(cctx->prefixDict.dict == NULL);
+
+    dl->cdict = ZSTD_createCDict_advanced(
+            dl->dict,
+            dl->dictSize,
+            ZSTD_dlm_byRef,
+            dl->dictContentType,
+            cParams,
+            cctx->customMem);
+    RETURN_ERROR_IF(!dl->cdict, memory_allocation);
+    cctx->cdict = dl->cdict;
+    return 0;
+}
+
 size_t ZSTD_CCtx_loadDictionary_advanced(
         ZSTD_CCtx* cctx, const void* dict, size_t dictSize,
         ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType)
@@ -793,20 +850,20 @@ size_t ZSTD_CCtx_loadDictionary_advanced(
     RETURN_ERROR_IF(cctx->staticSize, memory_allocation,
                     "no malloc for static CCtx");
     DEBUGLOG(4, "ZSTD_CCtx_loadDictionary_advanced (size: %u)", (U32)dictSize);
-    ZSTD_freeCDict(cctx->cdictLocal);  /* in case one already exists */
-    if (dict==NULL || dictSize==0) {   /* no dictionary mode */
-        cctx->cdictLocal = NULL;
-        cctx->cdict = NULL;
+    ZSTD_clearAllDicts(cctx);  /* in case one already exists */
+    if (dict == NULL || dictSize == 0)  /* no dictionary mode */
+        return 0;
+    if (dictLoadMethod == ZSTD_dlm_byRef) {
+        cctx->localDict.dict = dict;
     } else {
-        ZSTD_compressionParameters const cParams =
-                ZSTD_getCParamsFromCCtxParams(&cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, dictSize);
-        cctx->cdictLocal = ZSTD_createCDict_advanced(
-                                dict, dictSize,
-                                dictLoadMethod, dictContentType,
-                                cParams, cctx->customMem);
-        cctx->cdict = cctx->cdictLocal;
-        RETURN_ERROR_IF(cctx->cdictLocal == NULL, memory_allocation);
+        void* dictBuffer = ZSTD_malloc(dictSize, cctx->customMem);
+        RETURN_ERROR_IF(!dictBuffer, memory_allocation);
+        memcpy(dictBuffer, dict, dictSize);
+        cctx->localDict.dictBuffer = dictBuffer;
+        cctx->localDict.dict = dictBuffer;
     }
+    cctx->localDict.dictSize = dictSize;
+    cctx->localDict.dictContentType = dictContentType;
     return 0;
 }
 
@@ -828,10 +885,8 @@ size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
 {
     RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong);
     /* Free the existing local cdict (if any) to save memory. */
-    FORWARD_IF_ERROR( ZSTD_freeCDict(cctx->cdictLocal) );
-    cctx->cdictLocal = NULL;
+    ZSTD_clearAllDicts(cctx);
     cctx->cdict = cdict;
-    memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict));  /* exclusive */
     return 0;
 }
 
@@ -844,7 +899,7 @@ size_t ZSTD_CCtx_refPrefix_advanced(
         ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType)
 {
     RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong);
-    cctx->cdict = NULL;   /* prefix discards any prior cdict */
+    ZSTD_clearAllDicts(cctx);
     cctx->prefixDict.dict = prefix;
     cctx->prefixDict.dictSize = prefixSize;
     cctx->prefixDict.dictContentType = dictContentType;
@@ -863,7 +918,7 @@ size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset)
     if ( (reset == ZSTD_reset_parameters)
       || (reset == ZSTD_reset_session_and_parameters) ) {
         RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong);
-        cctx->cdict = NULL;
+        ZSTD_clearAllDicts(cctx);
         return ZSTD_CCtxParams_reset(&cctx->requestedParams);
     }
     return 0;
@@ -4079,6 +4134,7 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
     if (cctx->streamStage == zcss_init) {
         ZSTD_CCtx_params params = cctx->requestedParams;
         ZSTD_prefixDict const prefixDict = cctx->prefixDict;
+        FORWARD_IF_ERROR( ZSTD_initLocalDict(cctx) ); /* Init the local dict if present. */
         memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict));   /* single usage */
         assert(prefixDict.dict==NULL || cctx->cdict==NULL);    /* only one can be set */
         DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage");
diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index c90034d4f52..78b53550b0c 100644
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -54,6 +54,14 @@ typedef struct ZSTD_prefixDict_s {
     ZSTD_dictContentType_e dictContentType;
 } ZSTD_prefixDict;
 
+typedef struct {
+    void* dictBuffer;
+    void const* dict;
+    size_t dictSize;
+    ZSTD_dictContentType_e dictContentType;
+    ZSTD_CDict* cdict;
+} ZSTD_localDict;
+
 typedef struct {
     U32 CTable[HUF_CTABLE_SIZE_U32(255)];
     HUF_repeat repeatMode;
@@ -245,7 +253,7 @@ struct ZSTD_CCtx_s {
     U32    frameEnded;
 
     /* Dictionary */
-    ZSTD_CDict* cdictLocal;
+    ZSTD_localDict localDict;
     const ZSTD_CDict* cdict;
     ZSTD_prefixDict prefixDict;   /* single-usage dictionary */
 
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index f74ca5d7b35..7bc2f10cbb2 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1285,9 +1285,13 @@ static int basicUnitTests(U32 seed, double compressibility)
         {
             size_t ret;
             MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY);
+            /* Either operation is allowed to fail, but one must fail. */
             ret = ZSTD_CCtx_loadDictionary_advanced(
                     cctx, (const char*)dictBuffer+2, dictSize-2, ZSTD_dlm_byRef, ZSTD_dct_auto);
-            if (!ZSTD_isError(ret)) goto _output_error;
+            if (!ZSTD_isError(ret)) {
+                ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100));
+                if (!ZSTD_isError(ret)) goto _output_error;
+            }
         }
         DISPLAYLEVEL(3, "OK \n");
 
@@ -1298,6 +1302,8 @@ static int basicUnitTests(U32 seed, double compressibility)
             ret = ZSTD_CCtx_loadDictionary_advanced(
                     cctx, (const char*)dictBuffer+2, dictSize-2, ZSTD_dlm_byRef, ZSTD_dct_rawContent);
             if (ZSTD_isError(ret)) goto _output_error;
+            ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100));
+            if (ZSTD_isError(ret)) goto _output_error;
         }
         DISPLAYLEVEL(3, "OK \n");
 
@@ -1312,6 +1318,113 @@ static int basicUnitTests(U32 seed, double compressibility)
         }
         DISPLAYLEVEL(3, "OK \n");
 
+        DISPLAYLEVEL(3, "test%3i : Loading dictionary before setting parameters is the same as loading after : ", testNb++);
+        {
+            size_t size1, size2;
+            ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
+            CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) );
+            CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
+            size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
+            if (ZSTD_isError(size1)) goto _output_error;
+
+            ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
+            CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
+            CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) );
+            size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
+            if (ZSTD_isError(size2)) goto _output_error;
+
+            if (size1 != size2) goto _output_error;
+        }
+        DISPLAYLEVEL(3, "OK \n");
+
+        DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the prefix : ", testNb++);
+        {
+            CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
+            CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
+            CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
+        }
+        DISPLAYLEVEL(3, "OK \n");
+
+        DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the cdict : ", testNb++);
+        {
+            ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
+            CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
+            CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
+            CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
+            ZSTD_freeCDict(cdict);
+        }
+        DISPLAYLEVEL(3, "OK \n");
+
+        DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the prefix : ", testNb++);
+        {
+            ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
+            CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
+            CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
+            CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
+            ZSTD_freeCDict(cdict);
+        }
+        DISPLAYLEVEL(3, "OK \n");
+
+        DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the dictionary : ", testNb++);
+        {
+            ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
+            CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
+            CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
+            CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
+            ZSTD_freeCDict(cdict);
+        }
+        DISPLAYLEVEL(3, "OK \n");
+
+        DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the dictionary : ", testNb++);
+        {
+            CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
+            CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
+            CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
+        }
+        DISPLAYLEVEL(3, "OK \n");
+
+        DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the cdict : ", testNb++);
+        {
+            ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
+            CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
+            CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
+            CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
+            ZSTD_freeCDict(cdict);
+        }
+        DISPLAYLEVEL(3, "OK \n");
+
+        DISPLAYLEVEL(3, "test%3i : Loaded dictionary persists across reset session : ", testNb++);
+        {
+            size_t size1, size2;
+            ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
+            CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
+            size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
+            if (ZSTD_isError(size1)) goto _output_error;
+
+            ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
+            size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
+            if (ZSTD_isError(size2)) goto _output_error;
+
+            if (size1 != size2) goto _output_error;
+        }
+        DISPLAYLEVEL(3, "OK \n");
+
+        DISPLAYLEVEL(3, "test%3i : Loaded dictionary is cleared after resetting parameters : ", testNb++);
+        {
+            size_t size1, size2;
+            ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
+            CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
+            size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
+            if (ZSTD_isError(size1)) goto _output_error;
+
+            ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
+            size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
+            if (ZSTD_isError(size2)) goto _output_error;
+
+            if (size1 == size2) goto _output_error;
+        }
+        DISPLAYLEVEL(3, "OK \n");
+
         DISPLAYLEVEL(3, "test%3i : Dictionary with non-default repcodes : ", testNb++);
         { U32 u; for (u=0; u
Date: Fri, 22 Mar 2019 12:28:55 -0700
Subject: [PATCH 108/178] [cover] Improvements for small or homogeneous data

* The algorithm would bail as soon as it found one epoch that
  contained no new segments. Change it so it now has to fail
  >= 10 times in a row (10 for fastcover, 10-100 for cover).
* The algorithm uses the `maxDict` size to decide the epoch size.
  When this size is absurdly large, it causes tiny epochs. Lower
  bound the epoch size at 10x the segment size, and warn the user
  that their training set is too small.

Fixes #1554
---
 lib/dictBuilder/cover.c     | 66 ++++++++++++++++++++++++++++++-------
 lib/dictBuilder/cover.h     | 29 ++++++++++++++++
 lib/dictBuilder/fastcover.c | 34 ++++++++++++-------
 3 files changed, 107 insertions(+), 22 deletions(-)

diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c
index b55bfb510b7..180a68ae8c6 100644
--- a/lib/dictBuilder/cover.c
+++ b/lib/dictBuilder/cover.c
@@ -627,6 +627,38 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
   return 1;
 }
 
+void COVER_warnOnSmallCorpus(size_t maxDictSize, size_t nbDmers)
+{
+  const double ratio = (double)nbDmers / maxDictSize;
+  if (ratio >= 10) {
+      return;
+  }
+  DISPLAYLEVEL(1, "WARNING: The maximum dictionary size %u is too large "
+                  "compared to the source size %u! "
+                  "size(source)/size(dictionary) = %f, but it should be >= "
+                  "10! This may lead to a subpar dictionary! We recommend "
+                  "training on sources at least 10x, and up to 100x the "
+                  "size of the dictionary!\n", (U32)maxDictSize,
+                  (U32)nbDmers, ratio);
+}
+
+COVER_epoch_info_t COVER_computeEpochs(U32 maxDictSize,
+                                       U32 nbDmers, U32 k, U32 passes)
+{
+  const U32 minEpochSize = k * 10;
+  COVER_epoch_info_t epochs;
+  epochs.num = MAX(1, maxDictSize / k / passes);
+  epochs.size = nbDmers / epochs.num;
+  if (epochs.size >= minEpochSize) {
+      assert(epochs.size * epochs.num <= nbDmers);
+      return epochs;
+  }
+  epochs.size = MIN(minEpochSize, nbDmers);
+  epochs.num = nbDmers / epochs.size;
+  assert(epochs.size * epochs.num <= nbDmers);
+  return epochs;
+}
+
 /**
  * Given the prepared context build the dictionary.
  */
@@ -636,28 +668,34 @@ static size_t COVER_buildDictionary(const COVER_ctx_t *ctx, U32 *freqs,
                                     ZDICT_cover_params_t parameters) {
   BYTE *const dict = (BYTE *)dictBuffer;
   size_t tail = dictBufferCapacity;
-  /* Divide the data up into epochs of equal size.
-   * We will select at least one segment from each epoch.
-   */
-  const unsigned epochs = MAX(1, (U32)(dictBufferCapacity / parameters.k / 4));
-  const unsigned epochSize = (U32)(ctx->suffixSize / epochs);
+  /* Divide the data into epochs. We will select one segment from each epoch. */
+  const COVER_epoch_info_t epochs = COVER_computeEpochs(
+      (U32)dictBufferCapacity, (U32)ctx->suffixSize, parameters.k, 4);
+  const size_t maxZeroScoreRun = MAX(10, MIN(100, epochs.num >> 3));
+  size_t zeroScoreRun = 0;
   size_t epoch;
   DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n",
-                epochs, epochSize);
+                (U32)epochs.num, (U32)epochs.size);
   /* Loop through the epochs until there are no more segments or the dictionary
    * is full.
    */
-  for (epoch = 0; tail > 0; epoch = (epoch + 1) % epochs) {
-    const U32 epochBegin = (U32)(epoch * epochSize);
-    const U32 epochEnd = epochBegin + epochSize;
+  for (epoch = 0; tail > 0; epoch = (epoch + 1) % epochs.num) {
+    const U32 epochBegin = (U32)(epoch * epochs.size);
+    const U32 epochEnd = epochBegin + epochs.size;
     size_t segmentSize;
     /* Select a segment */
     COVER_segment_t segment = COVER_selectSegment(
         ctx, freqs, activeDmers, epochBegin, epochEnd, parameters);
-    /* If the segment covers no dmers, then we are out of content */
+    /* If the segment covers no dmers, then we are out of content.
+     * There may be new content in other epochs, for continue for some time.
+     */
     if (segment.score == 0) {
-      break;
+      if (++zeroScoreRun >= maxZeroScoreRun) {
+          break;
+      }
+      continue;
     }
+    zeroScoreRun = 0;
     /* Trim the segment if necessary and if it is too small then we are done */
     segmentSize = MIN(segment.end - segment.begin + parameters.d - 1, tail);
     if (segmentSize < parameters.d) {
@@ -706,6 +744,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
                       parameters.d, parameters.splitPoint)) {
     return ERROR(GENERIC);
   }
+  COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.suffixSize);
   if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) {
     DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n");
     COVER_ctx_destroy(&ctx);
@@ -977,6 +1016,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
   unsigned k;
   COVER_best_t best;
   POOL_ctx *pool = NULL;
+  int warned = 0;
 
   /* Checks */
   if (splitPoint <= 0 || splitPoint > 1) {
@@ -1019,6 +1059,10 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
       POOL_free(pool);
       return ERROR(GENERIC);
     }
+    if (!warned) {
+      COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.suffixSize);
+      warned = 1;
+    }
     /* Loop through k reusing the same context */
     for (k = kMinK; k <= kMaxK; k += kStepSize) {
       /* Prepare the arguments */
diff --git a/lib/dictBuilder/cover.h b/lib/dictBuilder/cover.h
index 82e2e1cea43..71c520e9e62 100644
--- a/lib/dictBuilder/cover.h
+++ b/lib/dictBuilder/cover.h
@@ -38,6 +38,35 @@ typedef struct {
   U32 score;
 } COVER_segment_t;
 
+/**
+ *Number of epochs and size of each epoch.
+ */
+typedef struct {
+  U32 num;
+  U32 size;
+} COVER_epoch_info_t;
+
+/**
+ * Computes the number of epochs and the size of each epoch.
+ * We will make sure that each epoch gets at least 10 * k bytes.
+ *
+ * The COVER algorithms divide the data up into epochs of equal size and
+ * select one segemnt from each epoch.
+ *
+ * @param maxDictSize The maximum allowed dictioary size.
+ * @param nbDmers     The number of dmers we are training on.
+ * @param k           The parameter k (segment size).
+ * @param passes      The target number of passes over the dmer corpus.
+ *                    More passes means a better dictionary.
+ */
+COVER_epoch_info_t COVER_computeEpochs(U32 maxDictSize, U32 nbDmers,
+                                       U32 k, U32 passes);
+
+/**
+ * Warns the user when their corpus is too small.
+ */
+void COVER_warnOnSmallCorpus(size_t maxDictSize, size_t nbDmers);
+
 /**
  *  Checks total compressed size of a dictionary
  */
diff --git a/lib/dictBuilder/fastcover.c b/lib/dictBuilder/fastcover.c
index c289c069014..8cb89c93886 100644
--- a/lib/dictBuilder/fastcover.c
+++ b/lib/dictBuilder/fastcover.c
@@ -386,29 +386,35 @@ FASTCOVER_buildDictionary(const FASTCOVER_ctx_t* ctx,
 {
   BYTE *const dict = (BYTE *)dictBuffer;
   size_t tail = dictBufferCapacity;
-  /* Divide the data up into epochs of equal size.
-   * We will select at least one segment from each epoch.
-   */
-  const unsigned epochs = MAX(1, (U32)(dictBufferCapacity / parameters.k));
-  const unsigned epochSize = (U32)(ctx->nbDmers / epochs);
+  /* Divide the data into epochs. We will select one segment from each epoch. */
+  const COVER_epoch_info_t epochs = COVER_computeEpochs(
+      (U32)dictBufferCapacity, (U32)ctx->nbDmers, parameters.k, 1);
+  const size_t maxZeroScoreRun = 10;
+  size_t zeroScoreRun = 0;
   size_t epoch;
   DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n",
-                epochs, epochSize);
+                (U32)epochs.num, (U32)epochs.size);
   /* Loop through the epochs until there are no more segments or the dictionary
    * is full.
    */
-  for (epoch = 0; tail > 0; epoch = (epoch + 1) % epochs) {
-    const U32 epochBegin = (U32)(epoch * epochSize);
-    const U32 epochEnd = epochBegin + epochSize;
+  for (epoch = 0; tail > 0; epoch = (epoch + 1) % epochs.num) {
+    const U32 epochBegin = (U32)(epoch * epochs.size);
+    const U32 epochEnd = epochBegin + epochs.size;
     size_t segmentSize;
     /* Select a segment */
     COVER_segment_t segment = FASTCOVER_selectSegment(
         ctx, freqs, epochBegin, epochEnd, parameters, segmentFreqs);
 
-    /* If the segment covers no dmers, then we are out of content */
+    /* If the segment covers no dmers, then we are out of content.
+     * There may be new content in other epochs, for continue for some time.
+     */
     if (segment.score == 0) {
-      break;
+      if (++zeroScoreRun >= maxZeroScoreRun) {
+          break;
+      }
+      continue;
     }
+    zeroScoreRun = 0;
 
     /* Trim the segment if necessary and if it is too small then we are done */
     segmentSize = MIN(segment.end - segment.begin + parameters.d - 1, tail);
@@ -564,6 +570,7 @@ ZDICT_trainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity,
       DISPLAYLEVEL(1, "Failed to initialize context\n");
       return ERROR(GENERIC);
     }
+    COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.nbDmers);
     /* Build the dictionary */
     DISPLAYLEVEL(2, "Building dictionary\n");
     {
@@ -616,6 +623,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
     unsigned k;
     COVER_best_t best;
     POOL_ctx *pool = NULL;
+    int warned = 0;
     /* Checks */
     if (splitPoint <= 0 || splitPoint > 1) {
       LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect splitPoint\n");
@@ -664,6 +672,10 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
         POOL_free(pool);
         return ERROR(GENERIC);
       }
+      if (!warned) {
+        COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.nbDmers);
+        warned = 1;
+      }
       /* Loop through k reusing the same context */
       for (k = kMinK; k <= kMaxK; k += kStepSize) {
         /* Prepare the arguments */

From 7ac2052dbcfccb3e052e09e2debdb779b243e099 Mon Sep 17 00:00:00 2001
From: Nehal J Wani 
Date: Sat, 23 Mar 2019 17:33:25 +0530
Subject: [PATCH 109/178] [libzstd] Specify soversion and version correctly for
 CMake build

Fixes #1512
---
 build/cmake/lib/CMakeLists.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt
index b99bb699c5a..508bee37819 100644
--- a/build/cmake/lib/CMakeLists.txt
+++ b/build/cmake/lib/CMakeLists.txt
@@ -149,7 +149,8 @@ if (ZSTD_BUILD_SHARED)
             libzstd_shared
             PROPERTIES
             OUTPUT_NAME zstd
-            SOVERSION ${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH})
+            VERSION ${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}
+            SOVERSION ${zstd_VERSION_MAJOR})
 endif ()
 
 if (ZSTD_BUILD_STATIC)

From f5cbee988b40eae8cd54f882f4c4884d22c435d2 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 22 Mar 2019 14:57:23 -0700
Subject: [PATCH 110/178] [examples] Update streaming_compression to the new
 API

---
 examples/streaming_compression.c | 81 +++++++++++++++++++++-----------
 examples/utils.h                 | 28 +++++++++++
 2 files changed, 81 insertions(+), 28 deletions(-)

diff --git a/examples/streaming_compression.c b/examples/streaming_compression.c
index e056f86f0bd..345b60ed1a0 100644
--- a/examples/streaming_compression.c
+++ b/examples/streaming_compression.c
@@ -12,51 +12,76 @@
 #include     // malloc, free, exit
 #include      // fprintf, perror, feof, fopen, etc.
 #include     // strlen, memset, strcat
+#define ZSTD_STATIC_LINKING_ONLY // TODO: Remove once the API is stable
 #include       // presumes zstd library is installed
 #include "utils.h"
 
+
 static void compressFile_orDie(const char* fname, const char* outName, int cLevel)
 {
+    /* Open the input and output files. */
     FILE* const fin  = fopen_orDie(fname, "rb");
     FILE* const fout = fopen_orDie(outName, "wb");
-    size_t const buffInSize = ZSTD_CStreamInSize();    /* can always read one full block */
+    /* Create the input and output buffers.
+     * They may be any size, but we recommend using these functions to size them.
+     * Performance will only suffer significantly for very tiny buffers.
+     */
+    size_t const buffInSize = ZSTD_CStreamInSize();
     void*  const buffIn  = malloc_orDie(buffInSize);
-    size_t const buffOutSize = ZSTD_CStreamOutSize();  /* can always flush a full block */
+    size_t const buffOutSize = ZSTD_CStreamOutSize();
     void*  const buffOut = malloc_orDie(buffOutSize);
 
-    ZSTD_CStream* const cstream = ZSTD_createCStream();
-    if (cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); }
-    size_t const initResult = ZSTD_initCStream(cstream, cLevel);
-    if (ZSTD_isError(initResult)) {
-        fprintf(stderr, "ZSTD_initCStream() error : %s \n",
-                    ZSTD_getErrorName(initResult));
-        exit(11);
-    }
+    /* Create the context. */
+    ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+    CHECK(cctx != NULL, "ZSTD_createCCtx() failed!");
 
-    size_t read, toRead = buffInSize;
-    while( (read = fread_orDie(buffIn, toRead, fin)) ) {
+    /* Set any parameters you want.
+     * Here we set the compression level, and enable the checksum.
+     */
+    CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel) );
+    CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) );
+
+    /* This loop read from the input file, compresses that entire chunk,
+     * and writes all output produced to the output file.
+     */
+    size_t const toRead = buffInSize;
+    size_t read;
+    while ((read = fread_orDie(buffIn, toRead, fin))) {
+        /* Select the flush mode.
+         * If the read may not be finished (read == toRead) we use
+         * ZSTD_e_continue. If this is the last chunk, we use ZSTD_e_end.
+         * Zstd optimizes the case where the first flush mode is ZSTD_e_end,
+         * since it knows it is compressing the entire source in one pass.
+         */
+        int const lastChunk = (read < toRead);
+        ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue;
+        /* Set the input buffer to what we just read.
+         * We compress until the input buffer is empty, each time flushing the
+         * output.
+         */
         ZSTD_inBuffer input = { buffIn, read, 0 };
-        while (input.pos < input.size) {
+        int finished;
+        do {
+            /* Compress into the output buffer and write all of the output to
+             * the file so we can reuse the buffer next iteration.
+             */
             ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
-            toRead = ZSTD_compressStream(cstream, &output , &input);   /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */
-            if (ZSTD_isError(toRead)) {
-                fprintf(stderr, "ZSTD_compressStream() error : %s \n",
-                                ZSTD_getErrorName(toRead));
-                exit(12);
-            }
-            if (toRead > buffInSize) toRead = buffInSize;   /* Safely handle case when `buffInSize` is manually changed to a value < ZSTD_CStreamInSize()*/
+            size_t const remaining = ZSTD_compressStream2(cctx, &output , &input, mode);
+            CHECK_ZSTD(remaining);
             fwrite_orDie(buffOut, output.pos, fout);
-        }
+            /* If we're on the last chunk we're finished when zstd returns 0,
+             * which means its consumed all the input AND finished the frame.
+             * Otherwise, we're finished when we've consumed all the input.
+             */
+            finished = lastChunk ? (remaining == 0) : (input.pos == input.size);
+        } while (!finished);
+        assert(input.pos == input.size);
     }
 
-    ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
-    size_t const remainingToFlush = ZSTD_endStream(cstream, &output);   /* close frame */
-    if (remainingToFlush) { fprintf(stderr, "not fully flushed"); exit(13); }
-    fwrite_orDie(buffOut, output.pos, fout);
-
-    ZSTD_freeCStream(cstream);
+    ZSTD_freeCCtx(cctx);
     fclose_orDie(fout);
-    fclose_orDie(fin);    free(buffIn);
+    fclose_orDie(fin);
+    free(buffIn);
     free(buffOut);
 }
 
diff --git a/examples/utils.h b/examples/utils.h
index 77c7a4f0ca9..9ee93339c83 100644
--- a/examples/utils.h
+++ b/examples/utils.h
@@ -20,6 +20,7 @@
 #include      // errno
 #include     // assert
 #include   // stat
+#include 
 
 /*
  * Define the returned error code from utility functions.
@@ -204,4 +205,31 @@ static void saveFile_orDie(const char* fileName, const void* buff, size_t buffSi
     }
 }
 
+/*! CHECK
+ * Check that the condition holds. If it doesn't print a message and die.
+ */
+#define CHECK(cond, ...)                        \
+    do {                                        \
+        if (!(cond)) {                          \
+            fprintf(stderr,                     \
+                    "%s:%d CHECK(%s) failed: ", \
+                    __FILE__,                   \
+                    __LINE__,                   \
+                    #cond);                     \
+            fprintf(stderr, "" __VA_ARGS__);    \
+            fprintf(stderr, "\n");              \
+            exit(1);                            \
+        }                                       \
+    } while (0)
+
+/*! CHECK_ZSTD
+ * Check the zstd error code and die if an error occurred after printing a
+ * message.
+ */
+#define CHECK_ZSTD(fn, ...)                                      \
+    do {                                                         \
+        size_t const err = (fn);                                 \
+        CHECK(!ZSTD_isError(err), "%s", ZSTD_getErrorName(err)); \
+    } while (0)
+
 #endif

From e5811e5520cc3f03f279a26a19e9be3618172f2e Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Sat, 23 Mar 2019 19:04:56 -0700
Subject: [PATCH 111/178] Extract file comparison into utility func

---
 programs/fileio.c | 26 ++++----------------------
 programs/util.c   | 17 +++++++++++++++++
 programs/util.h   |  1 +
 3 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 8a5715113c8..ffa43ab2230 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -515,28 +515,10 @@ static FILE* FIO_openDstFile(FIO_prefs_t* const prefs, const char* srcFileName,
         return stdout;
     }
 
-    /* ensure dst is not the same file as src */
-    if (srcFileName != NULL) {
-#ifdef _MSC_VER
-        /* note : Visual does not support file identification by inode.
-         *        The following work-around is limited to detecting exact name repetition only,
-         *        aka `filename` is considered different from `subdir/../filename` */
-        if (!strcmp(srcFileName, dstFileName)) {
-            DISPLAYLEVEL(1, "zstd: Refusing to open a output file which will overwrite the input file \n");
-            return NULL;
-        }
-#else
-        stat_t srcStat;
-        stat_t dstStat;
-        if (UTIL_getFileStat(srcFileName, &srcStat)
-            && UTIL_getFileStat(dstFileName, &dstStat)) {
-            if (srcStat.st_dev == dstStat.st_dev
-                && srcStat.st_ino == dstStat.st_ino) {
-                DISPLAYLEVEL(1, "zstd: Refusing to open a output file which will overwrite the input file \n");
-                return NULL;
-            }
-        }
-#endif
+    /* check if src file is the same as dst */
+    if (srcFileName != NULL && UTIL_isSameFile(srcFileName, dstFileName)) {
+        DISPLAYLEVEL(1, "zstd: Refusing to open an output file which will overwrite the input file \n");
+        return NULL;
     }
 
     if (prefs->sparseFileSupport == 1) {
diff --git a/programs/util.c b/programs/util.c
index d6466063594..622e5025f37 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -87,6 +87,23 @@ U32 UTIL_isDirectory(const char* infilename)
     return 0;
 }
 
+int UTIL_isSameFile(const char* file1, const char* file2)
+{
+#if defined(_MSC_VER)
+    /* note : Visual does not support file identification by inode.
+     *        The following work-around is limited to detecting exact name repetition only,
+     *        aka `filename` is considered different from `subdir/../filename` */
+    return !strcmp(file1, file2);
+#else
+    stat_t file1Stat;
+    stat_t file2Stat;
+    return UTIL_getFileStat(file1, &file1Stat)
+        && UTIL_getFileStat(file2, &file2Stat)
+        && (file1Stat.st_dev == file2Stat.st_dev)
+        && (file1Stat.st_ino == file2Stat.st_ino);
+#endif
+}
+
 U32 UTIL_isLink(const char* infilename)
 {
 /* macro guards, as defined in : https://linux.die.net/man/2/lstat */
diff --git a/programs/util.h b/programs/util.h
index f78bcbe1b3f..eee7ebfc33b 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -174,6 +174,7 @@ int UTIL_isRegularFile(const char* infilename);
 int UTIL_setFileStat(const char* filename, stat_t* statbuf);
 U32 UTIL_isDirectory(const char* infilename);
 int UTIL_getFileStat(const char* infilename, stat_t* statbuf);
+int UTIL_isSameFile(const char* file1, const char* file2);
 
 U32 UTIL_isLink(const char* infilename);
 #define UTIL_FILESIZE_UNKNOWN  ((U64)(-1))

From 1290933d192c4a42db5ad5b90978f53a8a8a8d26 Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Sat, 23 Mar 2019 21:53:13 -0700
Subject: [PATCH 112/178] Implement file check

---
 programs/fileio.c | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index ffa43ab2230..5672de02d41 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -515,7 +515,7 @@ static FILE* FIO_openDstFile(FIO_prefs_t* const prefs, const char* srcFileName,
         return stdout;
     }
 
-    /* check if src file is the same as dst */
+    /* ensure dst is not the same as src */
     if (srcFileName != NULL && UTIL_isSameFile(srcFileName, dstFileName)) {
         DISPLAYLEVEL(1, "zstd: Refusing to open an output file which will overwrite the input file \n");
         return NULL;
@@ -610,6 +610,7 @@ typedef struct {
     size_t srcBufferSize;
     void*  dstBuffer;
     size_t dstBufferSize;
+    const char* dictFileName;
     ZSTD_CStream* cctx;
 } cRess_t;
 
@@ -637,6 +638,7 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
         size_t const dictBuffSize = FIO_createDictBuffer(&dictBuffer, dictFileName);   /* works with dictFileName==NULL */
         if (dictFileName && (dictBuffer==NULL))
             EXM_THROW(32, "allocation error : can't create dictBuffer");
+        ress.dictFileName = dictFileName;
 
         if (prefs->adaptiveMode && !prefs->ldmFlag && !comprParams.windowLog)
             comprParams.windowLog = ADAPT_WINDOWLOG_DEFAULT;
@@ -1290,12 +1292,18 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs,
 {
     int result;
 
-    /* File check */
+    /* ensure src is not a directory */
     if (UTIL_isDirectory(srcFileName)) {
         DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName);
         return 1;
     }
 
+    /* ensure src is not the same as dict */
+    if (UTIL_isSameFile(srcFileName, ress.dictFileName)) {
+        DISPLAYLEVEL(1, "zstd: Refusing to use %s as an input file and dictionary \n", srcFileName);
+        return 1;
+    }
+
     ress.srcFile = FIO_openSrcFile(srcFileName);
     if (ress.srcFile == NULL) return 1;   /* srcFile could not be opened */
 

From 8ea219d8c663264e57f4882ce1d9401dab0769c9 Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Sat, 23 Mar 2019 21:59:30 -0700
Subject: [PATCH 113/178] Modify error msg

---
 programs/fileio.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 5672de02d41..7da7baaffc8 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1300,7 +1300,7 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs,
 
     /* ensure src is not the same as dict */
     if (UTIL_isSameFile(srcFileName, ress.dictFileName)) {
-        DISPLAYLEVEL(1, "zstd: Refusing to use %s as an input file and dictionary \n", srcFileName);
+        DISPLAYLEVEL(1, "zstd: cannot use %s as an input file and dictionary \n", srcFileName);
         return 1;
     }
 

From 5333e41ab38668732b6403cf5a170e6c92d5b45c Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Sun, 24 Mar 2019 00:23:50 -0700
Subject: [PATCH 114/178] Add NULL check for dict

---
 programs/fileio.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 7da7baaffc8..cbd7b92bc61 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1298,8 +1298,8 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs,
         return 1;
     }
 
-    /* ensure src is not the same as dict */
-    if (UTIL_isSameFile(srcFileName, ress.dictFileName)) {
+    /* ensure src is not the same as dict (if present) */
+    if (ress.dictFileName != NULL && UTIL_isSameFile(srcFileName, ress.dictFileName)) {
         DISPLAYLEVEL(1, "zstd: cannot use %s as an input file and dictionary \n", srcFileName);
         return 1;
     }

From 2b4491d81a524ab7e5307ffb430d2f7d0c4b1e9f Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Sun, 24 Mar 2019 00:47:13 -0700
Subject: [PATCH 115/178] Add CLI test to validate error

---
 tests/playTests.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/playTests.sh b/tests/playTests.sh
index 493542960ed..137f1a44170 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -410,6 +410,8 @@ $ECHO "- Create first dictionary "
 TESTFILE=../programs/zstdcli.c
 $ZSTD --train *.c ../programs/*.c -o tmpDict
 cp $TESTFILE tmp
+$ECHO "- Compress dictionary with itself"
+$ZSTD -f tmpDict -D tmpDict | grep "cannot use" && die "expected error : cannot compress dictionary with itself"
 $ECHO "- Dictionary compression roundtrip"
 $ZSTD -f tmp -D tmpDict
 $ZSTD -d tmp.zst -D tmpDict -fo result

From b25d7eacf27bce19ea823e1baba898f3553e3047 Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Sun, 24 Mar 2019 03:40:03 -0700
Subject: [PATCH 116/178] Rename test

---
 tests/playTests.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/playTests.sh b/tests/playTests.sh
index 137f1a44170..f50bc903076 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -410,8 +410,8 @@ $ECHO "- Create first dictionary "
 TESTFILE=../programs/zstdcli.c
 $ZSTD --train *.c ../programs/*.c -o tmpDict
 cp $TESTFILE tmp
-$ECHO "- Compress dictionary with itself"
-$ZSTD -f tmpDict -D tmpDict | grep "cannot use" && die "expected error : cannot compress dictionary with itself"
+$ECHO "- Test dictionary compression with tmpDict as an input file and dictionary"
+$ZSTD -f tmpDict -D tmpDict&& die "compression error not detected!"
 $ECHO "- Dictionary compression roundtrip"
 $ZSTD -f tmp -D tmpDict
 $ZSTD -d tmp.zst -D tmpDict -fo result

From 44f77b5c71d5dae08f3c50e495184d422135997c Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Sun, 24 Mar 2019 03:42:11 -0700
Subject: [PATCH 117/178] Add whitespace to test case

---
 tests/playTests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/playTests.sh b/tests/playTests.sh
index f50bc903076..d22f617e456 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -411,7 +411,7 @@ TESTFILE=../programs/zstdcli.c
 $ZSTD --train *.c ../programs/*.c -o tmpDict
 cp $TESTFILE tmp
 $ECHO "- Test dictionary compression with tmpDict as an input file and dictionary"
-$ZSTD -f tmpDict -D tmpDict&& die "compression error not detected!"
+$ZSTD -f tmpDict -D tmpDict && die "compression error not detected!"
 $ECHO "- Dictionary compression roundtrip"
 $ZSTD -f tmp -D tmpDict
 $ZSTD -d tmp.zst -D tmpDict -fo result

From dca73db30c999ddce7d734d6f4f603b36c92ce51 Mon Sep 17 00:00:00 2001
From: shakeelrao 
Date: Thu, 28 Mar 2019 17:50:34 -0700
Subject: [PATCH 118/178] fix srcSize typo and add new UTIL func to comment

---
 lib/zstd.h        | 2 +-
 programs/fileio.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/zstd.h b/lib/zstd.h
index c1c0fe744c0..ea2b9a5e490 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -1136,7 +1136,7 @@ ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t
  *  note 3  : when the decompressed size field isn't available, the upper-bound for that frame is calculated by:
  *              upper-bound = # blocks * min(128 KB, Window_Size)
  */
-ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSice);
+ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize);
 
 /*! ZSTD_frameHeaderSize() :
  *  srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX.
diff --git a/programs/fileio.c b/programs/fileio.c
index cbd7b92bc61..412ef4762c7 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -24,7 +24,7 @@
 *  Includes
 ***************************************/
 #include "platform.h"   /* Large Files support, SET_BINARY_MODE */
-#include "util.h"       /* UTIL_getFileSize, UTIL_isRegularFile */
+#include "util.h"       /* UTIL_getFileSize, UTIL_isRegularFile, UTIL_isSameFile */
 #include       /* fprintf, fopen, fread, _fileno, stdin, stdout */
 #include      /* malloc, free */
 #include      /* strcmp, strlen */

From f00407b640512b9110fc0016757103b8ea6b1b9a Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 29 Mar 2019 10:39:16 -0600
Subject: [PATCH 119/178] Split out zstd_fast dict match state function

---
 lib/compress/zstd_fast.c | 289 ++++++++++++++++++++++-----------------
 1 file changed, 166 insertions(+), 123 deletions(-)

diff --git a/lib/compress/zstd_fast.c b/lib/compress/zstd_fast.c
index 40ba0f73e65..d59e0e73ec2 100644
--- a/lib/compress/zstd_fast.c
+++ b/lib/compress/zstd_fast.c
@@ -45,7 +45,7 @@ FORCE_INLINE_TEMPLATE
 size_t ZSTD_compressBlock_fast_generic(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
         void const* src, size_t srcSize,
-        U32 const mls, ZSTD_dictMode_e const dictMode)
+        U32 const mls)
 {
     const ZSTD_compressionParameters* const cParams = &ms->cParams;
     U32* const hashTable = ms->hashTable;
@@ -63,47 +63,13 @@ size_t ZSTD_compressBlock_fast_generic(
     U32 offset_1=rep[0], offset_2=rep[1];
     U32 offsetSaved = 0;
 
-    const ZSTD_matchState_t* const dms = ms->dictMatchState;
-    const ZSTD_compressionParameters* const dictCParams =
-                                     dictMode == ZSTD_dictMatchState ?
-                                     &dms->cParams : NULL;
-    const U32* const dictHashTable = dictMode == ZSTD_dictMatchState ?
-                                     dms->hashTable : NULL;
-    const U32 dictStartIndex       = dictMode == ZSTD_dictMatchState ?
-                                     dms->window.dictLimit : 0;
-    const BYTE* const dictBase     = dictMode == ZSTD_dictMatchState ?
-                                     dms->window.base : NULL;
-    const BYTE* const dictStart    = dictMode == ZSTD_dictMatchState ?
-                                     dictBase + dictStartIndex : NULL;
-    const BYTE* const dictEnd      = dictMode == ZSTD_dictMatchState ?
-                                     dms->window.nextSrc : NULL;
-    const U32 dictIndexDelta       = dictMode == ZSTD_dictMatchState ?
-                                     prefixStartIndex - (U32)(dictEnd - dictBase) :
-                                     0;
-    const U32 dictAndPrefixLength  = (U32)(ip - prefixStart + dictEnd - dictStart);
-    const U32 dictHLog             = dictMode == ZSTD_dictMatchState ?
-                                     dictCParams->hashLog : hlog;
-
-    assert(dictMode == ZSTD_noDict || dictMode == ZSTD_dictMatchState);
-
-    /* otherwise, we would get index underflow when translating a dict index
-     * into a local index */
-    assert(dictMode != ZSTD_dictMatchState
-        || prefixStartIndex >= (U32)(dictEnd - dictBase));
-
     /* init */
-    ip += (dictAndPrefixLength == 0);
-    if (dictMode == ZSTD_noDict) {
+    ip += (ip == prefixStart);
+    {
         U32 const maxRep = (U32)(ip - prefixStart);
         if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0;
         if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0;
     }
-    if (dictMode == ZSTD_dictMatchState) {
-        /* dictMatchState repCode checks don't currently handle repCode == 0
-         * disabling. */
-        assert(offset_1 <= dictAndPrefixLength);
-        assert(offset_2 <= dictAndPrefixLength);
-    }
 
     /* Main Search Loop */
     while (ip < ilimit) {   /* < instead of <=, because repcode check at (ip+1) */
@@ -112,54 +78,13 @@ size_t ZSTD_compressBlock_fast_generic(
         U32 const current = (U32)(ip-base);
         U32 const matchIndex = hashTable[h];
         const BYTE* match = base + matchIndex;
-        const U32 repIndex = current + 1 - offset_1;
-        const BYTE* repMatch = (dictMode == ZSTD_dictMatchState
-                            && repIndex < prefixStartIndex) ?
-                               dictBase + (repIndex - dictIndexDelta) :
-                               base + repIndex;
         hashTable[h] = current;   /* update hash table */
 
-        if ( (dictMode == ZSTD_dictMatchState)
-          && ((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex isn't overlapping dict + prefix */
-          && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
-            const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
-            mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4;
-            ip++;
-            ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH);
-        } else if ( dictMode == ZSTD_noDict
-                 && ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) {
+        if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) {
             mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4;
             ip++;
             ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH);
-        } else if ( (matchIndex <= prefixStartIndex) ) {
-            if (dictMode == ZSTD_dictMatchState) {
-                size_t const dictHash = ZSTD_hashPtr(ip, dictHLog, mls);
-                U32 const dictMatchIndex = dictHashTable[dictHash];
-                const BYTE* dictMatch = dictBase + dictMatchIndex;
-                if (dictMatchIndex <= dictStartIndex ||
-                    MEM_read32(dictMatch) != MEM_read32(ip)) {
-                    assert(stepSize >= 1);
-                    ip += ((ip-anchor) >> kSearchStrength) + stepSize;
-                    continue;
-                } else {
-                    /* found a dict match */
-                    U32 const offset = (U32)(current-dictMatchIndex-dictIndexDelta);
-                    mLength = ZSTD_count_2segments(ip+4, dictMatch+4, iend, dictEnd, prefixStart) + 4;
-                    while (((ip>anchor) & (dictMatch>dictStart))
-                         && (ip[-1] == dictMatch[-1])) {
-                        ip--; dictMatch--; mLength++;
-                    } /* catch up */
-                    offset_2 = offset_1;
-                    offset_1 = offset;
-                    ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
-                }
-            } else {
-                assert(stepSize >= 1);
-                ip += ((ip-anchor) >> kSearchStrength) + stepSize;
-                continue;
-            }
-        } else if (MEM_read32(match) != MEM_read32(ip)) {
-            /* it's not a match, and we're not going to check the dictionary */
+        } else if ((matchIndex <= prefixStartIndex) || MEM_read32(match) != MEM_read32(ip)) {
             assert(stepSize >= 1);
             ip += ((ip-anchor) >> kSearchStrength) + stepSize;
             continue;
@@ -184,42 +109,20 @@ size_t ZSTD_compressBlock_fast_generic(
             hashTable[ZSTD_hashPtr(base+current+2, hlog, mls)] = current+2;  /* here because current+2 could be > iend-8 */
             hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base);
 
-            /* check immediate repcode */
-            if (dictMode == ZSTD_dictMatchState) {
-                while (ip <= ilimit) {
-                    U32 const current2 = (U32)(ip-base);
-                    U32 const repIndex2 = current2 - offset_2;
-                    const BYTE* repMatch2 = repIndex2 < prefixStartIndex ?
-                            dictBase - dictIndexDelta + repIndex2 :
-                            base + repIndex2;
-                    if ( ((U32)((prefixStartIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */)
-                       && (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
-                        const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
-                        size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
-                        U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset;   /* swap offset_2 <=> offset_1 */
-                        ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH);
-                        hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2;
-                        ip += repLength2;
-                        anchor = ip;
-                        continue;
-                    }
-                    break;
-                }
+            while ( (ip <= ilimit)
+                 && ( (offset_2>0)
+                    & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) {
+                /* store sequence */
+                size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4;
+                U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff;  /* swap offset_2 <=> offset_1 */
+                hashTable[ZSTD_hashPtr(ip, hlog, mls)] = (U32)(ip-base);
+                ZSTD_storeSeq(seqStore, 0, anchor, 0, rLength-MINMATCH);
+                ip += rLength;
+                anchor = ip;
+                continue;   /* faster when present ... (?) */
             }
-
-            if (dictMode == ZSTD_noDict) {
-                while ( (ip <= ilimit)
-                     && ( (offset_2>0)
-                        & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) {
-                    /* store sequence */
-                    size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4;
-                    U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff;  /* swap offset_2 <=> offset_1 */
-                    hashTable[ZSTD_hashPtr(ip, hlog, mls)] = (U32)(ip-base);
-                    ZSTD_storeSeq(seqStore, 0, anchor, 0, rLength-MINMATCH);
-                    ip += rLength;
-                    anchor = ip;
-                    continue;   /* faster when present ... (?) */
-    }   }   }   }
+        }
+    }
 
     /* save reps for next block */
     rep[0] = offset_1 ? offset_1 : offsetSaved;
@@ -241,16 +144,156 @@ size_t ZSTD_compressBlock_fast(
     {
     default: /* includes case 3 */
     case 4 :
-        return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_noDict);
+        return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 4);
     case 5 :
-        return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_noDict);
+        return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 5);
     case 6 :
-        return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_noDict);
+        return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 6);
     case 7 :
-        return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_noDict);
+        return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 7);
     }
 }
 
+FORCE_INLINE_TEMPLATE
+size_t ZSTD_compressBlock_fast_dictMatchState_generic(
+        ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+        void const* src, size_t srcSize, U32 const mls)
+{
+    const ZSTD_compressionParameters* const cParams = &ms->cParams;
+    U32* const hashTable = ms->hashTable;
+    U32 const hlog = cParams->hashLog;
+    /* support stepSize of 0 */
+    U32 const stepSize = cParams->targetLength + !(cParams->targetLength);
+    const BYTE* const base = ms->window.base;
+    const BYTE* const istart = (const BYTE*)src;
+    const BYTE* ip = istart;
+    const BYTE* anchor = istart;
+    const U32   prefixStartIndex = ms->window.dictLimit;
+    const BYTE* const prefixStart = base + prefixStartIndex;
+    const BYTE* const iend = istart + srcSize;
+    const BYTE* const ilimit = iend - HASH_READ_SIZE;
+    U32 offset_1=rep[0], offset_2=rep[1];
+    U32 offsetSaved = 0;
+
+    const ZSTD_matchState_t* const dms = ms->dictMatchState;
+    const ZSTD_compressionParameters* const dictCParams = &dms->cParams ;
+    const U32* const dictHashTable = dms->hashTable;
+    const U32 dictStartIndex       = dms->window.dictLimit;
+    const BYTE* const dictBase     = dms->window.base;
+    const BYTE* const dictStart    = dictBase + dictStartIndex;
+    const BYTE* const dictEnd      = dms->window.nextSrc;
+    const U32 dictIndexDelta       = prefixStartIndex - (U32)(dictEnd - dictBase);
+    const U32 dictAndPrefixLength  = (U32)(ip - prefixStart + dictEnd - dictStart);
+    const U32 dictHLog             = dictCParams->hashLog;
+
+    /* otherwise, we would get index underflow when translating a dict index
+     * into a local index */
+    assert(prefixStartIndex >= (U32)(dictEnd - dictBase));
+
+    /* init */
+    ip += (dictAndPrefixLength == 0);
+    /* dictMatchState repCode checks don't currently handle repCode == 0
+     * disabling. */
+    assert(offset_1 <= dictAndPrefixLength);
+    assert(offset_2 <= dictAndPrefixLength);
+
+    /* Main Search Loop */
+    while (ip < ilimit) {   /* < instead of <=, because repcode check at (ip+1) */
+        size_t mLength;
+        size_t const h = ZSTD_hashPtr(ip, hlog, mls);
+        U32 const current = (U32)(ip-base);
+        U32 const matchIndex = hashTable[h];
+        const BYTE* match = base + matchIndex;
+        const U32 repIndex = current + 1 - offset_1;
+        const BYTE* repMatch = (repIndex < prefixStartIndex) ?
+                               dictBase + (repIndex - dictIndexDelta) :
+                               base + repIndex;
+        hashTable[h] = current;   /* update hash table */
+
+        if ( ((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex isn't overlapping dict + prefix */
+          && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
+            const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
+            mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4;
+            ip++;
+            ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH);
+        } else if ( (matchIndex <= prefixStartIndex) ) {
+            size_t const dictHash = ZSTD_hashPtr(ip, dictHLog, mls);
+            U32 const dictMatchIndex = dictHashTable[dictHash];
+            const BYTE* dictMatch = dictBase + dictMatchIndex;
+            if (dictMatchIndex <= dictStartIndex ||
+                MEM_read32(dictMatch) != MEM_read32(ip)) {
+                assert(stepSize >= 1);
+                ip += ((ip-anchor) >> kSearchStrength) + stepSize;
+                continue;
+            } else {
+                /* found a dict match */
+                U32 const offset = (U32)(current-dictMatchIndex-dictIndexDelta);
+                mLength = ZSTD_count_2segments(ip+4, dictMatch+4, iend, dictEnd, prefixStart) + 4;
+                while (((ip>anchor) & (dictMatch>dictStart))
+                     && (ip[-1] == dictMatch[-1])) {
+                    ip--; dictMatch--; mLength++;
+                } /* catch up */
+                offset_2 = offset_1;
+                offset_1 = offset;
+                ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
+            }
+        } else if (MEM_read32(match) != MEM_read32(ip)) {
+            /* it's not a match, and we're not going to check the dictionary */
+            assert(stepSize >= 1);
+            ip += ((ip-anchor) >> kSearchStrength) + stepSize;
+            continue;
+        } else {
+            /* found a regular match */
+            U32 const offset = (U32)(ip-match);
+            mLength = ZSTD_count(ip+4, match+4, iend) + 4;
+            while (((ip>anchor) & (match>prefixStart))
+                 && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
+            offset_2 = offset_1;
+            offset_1 = offset;
+            ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
+        }
+
+        /* match found */
+        ip += mLength;
+        anchor = ip;
+
+        if (ip <= ilimit) {
+            /* Fill Table */
+            assert(base+current+2 > istart);  /* check base overflow */
+            hashTable[ZSTD_hashPtr(base+current+2, hlog, mls)] = current+2;  /* here because current+2 could be > iend-8 */
+            hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base);
+
+            /* check immediate repcode */
+            while (ip <= ilimit) {
+                U32 const current2 = (U32)(ip-base);
+                U32 const repIndex2 = current2 - offset_2;
+                const BYTE* repMatch2 = repIndex2 < prefixStartIndex ?
+                        dictBase - dictIndexDelta + repIndex2 :
+                        base + repIndex2;
+                if ( ((U32)((prefixStartIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */)
+                   && (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
+                    const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
+                    size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
+                    U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset;   /* swap offset_2 <=> offset_1 */
+                    ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH);
+                    hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2;
+                    ip += repLength2;
+                    anchor = ip;
+                    continue;
+                }
+                break;
+            }
+        }
+    }
+
+    /* save reps for next block */
+    rep[0] = offset_1 ? offset_1 : offsetSaved;
+    rep[1] = offset_2 ? offset_2 : offsetSaved;
+
+    /* Return the last literals size */
+    return iend - anchor;
+}
+
 size_t ZSTD_compressBlock_fast_dictMatchState(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
         void const* src, size_t srcSize)
@@ -262,13 +305,13 @@ size_t ZSTD_compressBlock_fast_dictMatchState(
     {
     default: /* includes case 3 */
     case 4 :
-        return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_dictMatchState);
+        return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 4);
     case 5 :
-        return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_dictMatchState);
+        return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 5);
     case 6 :
-        return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_dictMatchState);
+        return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 6);
     case 7 :
-        return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_dictMatchState);
+        return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 7);
     }
 }
 

From fb13d757af0e8c0c900ade00d3d5b9079685bc07 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Mon, 1 Apr 2019 16:41:06 -0700
Subject: [PATCH 120/178] [examples] Update multiple_streaming_compression.c

Update to use the new streaming API. Making progress on Issue #1548.

Tested that multiple files could be compressed, and that the output
is the same as calling `streaming_compression` multiple times with
the same compression level, and that it can be decompressed.
---
 examples/multiple_streaming_compression.c | 68 ++++++++++++++---------
 1 file changed, 42 insertions(+), 26 deletions(-)

diff --git a/examples/multiple_streaming_compression.c b/examples/multiple_streaming_compression.c
index 442ff40ae29..0054c7b0093 100644
--- a/examples/multiple_streaming_compression.c
+++ b/examples/multiple_streaming_compression.c
@@ -18,7 +18,7 @@
 #include      // fprintf, perror, feof
 #include     // strerror
 #include      // errno
-#define ZSTD_STATIC_LINKING_ONLY  // streaming API defined as "experimental" for the time being
+#define ZSTD_STATIC_LINKING_ONLY  // TODO: Remove once the API is stable
 #include       // presumes zstd library is installed
 #include "utils.h"
 
@@ -27,53 +27,68 @@ typedef struct {
     void* buffOut;
     size_t buffInSize;
     size_t buffOutSize;
-    ZSTD_CStream* cstream;
-} resources ;
+    ZSTD_CCtx* cctx;
+} resources;
 
-static resources createResources_orDie()
+static resources createResources_orDie(int cLevel)
 {
     resources ress;
     ress.buffInSize = ZSTD_CStreamInSize();   /* can always read one full block */
     ress.buffOutSize= ZSTD_CStreamOutSize();  /* can always flush a full block */
     ress.buffIn = malloc_orDie(ress.buffInSize);
     ress.buffOut= malloc_orDie(ress.buffOutSize);
-    ress.cstream = ZSTD_createCStream();
-    if (ress.cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); }
+    ress.cctx = ZSTD_createCCtx();
+    CHECK(ress.cctx != NULL, "ZSTD_createCCtx() failed!");
+
+    /* Set any compression parameters you want here.
+     * They will persist for every compression operation.
+     * Here we set the compression level, and enable the checksum.
+     */
+    CHECK_ZSTD( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, cLevel) );
+    CHECK_ZSTD( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_checksumFlag, 1) );
     return ress;
 }
 
 static void freeResources(resources ress)
 {
-    ZSTD_freeCStream(ress.cstream);
+    ZSTD_freeCCtx(ress.cctx);
     free(ress.buffIn);
     free(ress.buffOut);
 }
 
-static void compressFile_orDie(resources ress, const char* fname, const char* outName, int cLevel)
+static void compressFile_orDie(resources ress, const char* fname, const char* outName)
 {
+    // Open the input and output files.
     FILE* const fin  = fopen_orDie(fname, "rb");
     FILE* const fout = fopen_orDie(outName, "wb");
 
-    size_t const initResult = ZSTD_initCStream(ress.cstream, cLevel);
-    if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initCStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
+    /* Reset the context to a clean state to start a new compression operation.
+     * The parameters are sticky, so we keep the compression level and extra
+     * parameters that we set in createResources_orDie().
+     */
+    CHECK_ZSTD( ZSTD_CCtx_reset(ress.cctx, ZSTD_reset_session_only) );
+
+    size_t const toRead = ress.buffInSize;
+    size_t read;
+    while ( (read = fread_orDie(ress.buffIn, toRead, fin)) ) {
+        /* This loop is the same as streaming_compression.c.
+         * See that file for detailed comments.
+         */
+        int const lastChunk = (read < toRead);
+        ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue;
 
-    size_t read, toRead = ress.buffInSize;
-    while( (read = fread_orDie(ress.buffIn, toRead, fin)) ) {
         ZSTD_inBuffer input = { ress.buffIn, read, 0 };
-        while (input.pos < input.size) {
-            ZSTD_outBuffer output = { ress.buffOut, ress.buffOutSize, 0 };
-            toRead = ZSTD_compressStream(ress.cstream, &output , &input);   /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */
-            if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_compressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); }
-            if (toRead > ress.buffInSize) toRead = ress.buffInSize;   /* Safely handle when `buffInSize` is manually changed to a smaller value */
-            fwrite_orDie(ress.buffOut, output.pos, fout);
-        }
+        int finished;
+        do {
+          ZSTD_outBuffer output = { ress.buffOut, ress.buffOutSize, 0 };
+          size_t const remaining = ZSTD_compressStream2(ress.cctx, &output, &input, mode);
+          CHECK_ZSTD(remaining);
+          fwrite_orDie(ress.buffOut, output.pos, fout);
+          finished = lastChunk ? (remaining == 0) : (input.pos == input.size);
+        } while (!finished);
+        assert(input.pos == input.size);
     }
 
-    ZSTD_outBuffer output = { ress.buffOut, ress.buffOutSize, 0 };
-    size_t const remainingToFlush = ZSTD_endStream(ress.cstream, &output);   /* close frame */
-    if (remainingToFlush) { fprintf(stderr, "not fully flushed"); exit(13); }
-    fwrite_orDie(ress.buffOut, output.pos, fout);
-
     fclose_orDie(fout);
     fclose_orDie(fin);
 }
@@ -89,7 +104,8 @@ int main(int argc, const char** argv)
         return 1;
     }
 
-    resources const ress = createResources_orDie();
+    int const cLevel = 7;
+    resources const ress = createResources_orDie(cLevel);
     void* ofnBuffer = NULL;
     size_t ofnbSize = 0;
 
@@ -106,7 +122,7 @@ int main(int argc, const char** argv)
         memset(ofnBuffer, 0, ofnSize);
         strcat(ofnBuffer, ifn);
         strcat(ofnBuffer, ".zst");
-        compressFile_orDie(ress, ifn, ofnBuffer, 7);
+        compressFile_orDie(ress, ifn, ofnBuffer);
     }
 
     freeResources(ress);

From 04325cbc2f29b6b0915a9c3e69a0698d4e6d5409 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Mon, 1 Apr 2019 17:33:49 -0700
Subject: [PATCH 121/178] Fix indentation

---
 examples/multiple_streaming_compression.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/examples/multiple_streaming_compression.c b/examples/multiple_streaming_compression.c
index 0054c7b0093..0bfb337ca1c 100644
--- a/examples/multiple_streaming_compression.c
+++ b/examples/multiple_streaming_compression.c
@@ -80,11 +80,11 @@ static void compressFile_orDie(resources ress, const char* fname, const char* ou
         ZSTD_inBuffer input = { ress.buffIn, read, 0 };
         int finished;
         do {
-          ZSTD_outBuffer output = { ress.buffOut, ress.buffOutSize, 0 };
-          size_t const remaining = ZSTD_compressStream2(ress.cctx, &output, &input, mode);
-          CHECK_ZSTD(remaining);
-          fwrite_orDie(ress.buffOut, output.pos, fout);
-          finished = lastChunk ? (remaining == 0) : (input.pos == input.size);
+            ZSTD_outBuffer output = { ress.buffOut, ress.buffOutSize, 0 };
+            size_t const remaining = ZSTD_compressStream2(ress.cctx, &output, &input, mode);
+            CHECK_ZSTD(remaining);
+            fwrite_orDie(ress.buffOut, output.pos, fout);
+            finished = lastChunk ? (remaining == 0) : (input.pos == input.size);
         } while (!finished);
         assert(input.pos == input.size);
     }

From 56682a7709b792b826fbb39216044369067e8fa4 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Mon, 1 Apr 2019 17:51:28 -0700
Subject: [PATCH 122/178] Fix ZSTD_estimateCStreamSize_usingCCtxParams()

It wasn't using the ZSTD_CCtx_params correctly. It must actualize
the compression parameters by calling ZSTD_getCParamsFromCCtxParams()
to get the real window log.

Tested by updating the streaming memory usage example in the next
commit. The CHECK() failed before this patch, and passes after.

I also added a unit test to zstreamtest.c that failed before this
patch, and passes after.
---
 lib/compress/zstd_compress.c |  8 +++++---
 tests/zstreamtest.c          | 14 ++++++++++++++
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index fe59f522fd2..f2b9e03edc4 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1106,9 +1106,11 @@ size_t ZSTD_estimateCCtxSize(int compressionLevel)
 size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
 {
     RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only.");
-    {   size_t const CCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params);
-        size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params->cParams.windowLog);
-        size_t const inBuffSize = ((size_t)1 << params->cParams.windowLog) + blockSize;
+    {   ZSTD_compressionParameters const cParams =
+                ZSTD_getCParamsFromCCtxParams(params, 0, 0);
+        size_t const CCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params);
+        size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
+        size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize;
         size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1;
         size_t const streamingSize = inBuffSize + outBuffSize;
 
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index e26375e07b2..51cb27b5466 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -344,6 +344,20 @@ static int basicUnitTests(U32 seed, double compressibility)
         DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)(cstreamSize + cdictSize));
     }
 
+    /* context size functions */
+    DISPLAYLEVEL(3, "test%3i : estimate CStream size using CCtxParams : ", testNb++);
+    {   ZSTD_CCtx_params* const params = ZSTD_createCCtxParams();
+        size_t cstreamSize, cctxSize;
+        CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_compressionLevel, 19) );
+        cstreamSize = ZSTD_estimateCStreamSize_usingCCtxParams(params);
+        CHECK_Z(cstreamSize);
+        cctxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params);
+        CHECK_Z(cctxSize);
+        if (cstreamSize <= cctxSize + 2 * ZSTD_BLOCKSIZE_MAX) goto _output_error;
+        ZSTD_freeCCtxParams(params);
+        DISPLAYLEVEL(3, "OK \n");
+    }
+
     DISPLAYLEVEL(3, "test%3i : check actual CStream size : ", testNb++);
     {   size_t const s = ZSTD_sizeof_CStream(zc);
         if (ZSTD_isError(s)) goto _output_error;

From de58910b5a2fe16451d293f5b2e55a686d1a965d Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Mon, 1 Apr 2019 17:25:26 -0700
Subject: [PATCH 123/178] [examples] Update streaming_decompression.c

Update to use the new streaming API. Making progress on Issue #1548.

Tested that it can decompress files produced by `streaming_compression`.
Tested that it can decompress two frames concatenated together.
Tested that it fails on corrupted data.
---
 examples/streaming_decompression.c | 36 +++++++++++++++++++++---------
 1 file changed, 25 insertions(+), 11 deletions(-)

diff --git a/examples/streaming_decompression.c b/examples/streaming_decompression.c
index 13c3c509ead..5db5e7a73e3 100644
--- a/examples/streaming_decompression.c
+++ b/examples/streaming_decompression.c
@@ -25,26 +25,40 @@ static void decompressFile_orDie(const char* fname)
     size_t const buffOutSize = ZSTD_DStreamOutSize();  /* Guarantee to successfully flush at least one complete compressed block in all circumstances. */
     void*  const buffOut = malloc_orDie(buffOutSize);
 
-    ZSTD_DStream* const dstream = ZSTD_createDStream();
-    if (dstream==NULL) { fprintf(stderr, "ZSTD_createDStream() error \n"); exit(10); }
+    ZSTD_DCtx* const dctx = ZSTD_createDCtx();
+    CHECK(dctx != NULL, "ZSTD_createDCtx() failed!");
 
-    /* In more complex scenarios, a file may consist of multiple appended frames (ex : pzstd).
-    *  The following example decompresses only the first frame.
-    *  It is compatible with other provided streaming examples */
-    size_t const initResult = ZSTD_initDStream(dstream);
-    if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initDStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
-    size_t read, toRead = initResult;
+    /* This loop assumes that the input file is one or more concatenated zstd
+     * streams. This example won't work if there is trailing non-zstd data at
+     * the end, but streaming decompression in general handles this case.
+     * ZSTD_decompressStream() returns 0 exactly when the frame is completed,
+     * and doesn't consume input after the frame.
+     */
+    size_t const toRead = buffInSize;
+    size_t read;
     while ( (read = fread_orDie(buffIn, toRead, fin)) ) {
         ZSTD_inBuffer input = { buffIn, read, 0 };
+        /* Given a valid frame, zstd won't consume the last byte of the frame
+         * until it has flushed all of the decompressed data of the frame.
+         * Therefore, instead of checking if the return code is 0, we can
+         * decompress just check if input.pos < input.size.
+         */
         while (input.pos < input.size) {
             ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
-            toRead = ZSTD_decompressStream(dstream, &output , &input);  /* toRead : size of next compressed block */
-            if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_decompressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); }
+            /* The return code is zero if the frame is complete, but there may
+             * be multiple frames concatenated together. Zstd will automatically
+             * reset the context when a frame is complete. Still, calling
+             * ZSTD_DCtx_reset() can be useful to reset the context to a clean
+             * state, for instance if the last decompression call returned an
+             * error.
+             */
+            size_t const ret = ZSTD_decompressStream(dctx, &output , &input);
+            CHECK_ZSTD(ret);
             fwrite_orDie(buffOut, output.pos, fout);
         }
     }
 
-    ZSTD_freeDStream(dstream);
+    ZSTD_freeDCtx(dctx);
     fclose_orDie(fin);
     fclose_orDie(fout);
     free(buffIn);

From 95624b77e477752b3c380c22be7bcf67f06c9934 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 29 Mar 2019 12:31:21 -0600
Subject: [PATCH 124/178] [libzstd] Speed up single segment zstd_fast by 5%

This PR is based on top of PR #1563.

The optimization is to process two input pointers per loop.
It is based on ideas from [igzip] level 1, and talking to @gbtucker.

| Platform                | Silesia     | Enwik8 |
|-------------------------|-------------|--------|
| OSX clang-10            | +5.3%       | +5.4%  |
| i9 5 GHz gcc-8          | +6.6%       | +6.6%  |
| i9 5 GHz clang-7        | +8.0%       | +8.0%  |
| Skylake 2.4 GHz gcc-4.8 | +6.3%       | +7.9%  |
| Skylake 2.4 GHz clang-7 | +6.2%       | +7.5%  |

Testing on all Silesia files on my Intel i9-9900k with gcc-8

| Silesia File | Ratio Change | Speed Change |
|--------------|--------------|--------------|
| silesia.tar  | +0.17%       | +6.6%        |
| dickens      | +0.25%       | +7.0%        |
| mozilla      | +0.02%       | +6.8%        |
| mr           | -0.30%       | +10.9%       |
| nci          | +1.28%       | +4.5%        |
| ooffice      | -0.35%       | +10.7%       |
| osdb         | +0.75%       | +9.8%        |
| reymont      | +0.65%       | +4.6%        |
| samba        | +0.70%       | +5.9%        |
| sao          | -0.01%       | +14.0%       |
| webster      | +0.30%       | +5.5%        |
| xml          | +0.92%       | +5.3%        |
| x-ray        | -0.00%       | +1.4%        |

Same tests on Calgary. For brevity, I've only included files
where compression ratio regressed or was much better.

| Calgary File | Ratio Change | Speed Change |
|--------------|--------------|--------------|
| calgary.tar  | +0.30%       | +7.1%        |
| geo          | -0.14%       | +25.0%       |
| obj1         | -0.46%       | +15.2%       |
| obj2         | -0.18%       | +6.0%        |
| pic          | +1.80%       | +9.3%        |
| trans        | -0.35%       | +5.5%        |

We gain 0.1% of compression ratio on Silesia.
We gain 0.3% of compression ratio on enwik8.
I also tested on the GitHub and hg-commands datasets without a dictionary,
and we gain a small amount of compression ratio on each, as well as speed.

I tested the negative compression levels on Silesia on my
Intel i9-9900k with gcc-8:

| Level | Ratio Change | Speed Change |
|-------|--------------|--------------|
| -1    | +0.13%       | +6.4%        |
| -2    | +4.6%        | -1.5%        |
| -3    | +7.5%        | -4.8%        |
| -4    | +8.5%        | -6.9%        |
| -5    | +9.1%        | -9.1%        |

Roughly, the negative levels now scale half as quickly. E.g. the new
level 16 is roughly equivalent to the old level 8, but a bit quicker
and smaller.  If you don't think this is the right trade off, we can
change it to multiply the step size by 2, instead of adding 1. I think
this makes sense, because it gives a bit slower ratio decay.

[igzip]: https://github.com/01org/isa-l/tree/master/igzip
---
 lib/compress/zstd_fast.c | 118 +++++++++++++++++++++++++--------------
 1 file changed, 77 insertions(+), 41 deletions(-)

diff --git a/lib/compress/zstd_fast.c b/lib/compress/zstd_fast.c
index d59e0e73ec2..ed997b441c1 100644
--- a/lib/compress/zstd_fast.c
+++ b/lib/compress/zstd_fast.c
@@ -51,10 +51,12 @@ size_t ZSTD_compressBlock_fast_generic(
     U32* const hashTable = ms->hashTable;
     U32 const hlog = cParams->hashLog;
     /* support stepSize of 0 */
-    U32 const stepSize = cParams->targetLength + !(cParams->targetLength);
+    size_t const stepSize = cParams->targetLength + !(cParams->targetLength) + 1;
     const BYTE* const base = ms->window.base;
     const BYTE* const istart = (const BYTE*)src;
-    const BYTE* ip = istart;
+    /* We check ip0 (ip + 0) and ip1 (ip + 1) each loop */
+    const BYTE* ip0 = istart;
+    const BYTE* ip1;
     const BYTE* anchor = istart;
     const U32   prefixStartIndex = ms->window.dictLimit;
     const BYTE* const prefixStart = base + prefixStartIndex;
@@ -64,62 +66,96 @@ size_t ZSTD_compressBlock_fast_generic(
     U32 offsetSaved = 0;
 
     /* init */
-    ip += (ip == prefixStart);
+    ip0 += (ip0 == prefixStart);
+    ip1 = ip0 + 1;
     {
-        U32 const maxRep = (U32)(ip - prefixStart);
+        U32 const maxRep = (U32)(ip0 - prefixStart);
         if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0;
         if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0;
     }
 
     /* Main Search Loop */
-    while (ip < ilimit) {   /* < instead of <=, because repcode check at (ip+1) */
+    while (ip1 < ilimit) {   /* < instead of <=, because check at ip0+2 */
         size_t mLength;
-        size_t const h = ZSTD_hashPtr(ip, hlog, mls);
-        U32 const current = (U32)(ip-base);
-        U32 const matchIndex = hashTable[h];
-        const BYTE* match = base + matchIndex;
-        hashTable[h] = current;   /* update hash table */
-
-        if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) {
-            mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4;
-            ip++;
-            ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH);
-        } else if ((matchIndex <= prefixStartIndex) || MEM_read32(match) != MEM_read32(ip)) {
-            assert(stepSize >= 1);
-            ip += ((ip-anchor) >> kSearchStrength) + stepSize;
-            continue;
-        } else {
+        BYTE const* ip2 = ip0 + 2;
+        size_t const h0 = ZSTD_hashPtr(ip0, hlog, mls);
+        U32 const val0 = MEM_read32(ip0);
+        size_t const h1 = ZSTD_hashPtr(ip1, hlog, mls);
+        U32 const val1 = MEM_read32(ip1);
+        U32 const current0 = (U32)(ip0-base);
+        U32 const current1 = (U32)(ip1-base);
+        U32 const matchIndex0 = hashTable[h0];
+        U32 const matchIndex1 = hashTable[h1];
+        BYTE const* repMatch = ip2-offset_1;
+        const BYTE* match0 = base + matchIndex0;
+        const BYTE* match1 = base + matchIndex1;
+        U32 offcode;
+        hashTable[h0] = current0;   /* update hash table */
+        hashTable[h1] = current1;   /* update hash table */
+
+        assert(ip0 + 1 == ip1);
+
+        if ((offset_1 > 0) & (MEM_read32(repMatch) == MEM_read32(ip2))) {
+            mLength = ip2[-1] == repMatch[-1] ? 1 : 0;
+            ip0 = ip2 - mLength;
+            match0 = repMatch - mLength;
+            offcode = 0;
+            goto _match;
+        }
+        if ((matchIndex0 > prefixStartIndex) && MEM_read32(match0) == val0) {
             /* found a regular match */
-            U32 const offset = (U32)(ip-match);
-            mLength = ZSTD_count(ip+4, match+4, iend) + 4;
-            while (((ip>anchor) & (match>prefixStart))
-                 && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
-            offset_2 = offset_1;
-            offset_1 = offset;
-            ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
+            goto _offset;
         }
-
+        if ((matchIndex1 > prefixStartIndex) && MEM_read32(match1) == val1) {
+            /* found a regular match after one literal */
+            ip0 = ip1;
+            match0 = match1;
+            goto _offset;
+        }
+        {
+            size_t const step = ((ip0-anchor) >> (kSearchStrength - 1)) + stepSize;
+            assert(step >= 2);
+            ip0 += step;
+            ip1 += step;
+            continue;
+        }
+_offset: /* Requires: ip0, match0 */
+        /* Compute the offset code */
+        offset_2 = offset_1;
+        offset_1 = (U32)(ip0-match0);
+        offcode = offset_1 + ZSTD_REP_MOVE;
+        mLength = 0;
+        /* Count the backwards match length */
+        while (((ip0>anchor) & (match0>prefixStart))
+             && (ip0[-1] == match0[-1])) { ip0--; match0--; mLength++; } /* catch up */
+
+_match: /* Requires: ip0, match0, offcode */
+        /* Count the forward length */
+        mLength += ZSTD_count(ip0+mLength+4, match0+mLength+4, iend) + 4;
+        ZSTD_storeSeq(seqStore, ip0-anchor, anchor, offcode, mLength-MINMATCH);
         /* match found */
-        ip += mLength;
-        anchor = ip;
+        ip0 += mLength;
+        anchor = ip0;
+        ip1 = ip0 + 1;
 
-        if (ip <= ilimit) {
+        if (ip0 <= ilimit) {
             /* Fill Table */
-            assert(base+current+2 > istart);  /* check base overflow */
-            hashTable[ZSTD_hashPtr(base+current+2, hlog, mls)] = current+2;  /* here because current+2 could be > iend-8 */
-            hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base);
+            assert(base+current0+2 > istart);  /* check base overflow */
+            hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2;  /* here because current+2 could be > iend-8 */
+            hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base);
 
-            while ( (ip <= ilimit)
+            while ( (ip0 <= ilimit)
                  && ( (offset_2>0)
-                    & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) {
+                    & (MEM_read32(ip0) == MEM_read32(ip0 - offset_2)) )) {
                 /* store sequence */
-                size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4;
+                size_t const rLength = ZSTD_count(ip0+4, ip0+4-offset_2, iend) + 4;
                 U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff;  /* swap offset_2 <=> offset_1 */
-                hashTable[ZSTD_hashPtr(ip, hlog, mls)] = (U32)(ip-base);
+                hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base);
+                ip0 += rLength;
+                ip1 = ip0 + 1;
                 ZSTD_storeSeq(seqStore, 0, anchor, 0, rLength-MINMATCH);
-                ip += rLength;
-                anchor = ip;
-                continue;   /* faster when present ... (?) */
+                anchor = ip0;
+                continue;   /* faster when present (confirmed on gcc-8) ... (?) */
             }
         }
     }

From 00679da22bd6fe89778bc27ff91f4b40c11e0833 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Tue, 2 Apr 2019 19:20:52 -0700
Subject: [PATCH 125/178] [libzstd] Setting ZSTD_d_maxWindowLog to 0 means
 default

---
 lib/decompress/zstd_decompress.c | 1 +
 lib/zstd.h                       | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 142923fe4bc..aa7f6f58042 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1372,6 +1372,7 @@ size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value
     RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong);
     switch(dParam) {
         case ZSTD_d_windowLogMax:
+            if (value == 0) value = ZSTD_WINDOWLOG_LIMIT_DEFAULT;
             CHECK_DBOUNDS(ZSTD_d_windowLogMax, value);
             dctx->maxWindowSize = ((size_t)1) << value;
             return 0;
diff --git a/lib/zstd.h b/lib/zstd.h
index ea2b9a5e490..acc535cff54 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -852,7 +852,8 @@ typedef enum {
                               * the streaming API will refuse to allocate memory buffer
                               * in order to protect the host from unreasonable memory requirements.
                               * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
-                              * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) */
+                              * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT).
+                              * Special: value 0 means "use default maximum windowLog". */
 
     /* note : additional experimental parameters are also available
      * within the experimental section of the API.

From cdc8ae2e9bec9ad9b3f6bffc2b8d283180814853 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Mon, 1 Apr 2019 18:05:49 -0700
Subject: [PATCH 126/178] [examples] Update streaming_memory_usage.c

Update to use the new streaming API. Making progress on Issue #1548.

Tested that the checks don't fail.
Tested with window log 9-32. The lowest and highest fail as expected.
---
 examples/streaming_memory_usage.c | 130 +++++++++++++-----------------
 1 file changed, 58 insertions(+), 72 deletions(-)

diff --git a/examples/streaming_memory_usage.c b/examples/streaming_memory_usage.c
index 5e7e13e824e..c31d9f9f788 100644
--- a/examples/streaming_memory_usage.c
+++ b/examples/streaming_memory_usage.c
@@ -19,6 +19,7 @@
 #include    /* printf */
 #define ZSTD_STATIC_LINKING_ONLY
 #include "zstd.h"
+#include "utils.h"
 
 
 /*===   functions   ===*/
@@ -61,90 +62,75 @@ int main(int argc, char const *argv[]) {
         char const dataToCompress[INPUT_SIZE] = "abcde";
         char compressedData[COMPRESSED_SIZE];
         char decompressedData[INPUT_SIZE];
-        ZSTD_CStream* const cstream = ZSTD_createCStream();
-        if (cstream==NULL) {
-            printf("Level %i : ZSTD_CStream Memory allocation failure \n", compressionLevel);
-            return 1;
-        }
-
-        /* forces compressor to use maximum memory size for given compression level,
-         * by not providing any information on input size */
-        ZSTD_parameters params = ZSTD_getParams(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0);
-        if (wLog) { /* special mode : specific wLog */
-            printf("Using custom compression parameter : level 1 + wLog=%u \n", wLog);
-            params = ZSTD_getParams(1 /*compressionLevel*/,
-                                    1 << wLog /*estimatedSrcSize*/,
-                                    0 /*no dictionary*/);
-            size_t const error = ZSTD_initCStream_advanced(cstream, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN);
-            if (ZSTD_isError(error)) {
-                printf("ZSTD_initCStream_advanced error : %s \n", ZSTD_getErrorName(error));
-                return 1;
-            }
-        } else {
-            size_t const error = ZSTD_initCStream(cstream, compressionLevel);
-            if (ZSTD_isError(error)) {
-                printf("ZSTD_initCStream error : %s \n", ZSTD_getErrorName(error));
-                return 1;
-            }
-        }
-
+        /* the ZSTD_CCtx_params structure is a way to save parameters and use
+         * them across multiple contexts. We use them here so we can call the
+         * function ZSTD_estimateCStreamSize_usingCCtxParams().
+         */
+        ZSTD_CCtx_params* const cctxParams = ZSTD_createCCtxParams();
+        CHECK(cctxParams != NULL, "ZSTD_createCCtxParams() failed!");
+
+        /* Set the compression level. */
+        CHECK_ZSTD( ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_compressionLevel, compressionLevel) );
+        /* Set the window log.
+         * The value 0 means use the default window log, which is equivalent to
+         * not setting it.
+         */
+        CHECK_ZSTD( ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, wLog) );
+
+        /* Force the compressor to allocate the maximum memory size for a given
+         * level by not providing the pledged source size, or calling
+         * ZSTD_compressStream2() with ZSTD_e_end.
+         */
+        ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+        CHECK(cctx != NULL, "ZSTD_createCCtx() failed!");
+        CHECK_ZSTD( ZSTD_CCtx_setParametersUsingCCtxParams(cctx, cctxParams) );
         size_t compressedSize;
-        {   ZSTD_inBuffer inBuff = { dataToCompress, sizeof(dataToCompress), 0 };
+        {
+            ZSTD_inBuffer inBuff = { dataToCompress, sizeof(dataToCompress), 0 };
             ZSTD_outBuffer outBuff = { compressedData, sizeof(compressedData), 0 };
-            size_t const cError = ZSTD_compressStream(cstream, &outBuff, &inBuff);
-            if (ZSTD_isError(cError)) {
-                printf("ZSTD_compressStream error : %s \n", ZSTD_getErrorName(cError));
-                return 1;
-            }
-            size_t const fError = ZSTD_endStream(cstream, &outBuff);
-            if (ZSTD_isError(fError)) {
-                printf("ZSTD_endStream error : %s \n", ZSTD_getErrorName(fError));
-                return 1;
-            }
+            CHECK_ZSTD( ZSTD_compressStream(cctx, &outBuff, &inBuff) );
+            size_t const remaining = ZSTD_endStream(cctx, &outBuff);
+            CHECK_ZSTD(remaining);
+            CHECK(remaining == 0, "Frame not flushed!");
             compressedSize = outBuff.pos;
         }
 
-        ZSTD_DStream* dstream = ZSTD_createDStream();
-        if (dstream==NULL) {
-            printf("Level %i : ZSTD_DStream Memory allocation failure \n", compressionLevel);
-            return 1;
-        }
-        {   size_t const error = ZSTD_initDStream(dstream);
-            if (ZSTD_isError(error)) {
-                printf("ZSTD_initDStream error : %s \n", ZSTD_getErrorName(error));
-                return 1;
-            }
-        }
-        /* forces decompressor to use maximum memory size, as decompressed size is not known */
+        ZSTD_DCtx* const dctx = ZSTD_createDCtx();
+        CHECK(dctx != NULL, "ZSTD_createDCtx() failed!");
+        /* Set the maximum allowed window log.
+         * The value 0 means use the default window log, which is equivalent to
+         * not setting it.
+         */
+        CHECK_ZSTD( ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, wLog) );
+        /* forces decompressor to use maximum memory size, since the
+         * decompressed size is not stored in the frame header.
+         */
         {   ZSTD_inBuffer inBuff = { compressedData, compressedSize, 0 };
             ZSTD_outBuffer outBuff = { decompressedData, sizeof(decompressedData), 0 };
-            size_t const dResult = ZSTD_decompressStream(dstream, &outBuff, &inBuff);
-            if (ZSTD_isError(dResult)) {
-                printf("ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(dResult));
-                return 1;
-            }
-            if (dResult != 0) {
-                printf("ZSTD_decompressStream error : unfinished decompression \n");
-                return 1;
-            }
-            if (outBuff.pos != sizeof(dataToCompress)) {
-                printf("ZSTD_decompressStream error : incorrect decompression \n");
-                return 1;
-            }
+            size_t const remaining = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
+            CHECK_ZSTD(remaining);
+            CHECK(remaining == 0, "Frame not complete!");
+            CHECK(outBuff.pos == sizeof(dataToCompress), "Bad decompression!");
         }
 
-        size_t const cstreamSize = ZSTD_sizeof_CStream(cstream);
-        size_t const cstreamEstimatedSize = wLog ?
-                ZSTD_estimateCStreamSize_usingCParams(params.cParams) :
-                ZSTD_estimateCStreamSize(compressionLevel);
-        size_t const dstreamSize = ZSTD_sizeof_DStream(dstream);
+        size_t const cstreamSize = ZSTD_sizeof_CStream(cctx);
+        size_t const cstreamEstimatedSize = ZSTD_estimateCStreamSize_usingCCtxParams(cctxParams);
+        size_t const dstreamSize = ZSTD_sizeof_DStream(dctx);
+        size_t const dstreamEstimatedSize = ZSTD_estimateDStreamSize_fromFrame(compressedData, compressedSize);
+
+        CHECK(cstreamSize <= cstreamEstimatedSize, "Compression mem (%u) > estimated (%u)",
+                (unsigned)cstreamSize, (unsigned)cstreamEstimatedSize);
+        CHECK(dstreamSize <= dstreamEstimatedSize, "Decompression mem (%u) > estimated (%u)",
+                (unsigned)dstreamSize, (unsigned)dstreamEstimatedSize);
 
-        printf("Level %2i : Compression Mem = %5u KB (estimated : %5u KB) ; Decompression Mem = %4u KB \n",
+        printf("Level %2i : Compression Mem = %5u KB (estimated : %5u KB) ; Decompression Mem = %4u KB (estimated : %5u KB)\n",
                 compressionLevel,
-                (unsigned)(cstreamSize>>10), (unsigned)(cstreamEstimatedSize>>10), (unsigned)(dstreamSize>>10));
+                (unsigned)(cstreamSize>>10), (unsigned)(cstreamEstimatedSize>>10),
+                (unsigned)(dstreamSize>>10), (unsigned)(dstreamEstimatedSize>>10));
 
-        ZSTD_freeDStream(dstream);
-        ZSTD_freeCStream(cstream);
+        ZSTD_freeDCtx(dctx);
+        ZSTD_freeCCtx(cctx);
+        ZSTD_freeCCtxParams(cctxParams);
         if (wLog) break;  /* single test */
     }
     return 0;

From 0827edeace3fbc5de0d944688a5cc424e6011dd1 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Mon, 1 Apr 2019 18:43:32 -0700
Subject: [PATCH 127/178] [libzstd] Bump the library version to 1.4.0

Bumps the library version to 1.4.0 in preparation to stabilize the
advanced API.
---
 lib/zstd.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/zstd.h b/lib/zstd.h
index acc535cff54..667d8142dc7 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -70,8 +70,8 @@ extern "C" {
 
 /*------   Version   ------*/
 #define ZSTD_VERSION_MAJOR    1
-#define ZSTD_VERSION_MINOR    3
-#define ZSTD_VERSION_RELEASE  8
+#define ZSTD_VERSION_MINOR    4
+#define ZSTD_VERSION_RELEASE  0
 
 #define ZSTD_VERSION_NUMBER  (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
 ZSTDLIB_API unsigned ZSTD_versionNumber(void);   /**< to check runtime library version */

From d7d89513d6a216cfa920208ce82f9128616bab7c Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Thu, 28 Mar 2019 19:04:32 -0600
Subject: [PATCH 128/178] Stabilize advance API

This commit moves the candidate advanced API to the stable section.
It makes some minor whitespace changes, but it doesn't change any
of the wording of the documentation.

I'll put up a separate PR that tweaks some of the documentation
once this lands, so that it is easier to review.

NOTE: Even though these functions are now in stable, they aren't
stable until the next release (in under 1 month). It is possible
that they change until then.
---
 lib/zstd.h | 486 +++++++++++++++++++++++++----------------------------
 1 file changed, 228 insertions(+), 258 deletions(-)

diff --git a/lib/zstd.h b/lib/zstd.h
index 667d8142dc7..91956a7395a 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -89,6 +89,21 @@ ZSTDLIB_API const char* ZSTD_versionString(void);   /* requires v1.3.0+ */
 #  define ZSTD_CLEVEL_DEFAULT 3
 #endif
 
+/***************************************
+*  Constants
+***************************************/
+
+/* All magic numbers are supposed read/written to/from files/memory using little-endian convention */
+#define ZSTD_MAGICNUMBER            0xFD2FB528    /* valid since v0.8.0 */
+#define ZSTD_MAGIC_DICTIONARY       0xEC30A437    /* valid since v0.7.0 */
+#define ZSTD_MAGIC_SKIPPABLE_START  0x184D2A50    /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */
+#define ZSTD_MAGIC_SKIPPABLE_MASK   0xFFFFFFF0
+
+#define ZSTD_BLOCKSIZELOG_MAX  17
+#define ZSTD_BLOCKSIZE_MAX     (1<= first frame size
+ * @return : the compressed size of the first frame starting at `src`,
+ *           suitable to pass as `srcSize` to `ZSTD_decompress` or similar,
+ *        or an error code if input is invalid */
+ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
+
 
 /*======  Helper functions  ======*/
 #define ZSTD_COMPRESSBOUND(srcSize)   ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
 ZSTDLIB_API size_t      ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
 ZSTDLIB_API unsigned    ZSTD_isError(size_t code);          /*!< tells if a `size_t` function result is an error code */
 ZSTDLIB_API const char* ZSTD_getErrorName(size_t code);     /*!< provides readable string from an error code */
+ZSTDLIB_API int         ZSTD_minCLevel(void);               /*!< minimum negative compression level allowed */
 ZSTDLIB_API int         ZSTD_maxCLevel(void);               /*!< maximum compression level available */
 
 
@@ -273,216 +297,6 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
                                         const ZSTD_DDict* ddict);
 
 
-/****************************
-*  Streaming
-****************************/
-
-typedef struct ZSTD_inBuffer_s {
-  const void* src;    /**< start of input buffer */
-  size_t size;        /**< size of input buffer */
-  size_t pos;         /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */
-} ZSTD_inBuffer;
-
-typedef struct ZSTD_outBuffer_s {
-  void*  dst;         /**< start of output buffer */
-  size_t size;        /**< size of output buffer */
-  size_t pos;         /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */
-} ZSTD_outBuffer;
-
-
-
-/*-***********************************************************************
-*  Streaming compression - HowTo
-*
-*  A ZSTD_CStream object is required to track streaming operation.
-*  Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
-*  ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
-*  It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory.
-*
-*  For parallel execution, use one separate ZSTD_CStream per thread.
-*
-*  note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing.
-*
-*  Parameters are sticky : when starting a new compression on the same context,
-*  it will re-use the same sticky parameters as previous compression session.
-*  When in doubt, it's recommended to fully initialize the context before usage.
-*  Use ZSTD_initCStream() to set the parameter to a selected compression level.
-*  Use advanced API (ZSTD_CCtx_setParameter(), etc.) to set more specific parameters.
-*
-*  Use ZSTD_compressStream() as many times as necessary to consume input stream.
-*  The function will automatically update both `pos` fields within `input` and `output`.
-*  Note that the function may not consume the entire input,
-*  for example, because the output buffer is already full,
-*  in which case `input.pos < input.size`.
-*  The caller must check if input has been entirely consumed.
-*  If not, the caller must make some room to receive more compressed data,
-*  and then present again remaining input data.
-* @return : a size hint, preferred nb of bytes to use as input for next function call
-*           or an error code, which can be tested using ZSTD_isError().
-*           Note 1 : it's just a hint, to help latency a little, any value will work fine.
-*           Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize()
-*
-*  At any moment, it's possible to flush whatever data might remain stuck within internal buffer,
-*  using ZSTD_flushStream(). `output->pos` will be updated.
-*  Note that, if `output->size` is too small, a single invocation of ZSTD_flushStream() might not be enough (return code > 0).
-*  In which case, make some room to receive more compressed data, and call again ZSTD_flushStream().
-*  @return : 0 if internal buffers are entirely flushed,
-*            >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
-*            or an error code, which can be tested using ZSTD_isError().
-*
-*  ZSTD_endStream() instructs to finish a frame.
-*  It will perform a flush and write frame epilogue.
-*  The epilogue is required for decoders to consider a frame completed.
-*  flush() operation is the same, and follows same rules as ZSTD_flushStream().
-*  @return : 0 if frame fully completed and fully flushed,
-*            >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
-*            or an error code, which can be tested using ZSTD_isError().
-*
-* *******************************************************************/
-
-typedef ZSTD_CCtx ZSTD_CStream;  /**< CCtx and CStream are now effectively same object (>= v1.3.0) */
-                                 /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */
-/*===== ZSTD_CStream management functions =====*/
-ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
-ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
-
-/*===== Streaming compression functions =====*/
-/**
- * Equivalent to:
- *
- *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
- *     ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
- *     ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
- */
-ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
-/**
- * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).
- * NOTE: The return value is different. ZSTD_compressStream() returns a hint for
- * the next read size (if non-zero and not an error). ZSTD_compressStream2()
- * returns the number of bytes left to flush (if non-zero and not an error).
- */
-ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
-/** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */
-ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
-/** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */
-ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
-
-ZSTDLIB_API size_t ZSTD_CStreamInSize(void);    /**< recommended size for input buffer */
-ZSTDLIB_API size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */
-
-
-
-/*-***************************************************************************
-*  Streaming decompression - HowTo
-*
-*  A ZSTD_DStream object is required to track streaming operations.
-*  Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
-*  ZSTD_DStream objects can be re-used multiple times.
-*
-*  Use ZSTD_initDStream() to start a new decompression operation.
-* @return : recommended first input size
-*  Alternatively, use advanced API to set specific properties.
-*
-*  Use ZSTD_decompressStream() repetitively to consume your input.
-*  The function will update both `pos` fields.
-*  If `input.pos < input.size`, some input has not been consumed.
-*  It's up to the caller to present again remaining data.
-*  The function tries to flush all data decoded immediately, respecting output buffer size.
-*  If `output.pos < output.size`, decoder has flushed everything it could.
-*  But if `output.pos == output.size`, there might be some data left within internal buffers.,
-*  In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer.
-*  Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.
-* @return : 0 when a frame is completely decoded and fully flushed,
-*        or an error code, which can be tested using ZSTD_isError(),
-*        or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
-*                                the return value is a suggested next input size (just a hint for better latency)
-*                                that will never request more than the remaining frame size.
-* *******************************************************************************/
-
-typedef ZSTD_DCtx ZSTD_DStream;  /**< DCtx and DStream are now effectively same object (>= v1.3.0) */
-                                 /* For compatibility with versions <= v1.2.0, prefer differentiating them. */
-/*===== ZSTD_DStream management functions =====*/
-ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
-ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
-
-/*===== Streaming decompression functions =====*/
-ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
-ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
-
-ZSTDLIB_API size_t ZSTD_DStreamInSize(void);    /*!< recommended size for input buffer */
-ZSTDLIB_API size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */
-
-#endif  /* ZSTD_H_235446 */
-
-
-
-
-/****************************************************************************************
- *   ADVANCED AND EXPERIMENTAL FUNCTIONS
- ****************************************************************************************
- * The definitions in the following section are considered experimental.
- * They are provided for advanced scenarios.
- * They should never be used with a dynamic library, as prototypes may change in the future.
- * Use them only in association with static linking.
- * ***************************************************************************************/
-
-#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
-#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
-
-
-/****************************************************************************************
- *   Candidate API for promotion to stable status
- ****************************************************************************************
- * The following symbols and constants form the "staging area" :
- * they are considered to join "stable API" by v1.4.0.
- * The proposal is written so that it can be made stable "as is",
- * though it's still possible to suggest improvements.
- * Staging is in fact last chance for changes,
- * the API is locked once reaching "stable" status.
- * ***************************************************************************************/
-
-
-/* ===  Constants   === */
-
-/* all magic numbers are supposed read/written to/from files/memory using little-endian convention */
-#define ZSTD_MAGICNUMBER            0xFD2FB528    /* valid since v0.8.0 */
-#define ZSTD_MAGIC_DICTIONARY       0xEC30A437    /* valid since v0.7.0 */
-#define ZSTD_MAGIC_SKIPPABLE_START  0x184D2A50    /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */
-#define ZSTD_MAGIC_SKIPPABLE_MASK   0xFFFFFFF0
-
-#define ZSTD_BLOCKSIZELOG_MAX  17
-#define ZSTD_BLOCKSIZE_MAX     (1<= first frame size
- * @return : the compressed size of the first frame starting at `src`,
- *           suitable to pass as `srcSize` to `ZSTD_decompress` or similar,
- *        or an error code if input is invalid */
-ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
-
-
-/* ===   Memory management   === */
-
-/*! ZSTD_sizeof_*() :
- *  These functions give the _current_ memory usage of selected object.
- *  Note that object memory usage can evolve (increase or decrease) over time. */
-ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
-ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
-ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
-ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
-ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
-ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
-
-
 /***************************************
 *  Advanced compression API
 ***************************************/
@@ -655,7 +469,6 @@ typedef enum {
      ZSTD_c_experimentalParam5=1002,
 } ZSTD_cParameter;
 
-
 typedef struct {
     size_t error;
     int lowerBound;
@@ -755,7 +568,6 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
 ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
                                  const void* prefix, size_t prefixSize);
 
-
 typedef enum {
     ZSTD_reset_session_only = 1,
     ZSTD_reset_parameters = 2,
@@ -778,8 +590,6 @@ typedef enum {
  */
 ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset);
 
-
-
 /*! ZSTD_compress2() :
  *  Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API.
  *  ZSTD_compress2() always starts a new frame.
@@ -794,48 +604,10 @@ ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx,
                                    void* dst, size_t dstCapacity,
                              const void* src, size_t srcSize);
 
-typedef enum {
-    ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */
-    ZSTD_e_flush=1,    /* flush any data provided so far,
-                        * it creates (at least) one new block, that can be decoded immediately on reception;
-                        * frame will continue: any future data can still reference previously compressed data, improving compression. */
-    ZSTD_e_end=2       /* flush any remaining data _and_ close current frame.
-                        * note that frame is only closed after compressed data is fully flushed (return value == 0).
-                        * After that point, any additional data starts a new frame.
-                        * note : each frame is independent (does not reference any content from previous frame). */
-} ZSTD_EndDirective;
 
-/*! ZSTD_compressStream2() :
- *  Behaves about the same as ZSTD_compressStream, with additional control on end directive.
- *  - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
- *  - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
- *  - outpot->pos must be <= dstCapacity, input->pos must be <= srcSize
- *  - outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
- *  - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
- *  - When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads, flush whatever is available,
- *                                                  and then immediately returns, just indicating that there is some data remaining to be flushed.
- *                                                  The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
- *  - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking.
- *  - @return provides a minimum amount of data remaining to be flushed from internal buffers
- *            or an error code, which can be tested using ZSTD_isError().
- *            if @return != 0, flush is not fully completed, there is still some data left within internal buffers.
- *            This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers.
- *            For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed.
- *  - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0),
- *            only ZSTD_e_end or ZSTD_e_flush operations are allowed.
- *            Before starting a new compression job, or changing compression parameters,
- *            it is required to fully flush internal buffers.
- */
-ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
-                                         ZSTD_outBuffer* output,
-                                         ZSTD_inBuffer* input,
-                                         ZSTD_EndDirective endOp);
-
-
-
-/* ============================== */
-/*   Advanced decompression API   */
-/* ============================== */
+/***************************************
+*  Advanced decompression API
+***************************************/
 
 /* The advanced API pushes parameters one by one into an existing DCtx context.
  * Parameters are sticky, and remain valid for all following frames
@@ -845,7 +617,6 @@ ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
  *        Therefore, no new decompression function is necessary.
  */
 
-
 typedef enum {
 
     ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which
@@ -866,7 +637,6 @@ typedef enum {
 
 } ZSTD_dParameter;
 
-
 /*! ZSTD_dParam_getBounds() :
  *  All parameters must belong to an interval with lower and upper bounds,
  *  otherwise they will either trigger an error or be automatically clamped.
@@ -885,7 +655,6 @@ ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam);
  */
 ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value);
 
-
 /*! ZSTD_DCtx_loadDictionary() :
  *  Create an internal DDict from dict buffer,
  *  to be used to decompress next frames.
@@ -942,6 +711,207 @@ ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx,
 ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset);
 
 
+/****************************
+*  Streaming
+****************************/
+
+typedef struct ZSTD_inBuffer_s {
+  const void* src;    /**< start of input buffer */
+  size_t size;        /**< size of input buffer */
+  size_t pos;         /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */
+} ZSTD_inBuffer;
+
+typedef struct ZSTD_outBuffer_s {
+  void*  dst;         /**< start of output buffer */
+  size_t size;        /**< size of output buffer */
+  size_t pos;         /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */
+} ZSTD_outBuffer;
+
+
+
+/*-***********************************************************************
+*  Streaming compression - HowTo
+*
+*  A ZSTD_CStream object is required to track streaming operation.
+*  Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
+*  ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
+*  It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory.
+*
+*  For parallel execution, use one separate ZSTD_CStream per thread.
+*
+*  note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing.
+*
+*  Parameters are sticky : when starting a new compression on the same context,
+*  it will re-use the same sticky parameters as previous compression session.
+*  When in doubt, it's recommended to fully initialize the context before usage.
+*  Use ZSTD_initCStream() to set the parameter to a selected compression level.
+*  Use advanced API (ZSTD_CCtx_setParameter(), etc.) to set more specific parameters.
+*
+*  Use ZSTD_compressStream() as many times as necessary to consume input stream.
+*  The function will automatically update both `pos` fields within `input` and `output`.
+*  Note that the function may not consume the entire input,
+*  for example, because the output buffer is already full,
+*  in which case `input.pos < input.size`.
+*  The caller must check if input has been entirely consumed.
+*  If not, the caller must make some room to receive more compressed data,
+*  and then present again remaining input data.
+* @return : a size hint, preferred nb of bytes to use as input for next function call
+*           or an error code, which can be tested using ZSTD_isError().
+*           Note 1 : it's just a hint, to help latency a little, any value will work fine.
+*           Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize()
+*
+*  At any moment, it's possible to flush whatever data might remain stuck within internal buffer,
+*  using ZSTD_flushStream(). `output->pos` will be updated.
+*  Note that, if `output->size` is too small, a single invocation of ZSTD_flushStream() might not be enough (return code > 0).
+*  In which case, make some room to receive more compressed data, and call again ZSTD_flushStream().
+*  @return : 0 if internal buffers are entirely flushed,
+*            >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
+*            or an error code, which can be tested using ZSTD_isError().
+*
+*  ZSTD_endStream() instructs to finish a frame.
+*  It will perform a flush and write frame epilogue.
+*  The epilogue is required for decoders to consider a frame completed.
+*  flush() operation is the same, and follows same rules as ZSTD_flushStream().
+*  @return : 0 if frame fully completed and fully flushed,
+*            >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
+*            or an error code, which can be tested using ZSTD_isError().
+*
+* *******************************************************************/
+
+typedef ZSTD_CCtx ZSTD_CStream;  /**< CCtx and CStream are now effectively same object (>= v1.3.0) */
+                                 /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */
+/*===== ZSTD_CStream management functions =====*/
+ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
+ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
+
+/*===== Streaming compression functions =====*/
+typedef enum {
+    ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */
+    ZSTD_e_flush=1,    /* flush any data provided so far,
+                        * it creates (at least) one new block, that can be decoded immediately on reception;
+                        * frame will continue: any future data can still reference previously compressed data, improving compression. */
+    ZSTD_e_end=2       /* flush any remaining data _and_ close current frame.
+                        * note that frame is only closed after compressed data is fully flushed (return value == 0).
+                        * After that point, any additional data starts a new frame.
+                        * note : each frame is independent (does not reference any content from previous frame). */
+} ZSTD_EndDirective;
+
+/*! ZSTD_compressStream2() :
+ *  Behaves about the same as ZSTD_compressStream, with additional control on end directive.
+ *  - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
+ *  - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
+ *  - outpot->pos must be <= dstCapacity, input->pos must be <= srcSize
+ *  - outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
+ *  - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
+ *  - When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads, flush whatever is available,
+ *                                                  and then immediately returns, just indicating that there is some data remaining to be flushed.
+ *                                                  The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
+ *  - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking.
+ *  - @return provides a minimum amount of data remaining to be flushed from internal buffers
+ *            or an error code, which can be tested using ZSTD_isError().
+ *            if @return != 0, flush is not fully completed, there is still some data left within internal buffers.
+ *            This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers.
+ *            For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed.
+ *  - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0),
+ *            only ZSTD_e_end or ZSTD_e_flush operations are allowed.
+ *            Before starting a new compression job, or changing compression parameters,
+ *            it is required to fully flush internal buffers.
+ */
+ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
+                                         ZSTD_outBuffer* output,
+                                         ZSTD_inBuffer* input,
+                                         ZSTD_EndDirective endOp);
+
+/**
+ * Equivalent to:
+ *
+ *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ *     ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
+ *     ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
+ */
+ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
+/**
+ * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).
+ * NOTE: The return value is different. ZSTD_compressStream() returns a hint for
+ * the next read size (if non-zero and not an error). ZSTD_compressStream2()
+ * returns the number of bytes left to flush (if non-zero and not an error).
+ */
+ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+/** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */
+ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+/** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */
+ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+
+ZSTDLIB_API size_t ZSTD_CStreamInSize(void);    /**< recommended size for input buffer */
+ZSTDLIB_API size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */
+
+
+/*-***************************************************************************
+*  Streaming decompression - HowTo
+*
+*  A ZSTD_DStream object is required to track streaming operations.
+*  Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
+*  ZSTD_DStream objects can be re-used multiple times.
+*
+*  Use ZSTD_initDStream() to start a new decompression operation.
+* @return : recommended first input size
+*  Alternatively, use advanced API to set specific properties.
+*
+*  Use ZSTD_decompressStream() repetitively to consume your input.
+*  The function will update both `pos` fields.
+*  If `input.pos < input.size`, some input has not been consumed.
+*  It's up to the caller to present again remaining data.
+*  The function tries to flush all data decoded immediately, respecting output buffer size.
+*  If `output.pos < output.size`, decoder has flushed everything it could.
+*  But if `output.pos == output.size`, there might be some data left within internal buffers.,
+*  In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer.
+*  Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.
+* @return : 0 when a frame is completely decoded and fully flushed,
+*        or an error code, which can be tested using ZSTD_isError(),
+*        or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
+*                                the return value is a suggested next input size (just a hint for better latency)
+*                                that will never request more than the remaining frame size.
+* *******************************************************************************/
+
+typedef ZSTD_DCtx ZSTD_DStream;  /**< DCtx and DStream are now effectively same object (>= v1.3.0) */
+                                 /* For compatibility with versions <= v1.2.0, prefer differentiating them. */
+/*===== ZSTD_DStream management functions =====*/
+ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
+ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
+
+/*===== Streaming decompression functions =====*/
+ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
+ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+
+ZSTDLIB_API size_t ZSTD_DStreamInSize(void);    /*!< recommended size for input buffer */
+ZSTDLIB_API size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */
+
+/* ===   Memory management   === */
+
+/*! ZSTD_sizeof_*() :
+ *  These functions give the _current_ memory usage of selected object.
+ *  Note that object memory usage can evolve (increase or decrease) over time. */
+ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
+ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
+ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
+ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
+ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
+ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
+
+#endif  /* ZSTD_H_235446 */
+
+
+/****************************************************************************************
+ *   ADVANCED AND EXPERIMENTAL FUNCTIONS
+ ****************************************************************************************
+ * The definitions in the following section are considered experimental.
+ * They are provided for advanced scenarios.
+ * They should never be used with a dynamic library, as prototypes may change in the future.
+ * Use them only in association with static linking.
+ * ***************************************************************************************/
+
+#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
+#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
 
 /****************************************************************************************
  *   experimental API (static linking only)

From cf7d601bf583e56baff75f624753a586a9498250 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Wed, 3 Apr 2019 19:01:51 -0700
Subject: [PATCH 129/178] Move the dictionary API and mark the legacy API

* Move the dictionary API below the streaming API
* Mark the legacy streaming API as redundant
---
 lib/zstd.h | 382 ++++++++++++++++++++++++++++-------------------------
 1 file changed, 200 insertions(+), 182 deletions(-)

diff --git a/lib/zstd.h b/lib/zstd.h
index 91956a7395a..4e6791560dc 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -219,84 +219,6 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx,
                                  const void* src, size_t srcSize);
 
 
-/**************************
-*  Simple dictionary API
-***************************/
-/*! ZSTD_compress_usingDict() :
- *  Compression at an explicit compression level using a Dictionary.
- *  A dictionary can be any arbitrary data segment (also called a prefix),
- *  or a buffer with specified information (see dictBuilder/zdict.h).
- *  Note : This function loads the dictionary, resulting in significant startup delay.
- *         It's intended for a dictionary used only once.
- *  Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */
-ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
-                                           void* dst, size_t dstCapacity,
-                                     const void* src, size_t srcSize,
-                                     const void* dict,size_t dictSize,
-                                           int compressionLevel);
-
-/*! ZSTD_decompress_usingDict() :
- *  Decompression using a known Dictionary.
- *  Dictionary must be identical to the one used during compression.
- *  Note : This function loads the dictionary, resulting in significant startup delay.
- *         It's intended for a dictionary used only once.
- *  Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
-ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
-                                             void* dst, size_t dstCapacity,
-                                       const void* src, size_t srcSize,
-                                       const void* dict,size_t dictSize);
-
-
-/***********************************
- *  Bulk processing dictionary API
- **********************************/
-typedef struct ZSTD_CDict_s ZSTD_CDict;
-
-/*! ZSTD_createCDict() :
- *  When compressing multiple messages / blocks using the same dictionary, it's recommended to load it only once.
- *  ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup cost.
- *  ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
- * `dictBuffer` can be released after ZSTD_CDict creation, because its content is copied within CDict.
- *  Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate `dictBuffer` content.
- *  Note : A ZSTD_CDict can be created from an empty dictBuffer, but it is inefficient when used to compress small data. */
-ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
-                                         int compressionLevel);
-
-/*! ZSTD_freeCDict() :
- *  Function frees memory allocated by ZSTD_createCDict(). */
-ZSTDLIB_API size_t      ZSTD_freeCDict(ZSTD_CDict* CDict);
-
-/*! ZSTD_compress_usingCDict() :
- *  Compression using a digested Dictionary.
- *  Recommended when same dictionary is used multiple times.
- *  Note : compression level is _decided at dictionary creation time_,
- *     and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */
-ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
-                                            void* dst, size_t dstCapacity,
-                                      const void* src, size_t srcSize,
-                                      const ZSTD_CDict* cdict);
-
-
-typedef struct ZSTD_DDict_s ZSTD_DDict;
-
-/*! ZSTD_createDDict() :
- *  Create a digested dictionary, ready to start decompression operation without startup delay.
- *  dictBuffer can be released after DDict creation, as its content is copied inside DDict. */
-ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
-
-/*! ZSTD_freeDDict() :
- *  Function frees memory allocated with ZSTD_createDDict() */
-ZSTDLIB_API size_t      ZSTD_freeDDict(ZSTD_DDict* ddict);
-
-/*! ZSTD_decompress_usingDDict() :
- *  Decompression using a digested Dictionary.
- *  Recommended when same dictionary is used multiple times. */
-ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
-                                              void* dst, size_t dstCapacity,
-                                        const void* src, size_t srcSize,
-                                        const ZSTD_DDict* ddict);
-
-
 /***************************************
 *  Advanced compression API
 ***************************************/
@@ -514,60 +436,6 @@ ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param
  */
 ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
 
-/*! ZSTD_CCtx_loadDictionary() :
- *  Create an internal CDict from `dict` buffer.
- *  Decompression will have to use same dictionary.
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- *  Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary,
- *           meaning "return to no-dictionary mode".
- *  Note 1 : Dictionary is sticky, it will be used for all future compressed frames.
- *           To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters).
- *  Note 2 : Loading a dictionary involves building tables.
- *           It's also a CPU consuming operation, with non-negligible impact on latency.
- *           Tables are dependent on compression parameters, and for this reason,
- *           compression parameters can no longer be changed after loading a dictionary.
- *  Note 3 :`dict` content will be copied internally.
- *           Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead.
- *           In such a case, dictionary buffer must outlive its users.
- *  Note 4 : Use ZSTD_CCtx_loadDictionary_advanced()
- *           to precisely select how dictionary content must be interpreted. */
-ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
-
-/*! ZSTD_CCtx_refCDict() :
- *  Reference a prepared dictionary, to be used for all next compressed frames.
- *  Note that compression parameters are enforced from within CDict,
- *  and supersede any compression parameter previously set within CCtx.
- *  The parameters ignored are labled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
- *  The ignored parameters will be used again if the CCtx is returned to no-dictionary mode.
- *  The dictionary will remain valid for future compressed frames using same CCtx.
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- *  Special : Referencing a NULL CDict means "return to no-dictionary mode".
- *  Note 1 : Currently, only one dictionary can be managed.
- *           Referencing a new dictionary effectively "discards" any previous one.
- *  Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */
-ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
-
-/*! ZSTD_CCtx_refPrefix() :
- *  Reference a prefix (single-usage dictionary) for next compressed frame.
- *  A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end).
- *  Decompression will need same prefix to properly regenerate data.
- *  Compressing with a prefix is similar in outcome as performing a diff and compressing it,
- *  but performs much faster, especially during decompression (compression speed is tunable with compression level).
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- *  Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary
- *  Note 1 : Prefix buffer is referenced. It **must** outlive compression.
- *           Its content must remain unmodified during compression.
- *  Note 2 : If the intention is to diff some large src data blob with some prior version of itself,
- *           ensure that the window size is large enough to contain the entire source.
- *           See ZSTD_c_windowLog.
- *  Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
- *           It's a CPU consuming operation, with non-negligible impact on latency.
- *           If there is a need to use the same prefix multiple times, consider loadDictionary instead.
- *  Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent).
- *           Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */
-ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
-                                 const void* prefix, size_t prefixSize);
-
 typedef enum {
     ZSTD_reset_session_only = 1,
     ZSTD_reset_parameters = 2,
@@ -655,53 +523,6 @@ ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam);
  */
 ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value);
 
-/*! ZSTD_DCtx_loadDictionary() :
- *  Create an internal DDict from dict buffer,
- *  to be used to decompress next frames.
- *  The dictionary remains valid for all future frames, until explicitly invalidated.
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- *  Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
- *            meaning "return to no-dictionary mode".
- *  Note 1 : Loading a dictionary involves building tables,
- *           which has a non-negligible impact on CPU usage and latency.
- *           It's recommended to "load once, use many times", to amortize the cost
- *  Note 2 :`dict` content will be copied internally, so `dict` can be released after loading.
- *           Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead.
- *  Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of
- *           how dictionary content is loaded and interpreted.
- */
-ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
-
-/*! ZSTD_DCtx_refDDict() :
- *  Reference a prepared dictionary, to be used to decompress next frames.
- *  The dictionary remains active for decompression of future frames using same DCtx.
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- *  Note 1 : Currently, only one dictionary can be managed.
- *           Referencing a new dictionary effectively "discards" any previous one.
- *  Special: referencing a NULL DDict means "return to no-dictionary mode".
- *  Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
- */
-ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
-
-/*! ZSTD_DCtx_refPrefix() :
- *  Reference a prefix (single-usage dictionary) to decompress next frame.
- *  This is the reverse operation of ZSTD_CCtx_refPrefix(),
- *  and must use the same prefix as the one used during compression.
- *  Prefix is **only used once**. Reference is discarded at end of frame.
- *  End of frame is reached when ZSTD_decompressStream() returns 0.
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- *  Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary
- *  Note 2 : Prefix buffer is referenced. It **must** outlive decompression.
- *           Prefix buffer must remain unmodified up to the end of frame,
- *           reached when ZSTD_decompressStream() returns 0.
- *  Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
- *           Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
- *  Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
- *           A full dictionary is more costly, as it requires building tables.
- */
-ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx,
-                                 const void* prefix, size_t prefixSize);
-
 /*! ZSTD_DCtx_reset() :
  *  Return a DCtx to clean state.
  *  Session and parameters can be reset jointly or separately.
@@ -822,6 +643,16 @@ ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
                                          ZSTD_inBuffer* input,
                                          ZSTD_EndDirective endOp);
 
+ZSTDLIB_API size_t ZSTD_CStreamInSize(void);    /**< recommended size for input buffer */
+ZSTDLIB_API size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */
+
+/*******************************************************************************
+ * This is a legacy streaming API, and can be replaced by ZSTD_CCtx_reset() and
+ * ZSTD_compressStream2(). It is redundent, but is still fully supported.
+ * Advanced parameters and dictionary compression can only be used through the
+ * new API.
+ ******************************************************************************/
+
 /**
  * Equivalent to:
  *
@@ -842,9 +673,6 @@ ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
 /** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */
 ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
 
-ZSTDLIB_API size_t ZSTD_CStreamInSize(void);    /**< recommended size for input buffer */
-ZSTDLIB_API size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */
-
 
 /*-***************************************************************************
 *  Streaming decompression - HowTo
@@ -886,6 +714,196 @@ ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* outp
 ZSTDLIB_API size_t ZSTD_DStreamInSize(void);    /*!< recommended size for input buffer */
 ZSTDLIB_API size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */
 
+
+/**************************
+*  Simple dictionary API
+***************************/
+/*! ZSTD_compress_usingDict() :
+ *  Compression at an explicit compression level using a Dictionary.
+ *  A dictionary can be any arbitrary data segment (also called a prefix),
+ *  or a buffer with specified information (see dictBuilder/zdict.h).
+ *  Note : This function loads the dictionary, resulting in significant startup delay.
+ *         It's intended for a dictionary used only once.
+ *  Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */
+ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
+                                           void* dst, size_t dstCapacity,
+                                     const void* src, size_t srcSize,
+                                     const void* dict,size_t dictSize,
+                                           int compressionLevel);
+
+/*! ZSTD_decompress_usingDict() :
+ *  Decompression using a known Dictionary.
+ *  Dictionary must be identical to the one used during compression.
+ *  Note : This function loads the dictionary, resulting in significant startup delay.
+ *         It's intended for a dictionary used only once.
+ *  Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
+ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
+                                             void* dst, size_t dstCapacity,
+                                       const void* src, size_t srcSize,
+                                       const void* dict,size_t dictSize);
+
+
+/***********************************
+ *  Bulk processing dictionary API
+ **********************************/
+typedef struct ZSTD_CDict_s ZSTD_CDict;
+
+/*! ZSTD_createCDict() :
+ *  When compressing multiple messages / blocks using the same dictionary, it's recommended to load it only once.
+ *  ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup cost.
+ *  ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
+ * `dictBuffer` can be released after ZSTD_CDict creation, because its content is copied within CDict.
+ *  Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate `dictBuffer` content.
+ *  Note : A ZSTD_CDict can be created from an empty dictBuffer, but it is inefficient when used to compress small data. */
+ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
+                                         int compressionLevel);
+
+/*! ZSTD_freeCDict() :
+ *  Function frees memory allocated by ZSTD_createCDict(). */
+ZSTDLIB_API size_t      ZSTD_freeCDict(ZSTD_CDict* CDict);
+
+/*! ZSTD_compress_usingCDict() :
+ *  Compression using a digested Dictionary.
+ *  Recommended when same dictionary is used multiple times.
+ *  Note : compression level is _decided at dictionary creation time_,
+ *     and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */
+ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
+                                            void* dst, size_t dstCapacity,
+                                      const void* src, size_t srcSize,
+                                      const ZSTD_CDict* cdict);
+
+
+typedef struct ZSTD_DDict_s ZSTD_DDict;
+
+/*! ZSTD_createDDict() :
+ *  Create a digested dictionary, ready to start decompression operation without startup delay.
+ *  dictBuffer can be released after DDict creation, as its content is copied inside DDict. */
+ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
+
+/*! ZSTD_freeDDict() :
+ *  Function frees memory allocated with ZSTD_createDDict() */
+ZSTDLIB_API size_t      ZSTD_freeDDict(ZSTD_DDict* ddict);
+
+/*! ZSTD_decompress_usingDDict() :
+ *  Decompression using a digested Dictionary.
+ *  Recommended when same dictionary is used multiple times. */
+ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
+                                              void* dst, size_t dstCapacity,
+                                        const void* src, size_t srcSize,
+                                        const ZSTD_DDict* ddict);
+
+
+/*******************************************************************************
+ * Advanced dictionary and prefix API
+ *
+ * This API allows dictionaries to be used with ZSTD_compress2(),
+ * ZSTD_compressStream2(), and ZSTD_decompress(). Dictionaries are sticky, and
+ * only reset with the context is reset with ZSTD_reset_parameters or
+ * ZSTD_reset_session_and_parameters. Prefixes are single-use.
+ ******************************************************************************/
+
+
+/*! ZSTD_CCtx_loadDictionary() :
+ *  Create an internal CDict from `dict` buffer.
+ *  Decompression will have to use same dictionary.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary,
+ *           meaning "return to no-dictionary mode".
+ *  Note 1 : Dictionary is sticky, it will be used for all future compressed frames.
+ *           To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters).
+ *  Note 2 : Loading a dictionary involves building tables.
+ *           It's also a CPU consuming operation, with non-negligible impact on latency.
+ *           Tables are dependent on compression parameters, and for this reason,
+ *           compression parameters can no longer be changed after loading a dictionary.
+ *  Note 3 :`dict` content will be copied internally.
+ *           Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead.
+ *           In such a case, dictionary buffer must outlive its users.
+ *  Note 4 : Use ZSTD_CCtx_loadDictionary_advanced()
+ *           to precisely select how dictionary content must be interpreted. */
+ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
+
+/*! ZSTD_CCtx_refCDict() :
+ *  Reference a prepared dictionary, to be used for all next compressed frames.
+ *  Note that compression parameters are enforced from within CDict,
+ *  and supersede any compression parameter previously set within CCtx.
+ *  The parameters ignored are labled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
+ *  The ignored parameters will be used again if the CCtx is returned to no-dictionary mode.
+ *  The dictionary will remain valid for future compressed frames using same CCtx.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Special : Referencing a NULL CDict means "return to no-dictionary mode".
+ *  Note 1 : Currently, only one dictionary can be managed.
+ *           Referencing a new dictionary effectively "discards" any previous one.
+ *  Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */
+ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
+
+/*! ZSTD_CCtx_refPrefix() :
+ *  Reference a prefix (single-usage dictionary) for next compressed frame.
+ *  A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end).
+ *  Decompression will need same prefix to properly regenerate data.
+ *  Compressing with a prefix is similar in outcome as performing a diff and compressing it,
+ *  but performs much faster, especially during decompression (compression speed is tunable with compression level).
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary
+ *  Note 1 : Prefix buffer is referenced. It **must** outlive compression.
+ *           Its content must remain unmodified during compression.
+ *  Note 2 : If the intention is to diff some large src data blob with some prior version of itself,
+ *           ensure that the window size is large enough to contain the entire source.
+ *           See ZSTD_c_windowLog.
+ *  Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
+ *           It's a CPU consuming operation, with non-negligible impact on latency.
+ *           If there is a need to use the same prefix multiple times, consider loadDictionary instead.
+ *  Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent).
+ *           Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */
+ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
+                                 const void* prefix, size_t prefixSize);
+
+/*! ZSTD_DCtx_loadDictionary() :
+ *  Create an internal DDict from dict buffer,
+ *  to be used to decompress next frames.
+ *  The dictionary remains valid for all future frames, until explicitly invalidated.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
+ *            meaning "return to no-dictionary mode".
+ *  Note 1 : Loading a dictionary involves building tables,
+ *           which has a non-negligible impact on CPU usage and latency.
+ *           It's recommended to "load once, use many times", to amortize the cost
+ *  Note 2 :`dict` content will be copied internally, so `dict` can be released after loading.
+ *           Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead.
+ *  Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of
+ *           how dictionary content is loaded and interpreted.
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
+
+/*! ZSTD_DCtx_refDDict() :
+ *  Reference a prepared dictionary, to be used to decompress next frames.
+ *  The dictionary remains active for decompression of future frames using same DCtx.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Note 1 : Currently, only one dictionary can be managed.
+ *           Referencing a new dictionary effectively "discards" any previous one.
+ *  Special: referencing a NULL DDict means "return to no-dictionary mode".
+ *  Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
+
+/*! ZSTD_DCtx_refPrefix() :
+ *  Reference a prefix (single-usage dictionary) to decompress next frame.
+ *  This is the reverse operation of ZSTD_CCtx_refPrefix(),
+ *  and must use the same prefix as the one used during compression.
+ *  Prefix is **only used once**. Reference is discarded at end of frame.
+ *  End of frame is reached when ZSTD_decompressStream() returns 0.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary
+ *  Note 2 : Prefix buffer is referenced. It **must** outlive decompression.
+ *           Prefix buffer must remain unmodified up to the end of frame,
+ *           reached when ZSTD_decompressStream() returns 0.
+ *  Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
+ *           Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
+ *  Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
+ *           A full dictionary is more costly, as it requires building tables.
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx,
+                                 const void* prefix, size_t prefixSize);
+
 /* ===   Memory management   === */
 
 /*! ZSTD_sizeof_*() :

From 7231ea72a8b52b5942e6ec4b5ca43f3352eb4e2a Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Wed, 3 Apr 2019 19:21:05 -0700
Subject: [PATCH 130/178] [libzstd] Reword the streaming docs for the new API

---
 lib/zstd.h | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/lib/zstd.h b/lib/zstd.h
index 4e6791560dc..e7023361f25 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -565,14 +565,15 @@ typedef struct ZSTD_outBuffer_s {
 *  Parameters are sticky : when starting a new compression on the same context,
 *  it will re-use the same sticky parameters as previous compression session.
 *  When in doubt, it's recommended to fully initialize the context before usage.
-*  Use ZSTD_initCStream() to set the parameter to a selected compression level.
-*  Use advanced API (ZSTD_CCtx_setParameter(), etc.) to set more specific parameters.
+*  Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(),
+*  ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to
+*  set more specific parameters, the pledged source size, or load a dictionary.
 *
-*  Use ZSTD_compressStream() as many times as necessary to consume input stream.
-*  The function will automatically update both `pos` fields within `input` and `output`.
-*  Note that the function may not consume the entire input,
-*  for example, because the output buffer is already full,
-*  in which case `input.pos < input.size`.
+*  Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to
+*  consume input stream. The function will automatically update both `pos`
+*  fields within `input` and `output`.
+*  Note that the function may not consume the entire input, for example, because
+*  the output buffer is already full, in which case `input.pos < input.size`.
 *  The caller must check if input has been entirely consumed.
 *  If not, the caller must make some room to receive more compressed data,
 *  and then present again remaining input data.
@@ -582,17 +583,21 @@ typedef struct ZSTD_outBuffer_s {
 *           Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize()
 *
 *  At any moment, it's possible to flush whatever data might remain stuck within internal buffer,
-*  using ZSTD_flushStream(). `output->pos` will be updated.
-*  Note that, if `output->size` is too small, a single invocation of ZSTD_flushStream() might not be enough (return code > 0).
-*  In which case, make some room to receive more compressed data, and call again ZSTD_flushStream().
+*  using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated.
+*  Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0).
+*  In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush.
+*  You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the
+*  operation.
 *  @return : 0 if internal buffers are entirely flushed,
 *            >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
 *            or an error code, which can be tested using ZSTD_isError().
 *
-*  ZSTD_endStream() instructs to finish a frame.
+*  Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame.
 *  It will perform a flush and write frame epilogue.
 *  The epilogue is required for decoders to consider a frame completed.
-*  flush() operation is the same, and follows same rules as ZSTD_flushStream().
+*  flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush.
+*  You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to
+*  start a new frame.
 *  @return : 0 if frame fully completed and fully flushed,
 *            >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
 *            or an error code, which can be tested using ZSTD_isError().

From 50c634b86ef3c79c619ccf14b1b1f16a61ae272a Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 5 Apr 2019 10:46:22 -0700
Subject: [PATCH 131/178] [regression] Update results.csv for level 1 change

---
 tests/regression/results.csv | 270 +++++++++++++++++------------------
 1 file changed, 135 insertions(+), 135 deletions(-)

diff --git a/tests/regression/results.csv b/tests/regression/results.csv
index f5c63932039..7ac94f207a6 100644
--- a/tests/regression/results.csv
+++ b/tests/regression/results.csv
@@ -1,9 +1,9 @@
 Data,                               Config,                             Method,                             Total compressed size
-silesia.tar,                        level -5,                           compress simple,                    7160438
-silesia.tar,                        level -3,                           compress simple,                    6789024
-silesia.tar,                        level -1,                           compress simple,                    6195462
+silesia.tar,                        level -5,                           compress simple,                    6738558
+silesia.tar,                        level -3,                           compress simple,                    6446362
+silesia.tar,                        level -1,                           compress simple,                    6186038
 silesia.tar,                        level 0,                            compress simple,                    4875008
-silesia.tar,                        level 1,                            compress simple,                    5339697
+silesia.tar,                        level 1,                            compress simple,                    5334825
 silesia.tar,                        level 3,                            compress simple,                    4875008
 silesia.tar,                        level 4,                            compress simple,                    4813507
 silesia.tar,                        level 5,                            compress simple,                    4722235
@@ -15,12 +15,12 @@ silesia.tar,                        level 16,                           compress
 silesia.tar,                        level 19,                           compress simple,                    4281514
 silesia.tar,                        uncompressed literals,              compress simple,                    4875008
 silesia.tar,                        uncompressed literals optimal,      compress simple,                    4281514
-silesia.tar,                        huffman literals,                   compress simple,                    6195462
-silesia,                            level -5,                           compress cctx,                      7152294
-silesia,                            level -3,                           compress cctx,                      6789969
-silesia,                            level -1,                           compress cctx,                      6191548
+silesia.tar,                        huffman literals,                   compress simple,                    6186038
+silesia,                            level -5,                           compress cctx,                      6737567
+silesia,                            level -3,                           compress cctx,                      6444663
+silesia,                            level -1,                           compress cctx,                      6178442
 silesia,                            level 0,                            compress cctx,                      4862377
-silesia,                            level 1,                            compress cctx,                      5318036
+silesia,                            level 1,                            compress cctx,                      5313144
 silesia,                            level 3,                            compress cctx,                      4862377
 silesia,                            level 4,                            compress cctx,                      4800629
 silesia,                            level 5,                            compress cctx,                      4710178
@@ -39,17 +39,17 @@ silesia,                            small chain log,                    compress
 silesia,                            explicit params,                    compress cctx,                      4813352
 silesia,                            uncompressed literals,              compress cctx,                      4862377
 silesia,                            uncompressed literals optimal,      compress cctx,                      4293262
-silesia,                            huffman literals,                   compress cctx,                      6191548
+silesia,                            huffman literals,                   compress cctx,                      6178442
 silesia,                            multithreaded with advanced params, compress cctx,                      4862377
-github,                             level -5,                           compress cctx,                      232744
+github,                             level -5,                           compress cctx,                      205285
 github,                             level -5 with dict,                 compress cctx,                      47294
-github,                             level -3,                           compress cctx,                      220611
+github,                             level -3,                           compress cctx,                      190643
 github,                             level -3 with dict,                 compress cctx,                      48047
-github,                             level -1,                           compress cctx,                      176575
+github,                             level -1,                           compress cctx,                      175568
 github,                             level -1 with dict,                 compress cctx,                      43527
 github,                             level 0,                            compress cctx,                      136397
 github,                             level 0 with dict,                  compress cctx,                      41536
-github,                             level 1,                            compress cctx,                      143457
+github,                             level 1,                            compress cctx,                      142450
 github,                             level 1 with dict,                  compress cctx,                      42157
 github,                             level 3,                            compress cctx,                      136397
 github,                             level 3 with dict,                  compress cctx,                      41536
@@ -78,13 +78,13 @@ github,                             small chain log,                    compress
 github,                             explicit params,                    compress cctx,                      140924
 github,                             uncompressed literals,              compress cctx,                      136397
 github,                             uncompressed literals optimal,      compress cctx,                      133717
-github,                             huffman literals,                   compress cctx,                      176575
+github,                             huffman literals,                   compress cctx,                      175568
 github,                             multithreaded with advanced params, compress cctx,                      141473
-silesia,                            level -5,                           zstdcli,                            7152342
-silesia,                            level -3,                           zstdcli,                            6790021
-silesia,                            level -1,                           zstdcli,                            6191597
+silesia,                            level -5,                           zstdcli,                            6882514
+silesia,                            level -3,                           zstdcli,                            6568406
+silesia,                            level -1,                           zstdcli,                            6183433
 silesia,                            level 0,                            zstdcli,                            4862425
-silesia,                            level 1,                            zstdcli,                            5318084
+silesia,                            level 1,                            zstdcli,                            5314157
 silesia,                            level 3,                            zstdcli,                            4862425
 silesia,                            level 4,                            zstdcli,                            4800677
 silesia,                            level 5,                            zstdcli,                            4710226
@@ -103,13 +103,13 @@ silesia,                            small chain log,                    zstdcli,
 silesia,                            explicit params,                    zstdcli,                            4815380
 silesia,                            uncompressed literals,              zstdcli,                            5155472
 silesia,                            uncompressed literals optimal,      zstdcli,                            4325475
-silesia,                            huffman literals,                   zstdcli,                            5341405
+silesia,                            huffman literals,                   zstdcli,                            5331158
 silesia,                            multithreaded with advanced params, zstdcli,                            5155472
-silesia.tar,                        level -5,                           zstdcli,                            7161160
-silesia.tar,                        level -3,                           zstdcli,                            6789865
-silesia.tar,                        level -1,                           zstdcli,                            6196433
+silesia.tar,                        level -5,                           zstdcli,                            6738906
+silesia.tar,                        level -3,                           zstdcli,                            6448409
+silesia.tar,                        level -1,                           zstdcli,                            6186908
 silesia.tar,                        level 0,                            zstdcli,                            4875136
-silesia.tar,                        level 1,                            zstdcli,                            5340573
+silesia.tar,                        level 1,                            zstdcli,                            5336255
 silesia.tar,                        level 3,                            zstdcli,                            4875136
 silesia.tar,                        level 4,                            zstdcli,                            4814531
 silesia.tar,                        level 5,                            zstdcli,                            4723284
@@ -129,17 +129,17 @@ silesia.tar,                        small chain log,                    zstdcli,
 silesia.tar,                        explicit params,                    zstdcli,                            4839202
 silesia.tar,                        uncompressed literals,              zstdcli,                            5158134
 silesia.tar,                        uncompressed literals optimal,      zstdcli,                            4321098
-silesia.tar,                        huffman literals,                   zstdcli,                            5358479
+silesia.tar,                        huffman literals,                   zstdcli,                            5347560
 silesia.tar,                        multithreaded with advanced params, zstdcli,                            5158134
-github,                             level -5,                           zstdcli,                            234744
+github,                             level -5,                           zstdcli,                            207285
 github,                             level -5 with dict,                 zstdcli,                            48718
-github,                             level -3,                           zstdcli,                            222611
+github,                             level -3,                           zstdcli,                            192643
 github,                             level -3 with dict,                 zstdcli,                            47395
-github,                             level -1,                           zstdcli,                            178575
+github,                             level -1,                           zstdcli,                            177568
 github,                             level -1 with dict,                 zstdcli,                            45170
 github,                             level 0,                            zstdcli,                            138397
 github,                             level 0 with dict,                  zstdcli,                            43170
-github,                             level 1,                            zstdcli,                            145457
+github,                             level 1,                            zstdcli,                            144450
 github,                             level 1 with dict,                  zstdcli,                            43682
 github,                             level 3,                            zstdcli,                            138397
 github,                             level 3 with dict,                  zstdcli,                            43170
@@ -168,13 +168,13 @@ github,                             small chain log,                    zstdcli,
 github,                             explicit params,                    zstdcli,                            136140
 github,                             uncompressed literals,              zstdcli,                            169004
 github,                             uncompressed literals optimal,      zstdcli,                            158824
-github,                             huffman literals,                   zstdcli,                            145457
+github,                             huffman literals,                   zstdcli,                            144450
 github,                             multithreaded with advanced params, zstdcli,                            169004
-silesia,                            level -5,                           advanced one pass,                  7152294
-silesia,                            level -3,                           advanced one pass,                  6789969
-silesia,                            level -1,                           advanced one pass,                  6191548
+silesia,                            level -5,                           advanced one pass,                  6737567
+silesia,                            level -3,                           advanced one pass,                  6444663
+silesia,                            level -1,                           advanced one pass,                  6178442
 silesia,                            level 0,                            advanced one pass,                  4862377
-silesia,                            level 1,                            advanced one pass,                  5318036
+silesia,                            level 1,                            advanced one pass,                  5313144
 silesia,                            level 3,                            advanced one pass,                  4862377
 silesia,                            level 4,                            advanced one pass,                  4800629
 silesia,                            level 5,                            advanced one pass,                  4710178
@@ -194,13 +194,13 @@ silesia,                            small chain log,                    advanced
 silesia,                            explicit params,                    advanced one pass,                  4815369
 silesia,                            uncompressed literals,              advanced one pass,                  5155424
 silesia,                            uncompressed literals optimal,      advanced one pass,                  4325427
-silesia,                            huffman literals,                   advanced one pass,                  5341356
+silesia,                            huffman literals,                   advanced one pass,                  5326210
 silesia,                            multithreaded with advanced params, advanced one pass,                  5155424
-silesia.tar,                        level -5,                           advanced one pass,                  7160438
-silesia.tar,                        level -3,                           advanced one pass,                  6789024
-silesia.tar,                        level -1,                           advanced one pass,                  6195462
+silesia.tar,                        level -5,                           advanced one pass,                  6738558
+silesia.tar,                        level -3,                           advanced one pass,                  6446362
+silesia.tar,                        level -1,                           advanced one pass,                  6186038
 silesia.tar,                        level 0,                            advanced one pass,                  4875008
-silesia.tar,                        level 1,                            advanced one pass,                  5339697
+silesia.tar,                        level 1,                            advanced one pass,                  5334825
 silesia.tar,                        level 3,                            advanced one pass,                  4875008
 silesia.tar,                        level 4,                            advanced one pass,                  4813507
 silesia.tar,                        level 5,                            advanced one pass,                  4722235
@@ -220,17 +220,17 @@ silesia.tar,                        small chain log,                    advanced
 silesia.tar,                        explicit params,                    advanced one pass,                  4829974
 silesia.tar,                        uncompressed literals,              advanced one pass,                  5157992
 silesia.tar,                        uncompressed literals optimal,      advanced one pass,                  4321094
-silesia.tar,                        huffman literals,                   advanced one pass,                  5358079
+silesia.tar,                        huffman literals,                   advanced one pass,                  5347283
 silesia.tar,                        multithreaded with advanced params, advanced one pass,                  5158545
-github,                             level -5,                           advanced one pass,                  232744
+github,                             level -5,                           advanced one pass,                  205285
 github,                             level -5 with dict,                 advanced one pass,                  46718
-github,                             level -3,                           advanced one pass,                  220611
+github,                             level -3,                           advanced one pass,                  190643
 github,                             level -3 with dict,                 advanced one pass,                  45395
-github,                             level -1,                           advanced one pass,                  176575
+github,                             level -1,                           advanced one pass,                  175568
 github,                             level -1 with dict,                 advanced one pass,                  43170
 github,                             level 0,                            advanced one pass,                  136397
 github,                             level 0 with dict,                  advanced one pass,                  41170
-github,                             level 1,                            advanced one pass,                  143457
+github,                             level 1,                            advanced one pass,                  142450
 github,                             level 1 with dict,                  advanced one pass,                  41682
 github,                             level 3,                            advanced one pass,                  136397
 github,                             level 3 with dict,                  advanced one pass,                  41170
@@ -260,13 +260,13 @@ github,                             small chain log,                    advanced
 github,                             explicit params,                    advanced one pass,                  137670
 github,                             uncompressed literals,              advanced one pass,                  167004
 github,                             uncompressed literals optimal,      advanced one pass,                  156824
-github,                             huffman literals,                   advanced one pass,                  143457
+github,                             huffman literals,                   advanced one pass,                  142450
 github,                             multithreaded with advanced params, advanced one pass,                  167004
-silesia,                            level -5,                           advanced one pass small out,        7152294
-silesia,                            level -3,                           advanced one pass small out,        6789969
-silesia,                            level -1,                           advanced one pass small out,        6191548
+silesia,                            level -5,                           advanced one pass small out,        6737567
+silesia,                            level -3,                           advanced one pass small out,        6444663
+silesia,                            level -1,                           advanced one pass small out,        6178442
 silesia,                            level 0,                            advanced one pass small out,        4862377
-silesia,                            level 1,                            advanced one pass small out,        5318036
+silesia,                            level 1,                            advanced one pass small out,        5313144
 silesia,                            level 3,                            advanced one pass small out,        4862377
 silesia,                            level 4,                            advanced one pass small out,        4800629
 silesia,                            level 5,                            advanced one pass small out,        4710178
@@ -286,13 +286,13 @@ silesia,                            small chain log,                    advanced
 silesia,                            explicit params,                    advanced one pass small out,        4815369
 silesia,                            uncompressed literals,              advanced one pass small out,        5155424
 silesia,                            uncompressed literals optimal,      advanced one pass small out,        4325427
-silesia,                            huffman literals,                   advanced one pass small out,        5341356
+silesia,                            huffman literals,                   advanced one pass small out,        5326210
 silesia,                            multithreaded with advanced params, advanced one pass small out,        5155424
-silesia.tar,                        level -5,                           advanced one pass small out,        7160438
-silesia.tar,                        level -3,                           advanced one pass small out,        6789024
-silesia.tar,                        level -1,                           advanced one pass small out,        6195462
+silesia.tar,                        level -5,                           advanced one pass small out,        6738558
+silesia.tar,                        level -3,                           advanced one pass small out,        6446362
+silesia.tar,                        level -1,                           advanced one pass small out,        6186038
 silesia.tar,                        level 0,                            advanced one pass small out,        4875008
-silesia.tar,                        level 1,                            advanced one pass small out,        5339697
+silesia.tar,                        level 1,                            advanced one pass small out,        5334825
 silesia.tar,                        level 3,                            advanced one pass small out,        4875008
 silesia.tar,                        level 4,                            advanced one pass small out,        4813507
 silesia.tar,                        level 5,                            advanced one pass small out,        4722235
@@ -312,17 +312,17 @@ silesia.tar,                        small chain log,                    advanced
 silesia.tar,                        explicit params,                    advanced one pass small out,        4829974
 silesia.tar,                        uncompressed literals,              advanced one pass small out,        5157992
 silesia.tar,                        uncompressed literals optimal,      advanced one pass small out,        4321094
-silesia.tar,                        huffman literals,                   advanced one pass small out,        5358079
+silesia.tar,                        huffman literals,                   advanced one pass small out,        5347283
 silesia.tar,                        multithreaded with advanced params, advanced one pass small out,        5158545
-github,                             level -5,                           advanced one pass small out,        232744
+github,                             level -5,                           advanced one pass small out,        205285
 github,                             level -5 with dict,                 advanced one pass small out,        46718
-github,                             level -3,                           advanced one pass small out,        220611
+github,                             level -3,                           advanced one pass small out,        190643
 github,                             level -3 with dict,                 advanced one pass small out,        45395
-github,                             level -1,                           advanced one pass small out,        176575
+github,                             level -1,                           advanced one pass small out,        175568
 github,                             level -1 with dict,                 advanced one pass small out,        43170
 github,                             level 0,                            advanced one pass small out,        136397
 github,                             level 0 with dict,                  advanced one pass small out,        41170
-github,                             level 1,                            advanced one pass small out,        143457
+github,                             level 1,                            advanced one pass small out,        142450
 github,                             level 1 with dict,                  advanced one pass small out,        41682
 github,                             level 3,                            advanced one pass small out,        136397
 github,                             level 3 with dict,                  advanced one pass small out,        41170
@@ -352,13 +352,13 @@ github,                             small chain log,                    advanced
 github,                             explicit params,                    advanced one pass small out,        137670
 github,                             uncompressed literals,              advanced one pass small out,        167004
 github,                             uncompressed literals optimal,      advanced one pass small out,        156824
-github,                             huffman literals,                   advanced one pass small out,        143457
+github,                             huffman literals,                   advanced one pass small out,        142450
 github,                             multithreaded with advanced params, advanced one pass small out,        167004
-silesia,                            level -5,                           advanced streaming,                 7152294
-silesia,                            level -3,                           advanced streaming,                 6789973
-silesia,                            level -1,                           advanced streaming,                 6191549
+silesia,                            level -5,                           advanced streaming,                 6882466
+silesia,                            level -3,                           advanced streaming,                 6568358
+silesia,                            level -1,                           advanced streaming,                 6183385
 silesia,                            level 0,                            advanced streaming,                 4862377
-silesia,                            level 1,                            advanced streaming,                 5318036
+silesia,                            level 1,                            advanced streaming,                 5314109
 silesia,                            level 3,                            advanced streaming,                 4862377
 silesia,                            level 4,                            advanced streaming,                 4800629
 silesia,                            level 5,                            advanced streaming,                 4710178
@@ -378,13 +378,13 @@ silesia,                            small chain log,                    advanced
 silesia,                            explicit params,                    advanced streaming,                 4815380
 silesia,                            uncompressed literals,              advanced streaming,                 5155424
 silesia,                            uncompressed literals optimal,      advanced streaming,                 4325427
-silesia,                            huffman literals,                   advanced streaming,                 5341357
+silesia,                            huffman literals,                   advanced streaming,                 5331110
 silesia,                            multithreaded with advanced params, advanced streaming,                 5155424
-silesia.tar,                        level -5,                           advanced streaming,                 7160440
-silesia.tar,                        level -3,                           advanced streaming,                 6789026
-silesia.tar,                        level -1,                           advanced streaming,                 6195465
+silesia.tar,                        level -5,                           advanced streaming,                 6982738
+silesia.tar,                        level -3,                           advanced streaming,                 6641264
+silesia.tar,                        level -1,                           advanced streaming,                 6190789
 silesia.tar,                        level 0,                            advanced streaming,                 4875010
-silesia.tar,                        level 1,                            advanced streaming,                 5339701
+silesia.tar,                        level 1,                            advanced streaming,                 5336879
 silesia.tar,                        level 3,                            advanced streaming,                 4875010
 silesia.tar,                        level 4,                            advanced streaming,                 4813507
 silesia.tar,                        level 5,                            advanced streaming,                 4722240
@@ -404,17 +404,17 @@ silesia.tar,                        small chain log,                    advanced
 silesia.tar,                        explicit params,                    advanced streaming,                 4830002
 silesia.tar,                        uncompressed literals,              advanced streaming,                 5157995
 silesia.tar,                        uncompressed literals optimal,      advanced streaming,                 4321094
-silesia.tar,                        huffman literals,                   advanced streaming,                 5358083
+silesia.tar,                        huffman literals,                   advanced streaming,                 5352306
 silesia.tar,                        multithreaded with advanced params, advanced streaming,                 5158130
-github,                             level -5,                           advanced streaming,                 232744
+github,                             level -5,                           advanced streaming,                 205285
 github,                             level -5 with dict,                 advanced streaming,                 46718
-github,                             level -3,                           advanced streaming,                 220611
+github,                             level -3,                           advanced streaming,                 190643
 github,                             level -3 with dict,                 advanced streaming,                 45395
-github,                             level -1,                           advanced streaming,                 176575
+github,                             level -1,                           advanced streaming,                 175568
 github,                             level -1 with dict,                 advanced streaming,                 43170
 github,                             level 0,                            advanced streaming,                 136397
 github,                             level 0 with dict,                  advanced streaming,                 41170
-github,                             level 1,                            advanced streaming,                 143457
+github,                             level 1,                            advanced streaming,                 142450
 github,                             level 1 with dict,                  advanced streaming,                 41682
 github,                             level 3,                            advanced streaming,                 136397
 github,                             level 3 with dict,                  advanced streaming,                 41170
@@ -444,13 +444,13 @@ github,                             small chain log,                    advanced
 github,                             explicit params,                    advanced streaming,                 137670
 github,                             uncompressed literals,              advanced streaming,                 167004
 github,                             uncompressed literals optimal,      advanced streaming,                 156824
-github,                             huffman literals,                   advanced streaming,                 143457
+github,                             huffman literals,                   advanced streaming,                 142450
 github,                             multithreaded with advanced params, advanced streaming,                 167004
-silesia,                            level -5,                           old streaming,                      7152294
-silesia,                            level -3,                           old streaming,                      6789973
-silesia,                            level -1,                           old streaming,                      6191549
+silesia,                            level -5,                           old streaming,                      6882466
+silesia,                            level -3,                           old streaming,                      6568358
+silesia,                            level -1,                           old streaming,                      6183385
 silesia,                            level 0,                            old streaming,                      4862377
-silesia,                            level 1,                            old streaming,                      5318036
+silesia,                            level 1,                            old streaming,                      5314109
 silesia,                            level 3,                            old streaming,                      4862377
 silesia,                            level 4,                            old streaming,                      4800629
 silesia,                            level 5,                            old streaming,                      4710178
@@ -470,13 +470,13 @@ silesia,                            small chain log,                    old stre
 silesia,                            explicit params,                    old streaming,                      12000408
 silesia,                            uncompressed literals,              old streaming,                      4862377
 silesia,                            uncompressed literals optimal,      old streaming,                      4293262
-silesia,                            huffman literals,                   old streaming,                      6191549
+silesia,                            huffman literals,                   old streaming,                      6183385
 silesia,                            multithreaded with advanced params, old streaming,                      12000408
-silesia.tar,                        level -5,                           old streaming,                      7160440
-silesia.tar,                        level -3,                           old streaming,                      6789026
-silesia.tar,                        level -1,                           old streaming,                      6195465
+silesia.tar,                        level -5,                           old streaming,                      6982738
+silesia.tar,                        level -3,                           old streaming,                      6641264
+silesia.tar,                        level -1,                           old streaming,                      6190789
 silesia.tar,                        level 0,                            old streaming,                      4875010
-silesia.tar,                        level 1,                            old streaming,                      5339701
+silesia.tar,                        level 1,                            old streaming,                      5336879
 silesia.tar,                        level 3,                            old streaming,                      4875010
 silesia.tar,                        level 4,                            old streaming,                      4813507
 silesia.tar,                        level 5,                            old streaming,                      4722240
@@ -496,17 +496,17 @@ silesia.tar,                        small chain log,                    old stre
 silesia.tar,                        explicit params,                    old streaming,                      12022046
 silesia.tar,                        uncompressed literals,              old streaming,                      4875010
 silesia.tar,                        uncompressed literals optimal,      old streaming,                      4281514
-silesia.tar,                        huffman literals,                   old streaming,                      6195465
+silesia.tar,                        huffman literals,                   old streaming,                      6190789
 silesia.tar,                        multithreaded with advanced params, old streaming,                      12022046
-github,                             level -5,                           old streaming,                      232744
+github,                             level -5,                           old streaming,                      205285
 github,                             level -5 with dict,                 old streaming,                      46718
-github,                             level -3,                           old streaming,                      220611
+github,                             level -3,                           old streaming,                      190643
 github,                             level -3 with dict,                 old streaming,                      45395
-github,                             level -1,                           old streaming,                      176575
+github,                             level -1,                           old streaming,                      175568
 github,                             level -1 with dict,                 old streaming,                      43170
 github,                             level 0,                            old streaming,                      136397
 github,                             level 0 with dict,                  old streaming,                      41170
-github,                             level 1,                            old streaming,                      143457
+github,                             level 1,                            old streaming,                      142450
 github,                             level 1 with dict,                  old streaming,                      41682
 github,                             level 3,                            old streaming,                      136397
 github,                             level 3 with dict,                  old streaming,                      41170
@@ -536,13 +536,13 @@ github,                             small chain log,                    old stre
 github,                             explicit params,                    old streaming,                      412933
 github,                             uncompressed literals,              old streaming,                      136397
 github,                             uncompressed literals optimal,      old streaming,                      133717
-github,                             huffman literals,                   old streaming,                      176575
+github,                             huffman literals,                   old streaming,                      175568
 github,                             multithreaded with advanced params, old streaming,                      412933
-silesia,                            level -5,                           old streaming advanced,             7152294
-silesia,                            level -3,                           old streaming advanced,             6789973
-silesia,                            level -1,                           old streaming advanced,             6191549
+silesia,                            level -5,                           old streaming advanced,             6882466
+silesia,                            level -3,                           old streaming advanced,             6568358
+silesia,                            level -1,                           old streaming advanced,             6183385
 silesia,                            level 0,                            old streaming advanced,             4862377
-silesia,                            level 1,                            old streaming advanced,             5318036
+silesia,                            level 1,                            old streaming advanced,             5314109
 silesia,                            level 3,                            old streaming advanced,             4862377
 silesia,                            level 4,                            old streaming advanced,             4800629
 silesia,                            level 5,                            old streaming advanced,             4710178
@@ -562,13 +562,13 @@ silesia,                            small chain log,                    old stre
 silesia,                            explicit params,                    old streaming advanced,             12000408
 silesia,                            uncompressed literals,              old streaming advanced,             4862377
 silesia,                            uncompressed literals optimal,      old streaming advanced,             4293262
-silesia,                            huffman literals,                   old streaming advanced,             6191549
+silesia,                            huffman literals,                   old streaming advanced,             6183385
 silesia,                            multithreaded with advanced params, old streaming advanced,             12000408
-silesia.tar,                        level -5,                           old streaming advanced,             7160440
-silesia.tar,                        level -3,                           old streaming advanced,             6789026
-silesia.tar,                        level -1,                           old streaming advanced,             6195465
+silesia.tar,                        level -5,                           old streaming advanced,             6982738
+silesia.tar,                        level -3,                           old streaming advanced,             6641264
+silesia.tar,                        level -1,                           old streaming advanced,             6190789
 silesia.tar,                        level 0,                            old streaming advanced,             4875010
-silesia.tar,                        level 1,                            old streaming advanced,             5339701
+silesia.tar,                        level 1,                            old streaming advanced,             5336879
 silesia.tar,                        level 3,                            old streaming advanced,             4875010
 silesia.tar,                        level 4,                            old streaming advanced,             4813507
 silesia.tar,                        level 5,                            old streaming advanced,             4722240
@@ -588,17 +588,17 @@ silesia.tar,                        small chain log,                    old stre
 silesia.tar,                        explicit params,                    old streaming advanced,             12022046
 silesia.tar,                        uncompressed literals,              old streaming advanced,             4875010
 silesia.tar,                        uncompressed literals optimal,      old streaming advanced,             4281514
-silesia.tar,                        huffman literals,                   old streaming advanced,             6195465
+silesia.tar,                        huffman literals,                   old streaming advanced,             6190789
 silesia.tar,                        multithreaded with advanced params, old streaming advanced,             12022046
-github,                             level -5,                           old streaming advanced,             232744
+github,                             level -5,                           old streaming advanced,             205285
 github,                             level -5 with dict,                 old streaming advanced,             46718
-github,                             level -3,                           old streaming advanced,             220611
+github,                             level -3,                           old streaming advanced,             190643
 github,                             level -3 with dict,                 old streaming advanced,             45395
-github,                             level -1,                           old streaming advanced,             176575
+github,                             level -1,                           old streaming advanced,             175568
 github,                             level -1 with dict,                 old streaming advanced,             43170
 github,                             level 0,                            old streaming advanced,             136397
 github,                             level 0 with dict,                  old streaming advanced,             41170
-github,                             level 1,                            old streaming advanced,             143457
+github,                             level 1,                            old streaming advanced,             142450
 github,                             level 1 with dict,                  old streaming advanced,             41682
 github,                             level 3,                            old streaming advanced,             136397
 github,                             level 3 with dict,                  old streaming advanced,             41170
@@ -628,13 +628,13 @@ github,                             small chain log,                    old stre
 github,                             explicit params,                    old streaming advanced,             412933
 github,                             uncompressed literals,              old streaming advanced,             136397
 github,                             uncompressed literals optimal,      old streaming advanced,             133717
-github,                             huffman literals,                   old streaming advanced,             176575
+github,                             huffman literals,                   old streaming advanced,             175568
 github,                             multithreaded with advanced params, old streaming advanced,             412933
-silesia,                            level -5,                           old streaming cdcit,                7152294
-silesia,                            level -3,                           old streaming cdcit,                6789973
-silesia,                            level -1,                           old streaming cdcit,                6191549
+silesia,                            level -5,                           old streaming cdcit,                6882466
+silesia,                            level -3,                           old streaming cdcit,                6568358
+silesia,                            level -1,                           old streaming cdcit,                6183385
 silesia,                            level 0,                            old streaming cdcit,                4862377
-silesia,                            level 1,                            old streaming cdcit,                5318036
+silesia,                            level 1,                            old streaming cdcit,                5314109
 silesia,                            level 3,                            old streaming cdcit,                4862377
 silesia,                            level 4,                            old streaming cdcit,                4800629
 silesia,                            level 5,                            old streaming cdcit,                4710178
@@ -654,13 +654,13 @@ silesia,                            small chain log,                    old stre
 silesia,                            explicit params,                    old streaming cdcit,                12000408
 silesia,                            uncompressed literals,              old streaming cdcit,                4862377
 silesia,                            uncompressed literals optimal,      old streaming cdcit,                4293262
-silesia,                            huffman literals,                   old streaming cdcit,                6191549
+silesia,                            huffman literals,                   old streaming cdcit,                6183385
 silesia,                            multithreaded with advanced params, old streaming cdcit,                12000408
-silesia.tar,                        level -5,                           old streaming cdcit,                7160440
-silesia.tar,                        level -3,                           old streaming cdcit,                6789026
-silesia.tar,                        level -1,                           old streaming cdcit,                6195465
+silesia.tar,                        level -5,                           old streaming cdcit,                6982738
+silesia.tar,                        level -3,                           old streaming cdcit,                6641264
+silesia.tar,                        level -1,                           old streaming cdcit,                6190789
 silesia.tar,                        level 0,                            old streaming cdcit,                4875010
-silesia.tar,                        level 1,                            old streaming cdcit,                5339701
+silesia.tar,                        level 1,                            old streaming cdcit,                5336879
 silesia.tar,                        level 3,                            old streaming cdcit,                4875010
 silesia.tar,                        level 4,                            old streaming cdcit,                4813507
 silesia.tar,                        level 5,                            old streaming cdcit,                4722240
@@ -680,17 +680,17 @@ silesia.tar,                        small chain log,                    old stre
 silesia.tar,                        explicit params,                    old streaming cdcit,                12022046
 silesia.tar,                        uncompressed literals,              old streaming cdcit,                4875010
 silesia.tar,                        uncompressed literals optimal,      old streaming cdcit,                4281514
-silesia.tar,                        huffman literals,                   old streaming cdcit,                6195465
+silesia.tar,                        huffman literals,                   old streaming cdcit,                6190789
 silesia.tar,                        multithreaded with advanced params, old streaming cdcit,                12022046
-github,                             level -5,                           old streaming cdcit,                232744
+github,                             level -5,                           old streaming cdcit,                205285
 github,                             level -5 with dict,                 old streaming cdcit,                46718
-github,                             level -3,                           old streaming cdcit,                220611
+github,                             level -3,                           old streaming cdcit,                190643
 github,                             level -3 with dict,                 old streaming cdcit,                45395
-github,                             level -1,                           old streaming cdcit,                176575
+github,                             level -1,                           old streaming cdcit,                175568
 github,                             level -1 with dict,                 old streaming cdcit,                43170
 github,                             level 0,                            old streaming cdcit,                136397
 github,                             level 0 with dict,                  old streaming cdcit,                41170
-github,                             level 1,                            old streaming cdcit,                143457
+github,                             level 1,                            old streaming cdcit,                142450
 github,                             level 1 with dict,                  old streaming cdcit,                41682
 github,                             level 3,                            old streaming cdcit,                136397
 github,                             level 3 with dict,                  old streaming cdcit,                41170
@@ -720,13 +720,13 @@ github,                             small chain log,                    old stre
 github,                             explicit params,                    old streaming cdcit,                412933
 github,                             uncompressed literals,              old streaming cdcit,                136397
 github,                             uncompressed literals optimal,      old streaming cdcit,                133717
-github,                             huffman literals,                   old streaming cdcit,                176575
+github,                             huffman literals,                   old streaming cdcit,                175568
 github,                             multithreaded with advanced params, old streaming cdcit,                412933
-silesia,                            level -5,                           old streaming advanced cdict,       7152294
-silesia,                            level -3,                           old streaming advanced cdict,       6789973
-silesia,                            level -1,                           old streaming advanced cdict,       6191549
+silesia,                            level -5,                           old streaming advanced cdict,       6882466
+silesia,                            level -3,                           old streaming advanced cdict,       6568358
+silesia,                            level -1,                           old streaming advanced cdict,       6183385
 silesia,                            level 0,                            old streaming advanced cdict,       4862377
-silesia,                            level 1,                            old streaming advanced cdict,       5318036
+silesia,                            level 1,                            old streaming advanced cdict,       5314109
 silesia,                            level 3,                            old streaming advanced cdict,       4862377
 silesia,                            level 4,                            old streaming advanced cdict,       4800629
 silesia,                            level 5,                            old streaming advanced cdict,       4710178
@@ -746,13 +746,13 @@ silesia,                            small chain log,                    old stre
 silesia,                            explicit params,                    old streaming advanced cdict,       12000408
 silesia,                            uncompressed literals,              old streaming advanced cdict,       4862377
 silesia,                            uncompressed literals optimal,      old streaming advanced cdict,       4293262
-silesia,                            huffman literals,                   old streaming advanced cdict,       6191549
+silesia,                            huffman literals,                   old streaming advanced cdict,       6183385
 silesia,                            multithreaded with advanced params, old streaming advanced cdict,       12000408
-silesia.tar,                        level -5,                           old streaming advanced cdict,       7160440
-silesia.tar,                        level -3,                           old streaming advanced cdict,       6789026
-silesia.tar,                        level -1,                           old streaming advanced cdict,       6195465
+silesia.tar,                        level -5,                           old streaming advanced cdict,       6982738
+silesia.tar,                        level -3,                           old streaming advanced cdict,       6641264
+silesia.tar,                        level -1,                           old streaming advanced cdict,       6190789
 silesia.tar,                        level 0,                            old streaming advanced cdict,       4875010
-silesia.tar,                        level 1,                            old streaming advanced cdict,       5339701
+silesia.tar,                        level 1,                            old streaming advanced cdict,       5336879
 silesia.tar,                        level 3,                            old streaming advanced cdict,       4875010
 silesia.tar,                        level 4,                            old streaming advanced cdict,       4813507
 silesia.tar,                        level 5,                            old streaming advanced cdict,       4722240
@@ -772,17 +772,17 @@ silesia.tar,                        small chain log,                    old stre
 silesia.tar,                        explicit params,                    old streaming advanced cdict,       12022046
 silesia.tar,                        uncompressed literals,              old streaming advanced cdict,       4875010
 silesia.tar,                        uncompressed literals optimal,      old streaming advanced cdict,       4281514
-silesia.tar,                        huffman literals,                   old streaming advanced cdict,       6195465
+silesia.tar,                        huffman literals,                   old streaming advanced cdict,       6190789
 silesia.tar,                        multithreaded with advanced params, old streaming advanced cdict,       12022046
-github,                             level -5,                           old streaming advanced cdict,       232744
+github,                             level -5,                           old streaming advanced cdict,       205285
 github,                             level -5 with dict,                 old streaming advanced cdict,       46718
-github,                             level -3,                           old streaming advanced cdict,       220611
+github,                             level -3,                           old streaming advanced cdict,       190643
 github,                             level -3 with dict,                 old streaming advanced cdict,       45395
-github,                             level -1,                           old streaming advanced cdict,       176575
+github,                             level -1,                           old streaming advanced cdict,       175568
 github,                             level -1 with dict,                 old streaming advanced cdict,       43170
 github,                             level 0,                            old streaming advanced cdict,       136397
 github,                             level 0 with dict,                  old streaming advanced cdict,       41170
-github,                             level 1,                            old streaming advanced cdict,       143457
+github,                             level 1,                            old streaming advanced cdict,       142450
 github,                             level 1 with dict,                  old streaming advanced cdict,       41682
 github,                             level 3,                            old streaming advanced cdict,       136397
 github,                             level 3 with dict,                  old streaming advanced cdict,       41170
@@ -812,5 +812,5 @@ github,                             small chain log,                    old stre
 github,                             explicit params,                    old streaming advanced cdict,       412933
 github,                             uncompressed literals,              old streaming advanced cdict,       136397
 github,                             uncompressed literals optimal,      old streaming advanced cdict,       133717
-github,                             huffman literals,                   old streaming advanced cdict,       176575
+github,                             huffman literals,                   old streaming advanced cdict,       175568
 github,                             multithreaded with advanced params, old streaming advanced cdict,       412933

From ce388fe4d265407fc2aa2cdeb6dec1efc07bb7fc Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 5 Apr 2019 17:44:07 -0700
Subject: [PATCH 132/178] [libzstd] Fix return value docs for
 ZSTD_compressStream2()

---
 lib/zstd.h | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/lib/zstd.h b/lib/zstd.h
index e7023361f25..13dff417bdf 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -577,10 +577,8 @@ typedef struct ZSTD_outBuffer_s {
 *  The caller must check if input has been entirely consumed.
 *  If not, the caller must make some room to receive more compressed data,
 *  and then present again remaining input data.
-* @return : a size hint, preferred nb of bytes to use as input for next function call
+* @return : provides a minimum amount of data remaining to be flushed from internal buffers
 *           or an error code, which can be tested using ZSTD_isError().
-*           Note 1 : it's just a hint, to help latency a little, any value will work fine.
-*           Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize()
 *
 *  At any moment, it's possible to flush whatever data might remain stuck within internal buffer,
 *  using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated.

From 1dfe37fea9ad5ddc7d95c5b3248a1d2759e2f24e Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 5 Apr 2019 17:44:56 -0700
Subject: [PATCH 133/178] [libzstd] Stabilize ZSTD_getDictID_*() functions

---
 lib/zstd.h | 54 +++++++++++++++++++++++++++++-------------------------
 1 file changed, 29 insertions(+), 25 deletions(-)

diff --git a/lib/zstd.h b/lib/zstd.h
index 13dff417bdf..dc6348659bd 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -796,6 +796,35 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
                                         const ZSTD_DDict* ddict);
 
 
+/********************************
+ *  Dictionary helper functions
+ *******************************/
+
+/*! ZSTD_getDictID_fromDict() :
+ *  Provides the dictID stored within dictionary.
+ *  if @return == 0, the dictionary is not conformant with Zstandard specification.
+ *  It can still be loaded, but as a content-only dictionary. */
+ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
+
+/*! ZSTD_getDictID_fromDDict() :
+ *  Provides the dictID of the dictionary loaded into `ddict`.
+ *  If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
+ *  Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
+ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
+
+/*! ZSTD_getDictID_fromFrame() :
+ *  Provides the dictID required to decompressed the frame stored within `src`.
+ *  If @return == 0, the dictID could not be decoded.
+ *  This could for one of the following reasons :
+ *  - The frame does not require a dictionary to be decoded (most common case).
+ *  - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.
+ *    Note : this use case also happens when using a non-conformant dictionary.
+ *  - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
+ *  - This is not a Zstandard frame.
+ *  When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */
+ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
+
+
 /*******************************************************************************
  * Advanced dictionary and prefix API
  *
@@ -1463,31 +1492,6 @@ ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size);
  *  it must remain read accessible throughout the lifetime of DDict */
 ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
 
-
-/*! ZSTD_getDictID_fromDict() :
- *  Provides the dictID stored within dictionary.
- *  if @return == 0, the dictionary is not conformant with Zstandard specification.
- *  It can still be loaded, but as a content-only dictionary. */
-ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
-
-/*! ZSTD_getDictID_fromDDict() :
- *  Provides the dictID of the dictionary loaded into `ddict`.
- *  If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
- *  Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
-ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
-
-/*! ZSTD_getDictID_fromFrame() :
- *  Provides the dictID required to decompressed the frame stored within `src`.
- *  If @return == 0, the dictID could not be decoded.
- *  This could for one of the following reasons :
- *  - The frame does not require a dictionary to be decoded (most common case).
- *  - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.
- *    Note : this use case also happens when using a non-conformant dictionary.
- *  - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
- *  - This is not a Zstandard frame.
- *  When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */
-ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
-
 /*! ZSTD_DCtx_loadDictionary_byReference() :
  *  Same as ZSTD_DCtx_loadDictionary(),
  *  but references `dict` content instead of copying it into `dctx`.

From 1d0c1707d1f39e9dade995f5bc6b734055c093af Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 5 Apr 2019 18:11:17 -0700
Subject: [PATCH 134/178] [examples] Clean up and comment the examples

---
 examples/Makefile                         | 16 +++---
 examples/{utils.h => common.h}            | 65 +++++++++++------------
 examples/dictionary_compression.c         | 26 +++++----
 examples/dictionary_decompression.c       | 55 +++++++++++--------
 examples/multiple_simple_compression.c    | 25 +++++----
 examples/multiple_streaming_compression.c | 13 +++--
 examples/simple_compression.c             | 19 ++++---
 examples/simple_decompression.c           | 39 +++++++-------
 examples/streaming_compression.c          | 12 ++---
 examples/streaming_decompression.c        |  8 ++-
 examples/streaming_memory_usage.c         |  6 +--
 11 files changed, 143 insertions(+), 141 deletions(-)
 rename examples/{utils.h => common.h} (97%)

diff --git a/examples/Makefile b/examples/Makefile
index 25a0a62c2de..cd995f2f8b4 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -25,28 +25,28 @@ all: simple_compression simple_decompression \
 $(LIB) :
 	$(MAKE) -C ../lib libzstd.a
 
-simple_compression : simple_compression.c utils.h $(LIB)
+simple_compression : simple_compression.c common.h $(LIB)
 	$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
 
-simple_decompression : simple_decompression.c utils.h $(LIB)
+simple_decompression : simple_decompression.c common.h $(LIB)
 	$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
 
-multiple_simple_compression : multiple_simple_compression.c utils.h $(LIB)
+multiple_simple_compression : multiple_simple_compression.c common.h $(LIB)
 	$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
 
-dictionary_compression : dictionary_compression.c utils.h $(LIB)
+dictionary_compression : dictionary_compression.c common.h $(LIB)
 	$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
 
-dictionary_decompression : dictionary_decompression.c utils.h $(LIB)
+dictionary_decompression : dictionary_decompression.c common.h $(LIB)
 	$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
 
-streaming_compression : streaming_compression.c utils.h $(LIB)
+streaming_compression : streaming_compression.c common.h $(LIB)
 	$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
 
-multiple_streaming_compression : multiple_streaming_compression.c utils.h $(LIB)
+multiple_streaming_compression : multiple_streaming_compression.c common.h $(LIB)
 	$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
 
-streaming_decompression : streaming_decompression.c utils.h $(LIB)
+streaming_decompression : streaming_decompression.c common.h $(LIB)
 	$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
 
 streaming_memory_usage : streaming_memory_usage.c $(LIB)
diff --git a/examples/utils.h b/examples/common.h
similarity index 97%
rename from examples/utils.h
rename to examples/common.h
index 9ee93339c83..a714cbb72c3 100644
--- a/examples/utils.h
+++ b/examples/common.h
@@ -11,14 +11,13 @@
 /*
  * This header file has common utility functions used in examples.
  */
-#ifndef UTILS_H
-#define UTILS_H
+#ifndef COMMON_H
+#define COMMON_H
 
 #include     // malloc, free, exit
 #include      // fprintf, perror, fopen, etc.
-#include     // strlen, strcat, memset, strerror
+#include     // strerror
 #include      // errno
-#include     // assert
 #include   // stat
 #include 
 
@@ -35,7 +34,34 @@ typedef enum {
     ERROR_saveFile = 7,
     ERROR_malloc = 8,
     ERROR_largeFile = 9,
-} UTILS_ErrorCode;
+} COMMON_ErrorCode;
+
+/*! CHECK
+ * Check that the condition holds. If it doesn't print a message and die.
+ */
+#define CHECK(cond, ...)                        \
+    do {                                        \
+        if (!(cond)) {                          \
+            fprintf(stderr,                     \
+                    "%s:%d CHECK(%s) failed: ", \
+                    __FILE__,                   \
+                    __LINE__,                   \
+                    #cond);                     \
+            fprintf(stderr, "" __VA_ARGS__);    \
+            fprintf(stderr, "\n");              \
+            exit(1);                            \
+        }                                       \
+    } while (0)
+
+/*! CHECK_ZSTD
+ * Check the zstd error code and die if an error occurred after printing a
+ * message.
+ */
+#define CHECK_ZSTD(fn, ...)                                      \
+    do {                                                         \
+        size_t const err = (fn);                                 \
+        CHECK(!ZSTD_isError(err), "%s", ZSTD_getErrorName(err)); \
+    } while (0)
 
 /*! fsize_orDie() :
  * Get the size of a given file path.
@@ -154,7 +180,7 @@ static void* malloc_orDie(size_t size)
 static size_t loadFile_orDie(const char* fileName, void* buffer, size_t bufferSize)
 {
     size_t const fileSize = fsize_orDie(fileName);
-    assert(fileSize <= bufferSize);
+    CHECK(fileSize <= bufferSize, "File too large!");
 
     FILE* const inFile = fopen_orDie(fileName, "rb");
     size_t const readSize = fread(buffer, 1, fileSize, inFile);
@@ -205,31 +231,4 @@ static void saveFile_orDie(const char* fileName, const void* buff, size_t buffSi
     }
 }
 
-/*! CHECK
- * Check that the condition holds. If it doesn't print a message and die.
- */
-#define CHECK(cond, ...)                        \
-    do {                                        \
-        if (!(cond)) {                          \
-            fprintf(stderr,                     \
-                    "%s:%d CHECK(%s) failed: ", \
-                    __FILE__,                   \
-                    __LINE__,                   \
-                    #cond);                     \
-            fprintf(stderr, "" __VA_ARGS__);    \
-            fprintf(stderr, "\n");              \
-            exit(1);                            \
-        }                                       \
-    } while (0)
-
-/*! CHECK_ZSTD
- * Check the zstd error code and die if an error occurred after printing a
- * message.
- */
-#define CHECK_ZSTD(fn, ...)                                      \
-    do {                                                         \
-        size_t const err = (fn);                                 \
-        CHECK(!ZSTD_isError(err), "%s", ZSTD_getErrorName(err)); \
-    } while (0)
-
 #endif
diff --git a/examples/dictionary_compression.c b/examples/dictionary_compression.c
index 3c4a5bd1e02..9efdb785c11 100644
--- a/examples/dictionary_compression.c
+++ b/examples/dictionary_compression.c
@@ -7,13 +7,11 @@
  * in the COPYING file in the root directory of this source tree).
  * You may select, at your option, one of the above-listed licenses.
  */
-#include     // malloc, exit
 #include      // printf
-#include     // strerror
-#include      // errno
-#include   // stat
+#include     // free
+#include     // memset, strcat
 #include       // presumes zstd library is installed
-#include "utils.h"
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
 
 /* createDict() :
    `dictFileName` is supposed to have been created using `zstd --train` */
@@ -23,10 +21,7 @@ static ZSTD_CDict* createCDict_orDie(const char* dictFileName, int cLevel)
     printf("loading dictionary %s \n", dictFileName);
     void* const dictBuffer = mallocAndLoadFile_orDie(dictFileName, &dictSize);
     ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, cLevel);
-    if (!cdict) {
-        fprintf(stderr, "ZSTD_createCDict error \n");
-        exit(7);
-    }
+    CHECK(cdict != NULL, "ZSTD_createCDict() failed!");
     free(dictBuffer);
     return cdict;
 }
@@ -39,13 +34,16 @@ static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdi
     size_t const cBuffSize = ZSTD_compressBound(fSize);
     void* const cBuff = malloc_orDie(cBuffSize);
 
+    /* Compress using the dictionary.
+     * This function writes the dictionary id, and content size into the header.
+     * But, it doesn't use a checksum. You can control these options using the
+     * advanced API: ZSTD_CCtx_setParameter(), ZSTD_CCtx_refCDict(),
+     * and ZSTD_compress2().
+     */
     ZSTD_CCtx* const cctx = ZSTD_createCCtx();
-    if (cctx==NULL) { fprintf(stderr, "ZSTD_createCCtx() error \n"); exit(10); }
+    CHECK(cctx != NULL, "ZSTD_createCCtx() failed!");
     size_t const cSize = ZSTD_compress_usingCDict(cctx, cBuff, cBuffSize, fBuff, fSize, cdict);
-    if (ZSTD_isError(cSize)) {
-        fprintf(stderr, "error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize));
-        exit(7);
-    }
+    CHECK_ZSTD(cSize);
 
     saveFile_orDie(oname, cBuff, cSize);
 
diff --git a/examples/dictionary_decompression.c b/examples/dictionary_decompression.c
index 243e22236fd..f683bbb4380 100644
--- a/examples/dictionary_decompression.c
+++ b/examples/dictionary_decompression.c
@@ -9,15 +9,10 @@
  */
 
 
-
-#include     // malloc, exit
 #include      // printf
-#include     // strerror
-#include      // errno
-#include   // stat
-#define ZSTD_STATIC_LINKING_ONLY   // ZSTD_findDecompressedSize
+#include     // free
 #include       // presumes zstd library is installed
-#include "utils.h"
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
 
 /* createDict() :
    `dictFileName` is supposed to have been created using `zstd --train` */
@@ -27,7 +22,7 @@ static ZSTD_DDict* createDict_orDie(const char* dictFileName)
     printf("loading dictionary %s \n", dictFileName);
     void* const dictBuffer = mallocAndLoadFile_orDie(dictFileName, &dictSize);
     ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictSize);
-    if (ddict==NULL) { fprintf(stderr, "ZSTD_createDDict error \n"); exit(5); }
+    CHECK(ddict != NULL, "ZSTD_createDDict() failed!");
     free(dictBuffer);
     return ddict;
 }
@@ -36,24 +31,40 @@ static void decompress(const char* fname, const ZSTD_DDict* ddict)
 {
     size_t cSize;
     void* const cBuff = mallocAndLoadFile_orDie(fname, &cSize);
-    unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize);
-    if (rSize==ZSTD_CONTENTSIZE_ERROR) {
-        fprintf(stderr, "%s : it was not compressed by zstd.\n", fname);
-        exit(5);
-    } else if (rSize==ZSTD_CONTENTSIZE_UNKNOWN) {
-        fprintf(stderr, "%s : original size unknown \n", fname);
-        exit(6);
-    }
-
+    /* Read the content size from the frame header. For simplicity we require
+     * that it is always present. By default, zstd will write the content size
+     * in the header when it is known. If you can't guarantee that the frame
+     * content size is always written into the header, either use streaming
+     * decompression, or ZSTD_decompressBound().
+     */
+    unsigned long long const rSize = ZSTD_getFrameContentSize(cBuff, cSize);
+    CHECK(rSize != ZSTD_CONTENTSIZE_ERROR, "%s: not compressed by zstd!", fname);
+    CHECK(rSize != ZSTD_CONTENTSIZE_UNKNOWN, "%s: original size unknown!", fname);
     void* const rBuff = malloc_orDie((size_t)rSize);
 
+    /* Check that the dictionary ID matches.
+     * If a non-zstd dictionary is used, then both will be zero.
+     * By default zstd always writes the dictionary ID into the frame.
+     * Zstd will check if there is a dictionary ID mismatch as well.
+     */
+    unsigned const expectedDictID = ZSTD_getDictID_fromDDict(ddict);
+    unsigned const actualDictID = ZSTD_getDictID_fromFrame(cBuff, cSize);
+    CHECK(actualDictID == expectedDictID,
+          "DictID mismatch: expected %u got %u",
+          expectedDictID,
+          actualDictID);
+
+    /* Decompress using the dictionary.
+     * If you need to control the decompression parameters, then use the
+     * advanced API: ZSTD_DCtx_setParameter(), ZSTD_DCtx_refDDict(), and
+     * ZSTD_decompressDCtx().
+     */
     ZSTD_DCtx* const dctx = ZSTD_createDCtx();
-    if (dctx==NULL) { fprintf(stderr, "ZSTD_createDCtx() error \n"); exit(10); }
+    CHECK(dctx != NULL, "ZSTD_createDCtx() failed!");
     size_t const dSize = ZSTD_decompress_usingDDict(dctx, rBuff, rSize, cBuff, cSize, ddict);
-    if (dSize != rSize) {
-        fprintf(stderr, "error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize));
-        exit(7);
-    }
+    CHECK_ZSTD(dSize);
+    /* When zstd knows the content size, it will error if it doesn't match. */
+    CHECK(dSize == rSize, "Impossible because zstd will check this condition!");
 
     /* success */
     printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize);
diff --git a/examples/multiple_simple_compression.c b/examples/multiple_simple_compression.c
index b9bb29a9de4..51c9ec725d3 100644
--- a/examples/multiple_simple_compression.c
+++ b/examples/multiple_simple_compression.c
@@ -8,13 +8,11 @@
  * You may select, at your option, one of the above-listed licenses.
  */
 
-#include     // malloc, free, exit
-#include      // fprintf, perror, fopen, etc.
-#include     // strlen, strcat, memset, strerror
-#include      // errno
-#include   // stat
+#include      // printf
+#include     // free
+#include     // memcpy, strlen
 #include       // presumes zstd library is installed
-#include "utils.h"
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
 
 typedef struct {
     void* fBuffer;
@@ -52,7 +50,7 @@ static resources createResources_orDie(int argc, const char** argv, char **ofn,
     ress.fBuffer = malloc_orDie(ress.fBufferSize);
     ress.cBuffer = malloc_orDie(ress.cBufferSize);
     ress.cctx = ZSTD_createCCtx();
-    if (ress.cctx==NULL) { fprintf(stderr, "ZSTD_createCCtx() error \n"); exit(10); }
+    CHECK(ress.cctx != NULL, "ZSTD_createCCtx() failed!");
     return ress;
 }
 
@@ -69,16 +67,17 @@ static void compressFile_orDie(resources ress, const char* fname, const char* on
 {
     size_t fSize = loadFile_orDie(fname, ress.fBuffer, ress.fBufferSize);
 
+    /* Compress using the context.
+     * If you need more control over parameters, use the advanced API:
+     * ZSTD_CCtx_setParameter(), and ZSTD_compress2().
+     */
     size_t const cSize = ZSTD_compressCCtx(ress.cctx, ress.cBuffer, ress.cBufferSize, ress.fBuffer, fSize, 1);
-    if (ZSTD_isError(cSize)) {
-        fprintf(stderr, "error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize));
-        exit(8);
-    }
+    CHECK_ZSTD(cSize);
 
     saveFile_orDie(oname, ress.cBuffer, cSize);
 
     /* success */
-    // printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname);
+    printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname);
 }
 
 int main(int argc, const char** argv)
@@ -102,7 +101,7 @@ int main(int argc, const char** argv)
     for (argNb = 1; argNb < argc; argNb++) {
         const char* const inFilename = argv[argNb];
         size_t const inFilenameLen = strlen(inFilename);
-        assert(inFilenameLen + 5 <= outFilenameBufferLen);
+        CHECK(inFilenameLen + 5 <= outFilenameBufferLen, "File name too long!");
         memcpy(outFilename, inFilename, inFilenameLen);
         memcpy(outFilename+inFilenameLen, ".zst", 5);
         compressFile_orDie(ress, inFilename, outFilename);
diff --git a/examples/multiple_streaming_compression.c b/examples/multiple_streaming_compression.c
index 0bfb337ca1c..ad98b1bd1b0 100644
--- a/examples/multiple_streaming_compression.c
+++ b/examples/multiple_streaming_compression.c
@@ -14,13 +14,11 @@
 *  All structures and buffers will be created only once,
 *  and shared across all compression operations */
 
-#include     // malloc, exit
-#include      // fprintf, perror, feof
-#include     // strerror
-#include      // errno
-#define ZSTD_STATIC_LINKING_ONLY  // TODO: Remove once the API is stable
+#include      // printf
+#include     // free
+#include     // memset, strcat
 #include       // presumes zstd library is installed
-#include "utils.h"
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
 
 typedef struct {
     void* buffIn;
@@ -86,7 +84,8 @@ static void compressFile_orDie(resources ress, const char* fname, const char* ou
             fwrite_orDie(ress.buffOut, output.pos, fout);
             finished = lastChunk ? (remaining == 0) : (input.pos == input.size);
         } while (!finished);
-        assert(input.pos == input.size);
+        CHECK(input.pos == input.size,
+              "Impossible: zstd only returns 0 when the input is completely consumed!");
     }
 
     fclose_orDie(fout);
diff --git a/examples/simple_compression.c b/examples/simple_compression.c
index 829cbd7d8a5..019a143d4c8 100644
--- a/examples/simple_compression.c
+++ b/examples/simple_compression.c
@@ -8,13 +8,11 @@
  * You may select, at your option, one of the above-listed licenses.
  */
 
-#include     // malloc, free, exit
-#include      // fprintf, perror, fopen, etc.
-#include     // strlen, strcat, memset, strerror
-#include      // errno
-#include   // stat
+#include      // printf
+#include     // free
+#include     // strlen, strcat, memset
 #include       // presumes zstd library is installed
-#include "utils.h"
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
 
 static void compress_orDie(const char* fname, const char* oname)
 {
@@ -23,11 +21,12 @@ static void compress_orDie(const char* fname, const char* oname)
     size_t const cBuffSize = ZSTD_compressBound(fSize);
     void* const cBuff = malloc_orDie(cBuffSize);
 
+    /* Compress.
+     * If you are doing many compressions, you may want to reuse the context.
+     * See the multiple_simple_compression.c example.
+     */
     size_t const cSize = ZSTD_compress(cBuff, cBuffSize, fBuff, fSize, 1);
-    if (ZSTD_isError(cSize)) {
-        fprintf(stderr, "error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize));
-        exit(8);
-    }
+    CHECK_ZSTD(cSize);
 
     saveFile_orDie(oname, cBuff, cSize);
 
diff --git a/examples/simple_decompression.c b/examples/simple_decompression.c
index e7949376cbc..1aa57c7b093 100644
--- a/examples/simple_decompression.c
+++ b/examples/simple_decompression.c
@@ -8,37 +8,36 @@
  * You may select, at your option, one of the above-listed licenses.
  */
 
-#include     // malloc, exit
 #include      // printf
-#include     // strerror
-#include      // errno
-#include   // stat
-#define ZSTD_STATIC_LINKING_ONLY   // ZSTD_findDecompressedSize
+#include     // free
 #include       // presumes zstd library is installed
-#include "utils.h"
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
 
 static void decompress(const char* fname)
 {
     size_t cSize;
     void* const cBuff = mallocAndLoadFile_orDie(fname, &cSize);
-    unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize);
-    if (rSize==ZSTD_CONTENTSIZE_ERROR) {
-        fprintf(stderr, "%s : it was not compressed by zstd.\n", fname);
-        exit(5);
-    } else if (rSize==ZSTD_CONTENTSIZE_UNKNOWN) {
-        fprintf(stderr,
-                "%s : original size unknown. Use streaming decompression instead.\n", fname);
-        exit(6);
-    }
+    /* Read the content size from the frame header. For simplicity we require
+     * that it is always present. By default, zstd will write the content size
+     * in the header when it is known. If you can't guarantee that the frame
+     * content size is always written into the header, either use streaming
+     * decompression, or ZSTD_decompressBound().
+     */
+    unsigned long long const rSize = ZSTD_getFrameContentSize(cBuff, cSize);
+    CHECK(rSize != ZSTD_CONTENTSIZE_ERROR, "%s: not compressed by zstd!", fname);
+    CHECK(rSize != ZSTD_CONTENTSIZE_UNKNOWN, "%s: original size unknown!", fname);
 
     void* const rBuff = malloc_orDie((size_t)rSize);
 
+    /* Decompress.
+     * If you are doing many decompressions, you may want to reuse the context
+     * and use ZSTD_decompressDCtx(). If you want to set advanced parameters,
+     * use ZSTD_DCtx_setParameter().
+     */
     size_t const dSize = ZSTD_decompress(rBuff, rSize, cBuff, cSize);
-
-    if (dSize != rSize) {
-        fprintf(stderr, "error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize));
-        exit(7);
-    }
+    CHECK_ZSTD(dSize);
+    /* When zstd knows the content size, it will error if it doesn't match. */
+    CHECK(dSize == rSize, "Impossible because zstd will check this condition!");
 
     /* success */
     printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize);
diff --git a/examples/streaming_compression.c b/examples/streaming_compression.c
index 345b60ed1a0..d1353a684a6 100644
--- a/examples/streaming_compression.c
+++ b/examples/streaming_compression.c
@@ -9,12 +9,11 @@
  */
 
 
-#include     // malloc, free, exit
-#include      // fprintf, perror, feof, fopen, etc.
-#include     // strlen, memset, strcat
-#define ZSTD_STATIC_LINKING_ONLY // TODO: Remove once the API is stable
+#include      // printf
+#include     // free
+#include     // memset, strcat, strlen
 #include       // presumes zstd library is installed
-#include "utils.h"
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
 
 
 static void compressFile_orDie(const char* fname, const char* outName, int cLevel)
@@ -75,7 +74,8 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve
              */
             finished = lastChunk ? (remaining == 0) : (input.pos == input.size);
         } while (!finished);
-        assert(input.pos == input.size);
+        CHECK(input.pos == input.size,
+              "Impossible: zstd only returns 0 when the input is completely consumed!");
     }
 
     ZSTD_freeCCtx(cctx);
diff --git a/examples/streaming_decompression.c b/examples/streaming_decompression.c
index 5db5e7a73e3..bcd861b756c 100644
--- a/examples/streaming_decompression.c
+++ b/examples/streaming_decompression.c
@@ -9,12 +9,10 @@
  */
 
 
-#include     // malloc, exit
-#include      // fprintf, perror, feof
-#include     // strerror
-#include      // errno
+#include      // fprintf
+#include     // free
 #include       // presumes zstd library is installed
-#include "utils.h"
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
 
 static void decompressFile_orDie(const char* fname)
 {
diff --git a/examples/streaming_memory_usage.c b/examples/streaming_memory_usage.c
index c31d9f9f788..26835788abe 100644
--- a/examples/streaming_memory_usage.c
+++ b/examples/streaming_memory_usage.c
@@ -16,10 +16,10 @@
 
 
 /*===   Dependencies   ===*/
-#include    /* printf */
+#include      // printf
 #define ZSTD_STATIC_LINKING_ONLY
-#include "zstd.h"
-#include "utils.h"
+#include       // presumes zstd library is installed
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
 
 
 /*===   functions   ===*/

From 641e5943093a776ee4bbe58a28aef472025a03e7 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Sun, 7 Apr 2019 18:47:52 -0700
Subject: [PATCH 135/178] [libzstd] Remove ZSTDMT from the shared object

* Remove ZSTDMT from the shared object by default.
* Provide a macro `ZSTD_LEGACY_MULTITHREADED_API` to override it.
* Document it in `lib/README.md`.
---
 lib/Makefile                   |  5 +++
 lib/README.md                  |  6 ++--
 lib/compress/zstdmt_compress.h | 59 ++++++++++++++++++++--------------
 3 files changed, 44 insertions(+), 26 deletions(-)

diff --git a/lib/Makefile b/lib/Makefile
index f9955ba4f34..404f5b69210 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -56,6 +56,7 @@ ZSTD_FORCE_DECOMPRESS_SHORT ?= 0
 ZSTD_FORCE_DECOMPRESS_LONG ?= 0
 ZSTD_NO_INLINE ?= 0
 ZSTD_STRIP_ERROR_STRINGS ?= 0
+ZSTD_LEGACY_MULTITHREADED_API ?= 0
 
 ifeq ($(ZSTD_LIB_COMPRESSION), 0)
 	ZSTD_LIB_DICTBUILDER = 0
@@ -107,6 +108,10 @@ ifneq ($(ZSTD_STRIP_ERROR_STRINGS), 0)
 	CFLAGS += -DZSTD_STRIP_ERROR_STRINGS
 endif
 
+ifneq ($(ZSTD_LEGACY_MULTITHREADED_API), 0)
+	CFLAGS += -DZSTD_LEGACY_MULTITHREADED_API
+endif
+
 ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
 ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
 	ZSTD_FILES += $(shell ls legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
diff --git a/lib/README.md b/lib/README.md
index 0c9cd6d58f3..d1dc6378a0b 100644
--- a/lib/README.md
+++ b/lib/README.md
@@ -31,8 +31,6 @@ note that it's necessary to request the `-pthread` flag during link stage.
 
 Multithreading capabilities are exposed
 via the [advanced API defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/v1.3.8/lib/zstd.h#L592).
-This API is still labelled experimental,
-but is expected to become "stable" in the near future.
 
 
 #### API
@@ -110,6 +108,10 @@ The file structure is designed to make this selection manually achievable for an
   which removes the error messages that are otherwise returned by
   `ZSTD_getErrorName`.
 
+- While invoking `make libzstd`, the build macro `ZSTD_LEGACY_MULTITHREADED_API=1`
+  will expose the deprecated the `ZSTDMT` API exposed by `zstdmt_compress.h` in
+  the shared library, which is now hidden by default.
+
 
 #### Windows : using MinGW+MSYS to create DLL
 
diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h
index ee771681fa5..bae830e280b 100644
--- a/lib/compress/zstdmt_compress.h
+++ b/lib/compress/zstdmt_compress.h
@@ -17,10 +17,21 @@
 
 
 /* Note : This is an internal API.
- *        Some methods are still exposed (ZSTDLIB_API),
+ *        These APIs used to be exposed with ZSTDLIB_API,
  *        because it used to be the only way to invoke MT compression.
  *        Now, it's recommended to use ZSTD_compress_generic() instead.
- *        These methods will stop being exposed in a future version */
+ *
+ *        If you depend on these APIs and can't switch, then define
+ *        ZSTD_LEGACY_MULTITHREADED_API when making the dynamic library.
+ *        However, we may completely remove these functions in a future
+ *        release, so please switch soon.
+ */
+
+#ifdef ZSTD_LEGACY_MULTITHREADED_API
+#  define ZSTDMT_API ZSTDLIB_API
+#else
+#  define ZSTDMT_API
+#endif
 
 /* ===   Dependencies   === */
 #include                 /* size_t */
@@ -40,17 +51,17 @@
 
 /* ===   Memory management   === */
 typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx;
-ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbWorkers);
-ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers,
+ZSTDMT_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbWorkers);
+ZSTDMT_API ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers,
                                                     ZSTD_customMem cMem);
-ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx);
+ZSTDMT_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx);
 
-ZSTDLIB_API size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx);
+ZSTDMT_API size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx);
 
 
 /* ===   Simple one-pass compression function   === */
 
-ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
+ZSTDMT_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
                                        void* dst, size_t dstCapacity,
                                  const void* src, size_t srcSize,
                                        int compressionLevel);
@@ -59,31 +70,31 @@ ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
 
 /* ===   Streaming functions   === */
 
-ZSTDLIB_API size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel);
-ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize);  /**< if srcSize is not known at reset time, use ZSTD_CONTENTSIZE_UNKNOWN. Note: for compatibility with older programs, 0 means the same as ZSTD_CONTENTSIZE_UNKNOWN, but it will change in the future to mean "empty" */
+ZSTDMT_API size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel);
+ZSTDMT_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize);  /**< if srcSize is not known at reset time, use ZSTD_CONTENTSIZE_UNKNOWN. Note: for compatibility with older programs, 0 means the same as ZSTD_CONTENTSIZE_UNKNOWN, but it will change in the future to mean "empty" */
 
-ZSTDLIB_API size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx);
-ZSTDLIB_API size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+ZSTDMT_API size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx);
+ZSTDMT_API size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
 
-ZSTDLIB_API size_t ZSTDMT_flushStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output);   /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */
-ZSTDLIB_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output);     /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */
+ZSTDMT_API size_t ZSTDMT_flushStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output);   /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */
+ZSTDMT_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output);     /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */
 
 
 /* ===   Advanced functions and parameters  === */
 
-ZSTDLIB_API size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
-                                           void* dst, size_t dstCapacity,
-                                     const void* src, size_t srcSize,
-                                     const ZSTD_CDict* cdict,
-                                           ZSTD_parameters params,
-                                           int overlapLog);
+ZSTDMT_API size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
+                                          void* dst, size_t dstCapacity,
+                                    const void* src, size_t srcSize,
+                                    const ZSTD_CDict* cdict,
+                                          ZSTD_parameters params,
+                                          int overlapLog);
 
-ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx,
+ZSTDMT_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx,
                                         const void* dict, size_t dictSize,   /* dict can be released after init, a local copy is preserved within zcs */
                                         ZSTD_parameters params,
                                         unsigned long long pledgedSrcSize);  /* pledgedSrcSize is optional and can be zero == unknown */
 
-ZSTDLIB_API size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,
+ZSTDMT_API size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,
                                         const ZSTD_CDict* cdict,
                                         ZSTD_frameParameters fparams,
                                         unsigned long long pledgedSrcSize);  /* note : zero means empty */
@@ -101,12 +112,12 @@ typedef enum {
  * The function must be called typically after ZSTD_createCCtx() but __before ZSTDMT_init*() !__
  * Parameters not explicitly reset by ZSTDMT_init*() remain the same in consecutive compression sessions.
  * @return : 0, or an error code (which can be tested using ZSTD_isError()) */
-ZSTDLIB_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int value);
+ZSTDMT_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int value);
 
 /* ZSTDMT_getMTCtxParameter() :
  * Query the ZSTDMT_CCtx for a parameter value.
  * @return : 0, or an error code (which can be tested using ZSTD_isError()) */
-ZSTDLIB_API size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int* value);
+ZSTDMT_API size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int* value);
 
 
 /*! ZSTDMT_compressStream_generic() :
@@ -116,7 +127,7 @@ ZSTDLIB_API size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter
  *           0 if fully flushed
  *           or an error code
  *  note : needs to be init using any ZSTD_initCStream*() variant */
-ZSTDLIB_API size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
+ZSTDMT_API size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
                                                 ZSTD_outBuffer* output,
                                                 ZSTD_inBuffer* input,
                                                 ZSTD_EndDirective endOp);

From 947548c24f5ac9456fc41a4883bfeffbd6211614 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Mon, 8 Apr 2019 16:50:18 -0700
Subject: [PATCH 136/178] Remove double the from README

---
 lib/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/README.md b/lib/README.md
index d1dc6378a0b..792729b1f9b 100644
--- a/lib/README.md
+++ b/lib/README.md
@@ -109,7 +109,7 @@ The file structure is designed to make this selection manually achievable for an
   `ZSTD_getErrorName`.
 
 - While invoking `make libzstd`, the build macro `ZSTD_LEGACY_MULTITHREADED_API=1`
-  will expose the deprecated the `ZSTDMT` API exposed by `zstdmt_compress.h` in
+  will expose the deprecated `ZSTDMT` API exposed by `zstdmt_compress.h` in
   the shared library, which is now hidden by default.
 
 

From bfcd5b81d7a1f3612503e0e6ef0c6fe275e7017a Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Mon, 8 Apr 2019 19:57:41 -0700
Subject: [PATCH 137/178] [libzstd] Don't check the dictID in fuzzing mode

When `FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION` is defined don't check
the dictID. This check makes the fuzzers job harder, and it is at the
very beginning.
---
 lib/decompress/zstd_decompress.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index aa7f6f58042..14cc12a412f 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -427,8 +427,13 @@ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t he
     size_t const result = ZSTD_getFrameHeader_advanced(&(dctx->fParams), src, headerSize, dctx->format);
     if (ZSTD_isError(result)) return result;    /* invalid header */
     RETURN_ERROR_IF(result>0, srcSize_wrong, "headerSize too small");
+#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    /* Skip the dictID check in fuzzing mode, because it makes the search
+     * harder.
+     */
     RETURN_ERROR_IF(dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID),
                     dictionary_wrong);
+#endif
     if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0);
     return 0;
 }

From e649fad7aacebb4df62257fa430f301b4a00ccdd Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Mon, 8 Apr 2019 20:00:18 -0700
Subject: [PATCH 138/178] [dictBuilder] Fix displayLevel for corpus warning

Pass the displaylevel into the corpus warning, because it is used in
fast cover and cover, so it needs to respect the local level.
---
 lib/dictBuilder/cover.c     | 21 +++++++++++----------
 lib/dictBuilder/cover.h     |  2 +-
 lib/dictBuilder/fastcover.c |  4 ++--
 3 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c
index 180a68ae8c6..ed5a02f9849 100644
--- a/lib/dictBuilder/cover.c
+++ b/lib/dictBuilder/cover.c
@@ -627,19 +627,20 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
   return 1;
 }
 
-void COVER_warnOnSmallCorpus(size_t maxDictSize, size_t nbDmers)
+void COVER_warnOnSmallCorpus(size_t maxDictSize, size_t nbDmers, int displayLevel)
 {
   const double ratio = (double)nbDmers / maxDictSize;
   if (ratio >= 10) {
       return;
   }
-  DISPLAYLEVEL(1, "WARNING: The maximum dictionary size %u is too large "
-                  "compared to the source size %u! "
-                  "size(source)/size(dictionary) = %f, but it should be >= "
-                  "10! This may lead to a subpar dictionary! We recommend "
-                  "training on sources at least 10x, and up to 100x the "
-                  "size of the dictionary!\n", (U32)maxDictSize,
-                  (U32)nbDmers, ratio);
+  LOCALDISPLAYLEVEL(displayLevel, 1,
+                    "WARNING: The maximum dictionary size %u is too large "
+                    "compared to the source size %u! "
+                    "size(source)/size(dictionary) = %f, but it should be >= "
+                    "10! This may lead to a subpar dictionary! We recommend "
+                    "training on sources at least 10x, and up to 100x the "
+                    "size of the dictionary!\n", (U32)maxDictSize,
+                    (U32)nbDmers, ratio);
 }
 
 COVER_epoch_info_t COVER_computeEpochs(U32 maxDictSize,
@@ -744,7 +745,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
                       parameters.d, parameters.splitPoint)) {
     return ERROR(GENERIC);
   }
-  COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.suffixSize);
+  COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.suffixSize, g_displayLevel);
   if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) {
     DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n");
     COVER_ctx_destroy(&ctx);
@@ -1060,7 +1061,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
       return ERROR(GENERIC);
     }
     if (!warned) {
-      COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.suffixSize);
+      COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.suffixSize, displayLevel);
       warned = 1;
     }
     /* Loop through k reusing the same context */
diff --git a/lib/dictBuilder/cover.h b/lib/dictBuilder/cover.h
index 71c520e9e62..27e6fb7a3cc 100644
--- a/lib/dictBuilder/cover.h
+++ b/lib/dictBuilder/cover.h
@@ -65,7 +65,7 @@ COVER_epoch_info_t COVER_computeEpochs(U32 maxDictSize, U32 nbDmers,
 /**
  * Warns the user when their corpus is too small.
  */
-void COVER_warnOnSmallCorpus(size_t maxDictSize, size_t nbDmers);
+void COVER_warnOnSmallCorpus(size_t maxDictSize, size_t nbDmers, int displayLevel);
 
 /**
  *  Checks total compressed size of a dictionary
diff --git a/lib/dictBuilder/fastcover.c b/lib/dictBuilder/fastcover.c
index 8cb89c93886..6cf37026f44 100644
--- a/lib/dictBuilder/fastcover.c
+++ b/lib/dictBuilder/fastcover.c
@@ -570,7 +570,7 @@ ZDICT_trainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity,
       DISPLAYLEVEL(1, "Failed to initialize context\n");
       return ERROR(GENERIC);
     }
-    COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.nbDmers);
+    COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.nbDmers, g_displayLevel);
     /* Build the dictionary */
     DISPLAYLEVEL(2, "Building dictionary\n");
     {
@@ -673,7 +673,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
         return ERROR(GENERIC);
       }
       if (!warned) {
-        COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.nbDmers);
+        COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.nbDmers, displayLevel);
         warned = 1;
       }
       /* Loop through k reusing the same context */

From f871b5144e1c3ac7b628fdece1032c22cb0dabd8 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Mon, 8 Apr 2019 20:01:38 -0700
Subject: [PATCH 139/178] [fuzz] Use the new advanced API

---
 tests/fuzz/simple_round_trip.c | 10 +---------
 tests/fuzz/stream_decompress.c |  4 +---
 2 files changed, 2 insertions(+), 12 deletions(-)

diff --git a/tests/fuzz/simple_round_trip.c b/tests/fuzz/simple_round_trip.c
index 83608b6e79e..21ab32c7f8d 100644
--- a/tests/fuzz/simple_round_trip.c
+++ b/tests/fuzz/simple_round_trip.c
@@ -36,16 +36,8 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
 {
     size_t cSize;
     if (FUZZ_rand(&seed) & 1) {
-        ZSTD_inBuffer in = {src, srcSize, 0};
-        ZSTD_outBuffer out = {compressed, compressedCapacity, 0};
-        size_t err;
-
-        ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
         FUZZ_setRandomParameters(cctx, srcSize, &seed);
-        err = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
-        FUZZ_ZASSERT(err);
-        FUZZ_ASSERT(err == 0);
-        cSize = out.pos;
+        cSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize);
     } else {
         int const cLevel = FUZZ_rand(&seed) % kMaxClevel;
         cSize = ZSTD_compressCCtx(
diff --git a/tests/fuzz/stream_decompress.c b/tests/fuzz/stream_decompress.c
index 7ad571221df..68e120d7ef6 100644
--- a/tests/fuzz/stream_decompress.c
+++ b/tests/fuzz/stream_decompress.c
@@ -62,9 +62,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
     if (!dstream) {
         dstream = ZSTD_createDStream();
         FUZZ_ASSERT(dstream);
-        FUZZ_ASSERT(!ZSTD_isError(ZSTD_initDStream(dstream)));
     } else {
-        FUZZ_ASSERT(!ZSTD_isError(ZSTD_resetDStream(dstream)));
+        FUZZ_ZASSERT(ZSTD_DCtx_reset(dstream, ZSTD_reset_session_only));
     }
 
     while (size > 0) {
@@ -73,7 +72,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
             ZSTD_outBuffer out = makeOutBuffer();
             size_t const rc = ZSTD_decompressStream(dstream, &out, &in);
             if (ZSTD_isError(rc)) goto error;
-            if (rc == 0) FUZZ_ASSERT(!ZSTD_isError(ZSTD_resetDStream(dstream)));
         }
     }
 

From 462918560c0234ce69637f73ddc5193bad9515f1 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Mon, 8 Apr 2019 21:06:19 -0700
Subject: [PATCH 140/178] [fuzzer] Fix stream_round_trip for the new options

---
 tests/fuzz/stream_round_trip.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/fuzz/stream_round_trip.c b/tests/fuzz/stream_round_trip.c
index d903bcb29b1..d13c2dbe7e0 100644
--- a/tests/fuzz/stream_round_trip.c
+++ b/tests/fuzz/stream_round_trip.c
@@ -63,7 +63,7 @@ static size_t compress(uint8_t *dst, size_t capacity,
         ZSTD_inBuffer in = makeInBuffer(&src, &srcSize);
         /* Mode controls the action. If mode == -1 we pick a new mode */
         int mode = -1;
-        while (in.pos < in.size) {
+        while (in.pos < in.size || mode != -1) {
             ZSTD_outBuffer out = makeOutBuffer(dst, capacity);
             /* Previous action finished, pick a new mode. */
             if (mode == -1) mode = FUZZ_rand(&seed) % 10;

From 7a1fde2957788c07b69772f48d95389118206806 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Mon, 8 Apr 2019 21:07:28 -0700
Subject: [PATCH 141/178] [fuzzer] Add dictionary fuzzers

---
 tests/fuzz/Makefile                | 18 ++++--
 tests/fuzz/dictionary_decompress.c | 57 +++++++++++++++++
 tests/fuzz/dictionary_round_trip.c | 99 ++++++++++++++++++++++++++++++
 tests/fuzz/fuzz.py                 |  2 +
 tests/fuzz/zstd_helpers.c          | 55 ++++++++++++++++-
 tests/fuzz/zstd_helpers.h          | 13 ++++
 6 files changed, 239 insertions(+), 5 deletions(-)
 create mode 100644 tests/fuzz/dictionary_decompress.c
 create mode 100644 tests/fuzz/dictionary_round_trip.c

diff --git a/tests/fuzz/Makefile b/tests/fuzz/Makefile
index 8a22ad1c543..12ec9524bfc 100644
--- a/tests/fuzz/Makefile
+++ b/tests/fuzz/Makefile
@@ -27,7 +27,7 @@ PRGDIR = ../../programs
 
 FUZZ_CPPFLAGS := -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
 	-I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) \
-	$(CPPFLAGS)
+	-DZSTD_MULTITHREAD $(CPPFLAGS)
 FUZZ_EXTRA_FLAGS := -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
 	-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
 	-Wstrict-prototypes -Wundef \
@@ -36,7 +36,7 @@ FUZZ_EXTRA_FLAGS := -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
 	-g -fno-omit-frame-pointer
 FUZZ_CFLAGS := $(FUZZ_EXTRA_FLAGS) $(CFLAGS)
 FUZZ_CXXFLAGS := $(FUZZ_EXTRA_FLAGS) -std=c++11 $(CXXFLAGS)
-FUZZ_LDFLAGS := $(LDFLAGS)
+FUZZ_LDFLAGS := -pthread $(LDFLAGS)
 FUZZ_ARFLAGS := $(ARFLAGS)
 FUZZ_TARGET_FLAGS = $(FUZZ_CPPFLAGS) $(FUZZ_CXXFLAGS) $(FUZZ_LDFLAGS)
 
@@ -46,11 +46,13 @@ FUZZ_SRC := $(PRGDIR)/util.c zstd_helpers.c
 ZSTDCOMMON_SRC := $(ZSTDDIR)/common/*.c
 ZSTDCOMP_SRC   := $(ZSTDDIR)/compress/*.c
 ZSTDDECOMP_SRC := $(ZSTDDIR)/decompress/*.c
+ZSTDDICT_SRC := $(ZSTDDIR)/dictBuilder/*.c
 FUZZ_SRC       := \
 	$(FUZZ_SRC) \
 	$(ZSTDDECOMP_SRC) \
 	$(ZSTDCOMMON_SRC) \
-	$(ZSTDCOMP_SRC)
+	$(ZSTDCOMP_SRC) \
+	$(ZSTDDICT_SRC)
 
 FUZZ_OBJ := $(patsubst %.c,%.o, $(wildcard $(FUZZ_SRC)))
 
@@ -65,7 +67,9 @@ FUZZ_TARGETS :=       \
 	block_round_trip  \
 	simple_decompress \
 	stream_decompress \
-	block_decompress
+	block_decompress  \
+	dictionary_round_trip \
+	dictionary_decompress
 
 all: $(FUZZ_TARGETS)
 
@@ -90,6 +94,12 @@ stream_decompress: $(FUZZ_HEADERS) $(FUZZ_OBJ) stream_decompress.o
 block_decompress: $(FUZZ_HEADERS) $(FUZZ_OBJ) block_decompress.o
 	$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) block_decompress.o $(LIB_FUZZING_ENGINE) -o $@
 
+dictionary_round_trip: $(FUZZ_HEADERS) $(FUZZ_OBJ) dictionary_round_trip.o
+	$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) dictionary_round_trip.o $(LIB_FUZZING_ENGINE) -o $@
+
+dictionary_decompress: $(FUZZ_HEADERS) $(FUZZ_OBJ) dictionary_decompress.o
+	$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) dictionary_decompress.o $(LIB_FUZZING_ENGINE) -o $@
+
 libregression.a: $(FUZZ_HEADERS) $(PRGDIR)/util.h $(PRGDIR)/util.c regression_driver.o
 	$(AR) $(FUZZ_ARFLAGS) $@ regression_driver.o
 
diff --git a/tests/fuzz/dictionary_decompress.c b/tests/fuzz/dictionary_decompress.c
new file mode 100644
index 00000000000..bdecdef7ad3
--- /dev/null
+++ b/tests/fuzz/dictionary_decompress.c
@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+
+/**
+ * This fuzz target attempts to decompress the fuzzed data with the dictionary
+ * decompression function to ensure the decompressor never crashes. It does not
+ * fuzz the dictionary.
+ */
+
+#include 
+#include 
+#include 
+#include "fuzz_helpers.h"
+#include "zstd_helpers.h"
+
+static ZSTD_DCtx *dctx = NULL;
+static void* rBuf = NULL;
+static size_t bufSize = 0;
+
+int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
+{
+    size_t neededBufSize;
+
+    uint32_t seed = FUZZ_seed(&src, &size);
+    neededBufSize = MAX(20 * size, (size_t)256 << 10);
+
+    /* Allocate all buffers and contexts if not already allocated */
+    if (neededBufSize > bufSize) {
+        free(rBuf);
+        rBuf = malloc(neededBufSize);
+        bufSize = neededBufSize;
+        FUZZ_ASSERT(rBuf);
+    }
+    if (!dctx) {
+        dctx = ZSTD_createDCtx();
+        FUZZ_ASSERT(dctx);
+    }
+    {
+        FUZZ_dict_t dict = FUZZ_train(src, size, &seed);
+        ZSTD_decompress_usingDict(dctx,
+                rBuf, neededBufSize,
+                src, size,
+                dict.buff, dict.size);
+        free(dict.buff);
+    }
+
+#ifndef STATEFUL_FUZZING
+    ZSTD_freeDCtx(dctx); dctx = NULL;
+#endif
+    return 0;
+}
diff --git a/tests/fuzz/dictionary_round_trip.c b/tests/fuzz/dictionary_round_trip.c
new file mode 100644
index 00000000000..57a2eff3bf7
--- /dev/null
+++ b/tests/fuzz/dictionary_round_trip.c
@@ -0,0 +1,99 @@
+/*
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+
+/**
+ * This fuzz target performs a zstd round-trip test (compress & decompress) with
+ * a dictionary, compares the result with the original, and calls abort() on
+ * corruption.
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include "fuzz_helpers.h"
+#include "zstd_helpers.h"
+
+static const int kMaxClevel = 19;
+
+static ZSTD_CCtx *cctx = NULL;
+static ZSTD_DCtx *dctx = NULL;
+static void* cBuf = NULL;
+static void* rBuf = NULL;
+static size_t bufSize = 0;
+static uint32_t seed;
+
+static size_t roundTripTest(void *result, size_t resultCapacity,
+                            void *compressed, size_t compressedCapacity,
+                            const void *src, size_t srcSize)
+{
+    FUZZ_dict_t dict = FUZZ_train(src, srcSize, &seed);
+    size_t cSize;
+    if ((FUZZ_rand(&seed) & 15) == 0) {
+        int const cLevel = FUZZ_rand(&seed) % kMaxClevel;
+
+        cSize = ZSTD_compress_usingDict(cctx,
+                compressed, compressedCapacity,
+                src, srcSize,
+                dict.buff, dict.size,
+                cLevel);
+    } else {
+        FUZZ_setRandomParameters(cctx, srcSize, &seed);
+        FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary(cctx, dict.buff, dict.size));
+        cSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize);
+    }
+    FUZZ_ZASSERT(cSize);
+    {
+        size_t const ret = ZSTD_decompress_usingDict(dctx,
+                result, resultCapacity,
+                compressed, cSize,
+                dict.buff, dict.size);
+        free(dict.buff);
+        return ret;
+    }
+}
+
+int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
+{
+    size_t neededBufSize;
+
+    seed = FUZZ_seed(&src, &size);
+    neededBufSize = ZSTD_compressBound(size);
+
+    /* Allocate all buffers and contexts if not already allocated */
+    if (neededBufSize > bufSize) {
+        free(cBuf);
+        free(rBuf);
+        cBuf = malloc(neededBufSize);
+        rBuf = malloc(neededBufSize);
+        bufSize = neededBufSize;
+        FUZZ_ASSERT(cBuf && rBuf);
+    }
+    if (!cctx) {
+        cctx = ZSTD_createCCtx();
+        FUZZ_ASSERT(cctx);
+    }
+    if (!dctx) {
+        dctx = ZSTD_createDCtx();
+        FUZZ_ASSERT(dctx);
+    }
+
+    {
+        size_t const result =
+            roundTripTest(rBuf, neededBufSize, cBuf, neededBufSize, src, size);
+        FUZZ_ZASSERT(result);
+        FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size");
+        FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!");
+    }
+#ifndef STATEFUL_FUZZING
+    ZSTD_freeCCtx(cctx); cctx = NULL;
+    ZSTD_freeDCtx(dctx); dctx = NULL;
+#endif
+    return 0;
+}
diff --git a/tests/fuzz/fuzz.py b/tests/fuzz/fuzz.py
index 69376298518..ee27015a535 100755
--- a/tests/fuzz/fuzz.py
+++ b/tests/fuzz/fuzz.py
@@ -34,6 +34,8 @@ def abs_join(a, *p):
     'simple_decompress',
     'stream_decompress',
     'block_decompress',
+    'dictionary_round_trip',
+    'dictionary_decompress',
 ]
 ALL_TARGETS = TARGETS + ['all']
 FUZZ_RNG_SEED_SIZE = 4
diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c
index 10163e1512b..0e64400e6bc 100644
--- a/tests/fuzz/zstd_helpers.c
+++ b/tests/fuzz/zstd_helpers.c
@@ -8,10 +8,14 @@
  */
 
 #define ZSTD_STATIC_LINKING_ONLY
+#define ZDICT_STATIC_LINKING_ONLY
+
+#include 
 
 #include "zstd_helpers.h"
 #include "fuzz_helpers.h"
 #include "zstd.h"
+#include "zdict.h"
 
 static void set(ZSTD_CCtx *cctx, ZSTD_cParameter param, int value)
 {
@@ -71,7 +75,6 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state)
     setRand(cctx, ZSTD_c_contentSizeFlag, 0, 1, state);
     setRand(cctx, ZSTD_c_checksumFlag, 0, 1, state);
     setRand(cctx, ZSTD_c_dictIDFlag, 0, 1, state);
-    setRand(cctx, ZSTD_c_forceAttachDict, 0, 2, state);
     /* Select long distance matchig parameters */
     setRand(cctx, ZSTD_c_enableLongDistanceMatching, 0, 1, state);
     setRand(cctx, ZSTD_c_ldmHashLog, ZSTD_HASHLOG_MIN, 16, state);
@@ -81,4 +84,54 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state)
             state);
     setRand(cctx, ZSTD_c_ldmHashRateLog, ZSTD_LDM_HASHRATELOG_MIN,
             ZSTD_LDM_HASHRATELOG_MAX, state);
+    /* Set misc parameters */
+    setRand(cctx, ZSTD_c_nbWorkers, 0, 2, state);
+    setRand(cctx, ZSTD_c_rsyncable, 0, 1, state);
+    setRand(cctx, ZSTD_c_forceMaxWindow, 0, 1, state);
+    setRand(cctx, ZSTD_c_literalCompressionMode, 0, 2, state);
+    setRand(cctx, ZSTD_c_forceAttachDict, 0, 2, state);
+}
+
+FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, uint32_t *state)
+{
+    size_t const dictSize = MAX(srcSize / 8, 1024);
+    size_t const totalSampleSize = dictSize * 11;
+    FUZZ_dict_t dict = { malloc(dictSize), dictSize };
+    char* const samples = (char*)malloc(totalSampleSize);
+    unsigned nbSamples = 100;
+    size_t* const samplesSizes = (size_t*)malloc(sizeof(size_t) * nbSamples);
+    size_t pos = 0;
+    size_t sample = 0;
+    ZDICT_fastCover_params_t params;
+    FUZZ_ASSERT(dict.buff && samples && samplesSizes);
+
+    for (sample = 0; sample < nbSamples; ++sample) {
+      size_t const remaining = totalSampleSize - pos;
+      size_t const offset = FUZZ_rand32(state, 0, MAX(srcSize, 1) - 1);
+      size_t const limit = MIN(srcSize - offset, remaining);
+      size_t const toCopy = MIN(limit, remaining / (nbSamples - sample));
+      memcpy(samples + pos, src + offset, toCopy);
+      pos += toCopy;
+      samplesSizes[sample] = toCopy;
+
+    }
+    memset(samples + pos, 0, totalSampleSize - pos);
+
+    memset(¶ms, 0, sizeof(params));
+    params.accel = 5;
+    params.k = 40;
+    params.d = 8;
+    params.f = 14;
+    params.zParams.compressionLevel = 1;
+    dict.size = ZDICT_trainFromBuffer_fastCover(dict.buff, dictSize,
+        samples, samplesSizes, nbSamples, params);
+    if (ZSTD_isError(dict.size)) {
+        free(dict.buff);
+        memset(&dict, 0, sizeof(dict));
+    }
+
+    free(samplesSizes);
+    free(samples);
+
+    return dict;
 }
diff --git a/tests/fuzz/zstd_helpers.h b/tests/fuzz/zstd_helpers.h
index 3856bebecf7..457e6e995f0 100644
--- a/tests/fuzz/zstd_helpers.h
+++ b/tests/fuzz/zstd_helpers.h
@@ -14,6 +14,8 @@
 #ifndef ZSTD_HELPERS_H
 #define ZSTD_HELPERS_H
 
+#define ZSTD_STATIC_LINKING_ONLY
+
 #include "zstd.h"
 #include 
 
@@ -27,6 +29,17 @@ ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, uint32_t *state);
 ZSTD_frameParameters FUZZ_randomFParams(uint32_t *state);
 ZSTD_parameters FUZZ_randomParams(size_t srcSize, uint32_t *state);
 
+typedef struct {
+  void* buff;
+  size_t size;
+} FUZZ_dict_t;
+
+/* Quickly train a dictionary from a source for fuzzing.
+ * NOTE: Don't use this to train production dictionaries, it is only optimized
+ * for speed, and doesn't care about dictionary quality.
+ */
+FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, uint32_t *state);
+
 
 #ifdef __cplusplus
 }

From 48a6427d22f290157b8acc3f7c03c0f762a768be Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Tue, 9 Apr 2019 16:24:17 -0700
Subject: [PATCH 142/178] [libzstd] Fix ZSTD_compress2() for multithreaded
 compression

`ZSTD_compress2()` wouldn't wait for multithreaded compression to
finish. We didn't find this because ZSTDMT will block when it can
compress all in one go, but it can't do that if it doesn't have enough
output space, or if `ZSTD_c_rsyncable` is enabled.

Since we will already sometimes block when using `ZSTD_e_end`, I've
changed `ZSTD_e_end` and `ZSTD_e_flush` to guarantee maximum forward
progress. This simplifies the API, and helps users avoid the easy bug
that was made in `ZSTD_compress2()`

* Found by the libfuzzer fuzzers.
* Added a test case that catches the problem.
* I will make the fuzzers sometimes allocate less than
  `ZSTD_compressBound()` output space.
---
 lib/compress/zstd_compress.c | 18 ++++++++++++++----
 lib/zstd.h                   | 15 +++++++++++++--
 tests/fuzzer.c               | 13 +++++++++++++
 3 files changed, 40 insertions(+), 6 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index f2b9e03edc4..4a9f6b7c892 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -4179,18 +4179,28 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
     /* compression stage */
 #ifdef ZSTD_MULTITHREAD
     if (cctx->appliedParams.nbWorkers > 0) {
+        int const forceMaxProgress = (endOp == ZSTD_e_flush || endOp == ZSTD_e_end);
+        size_t flushMin;
+        assert(forceMaxProgress || endOp == ZSTD_e_continue /* Protection for a new flush type */);
         if (cctx->cParamsChanged) {
             ZSTDMT_updateCParams_whileCompressing(cctx->mtctx, &cctx->requestedParams);
             cctx->cParamsChanged = 0;
         }
-        {   size_t const flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp);
+        do {
+            flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp);
             if ( ZSTD_isError(flushMin)
               || (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */
                 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
             }
-            DEBUGLOG(5, "completed ZSTD_compressStream2 delegating to ZSTDMT_compressStream_generic");
-            return flushMin;
-    }   }
+            FORWARD_IF_ERROR(flushMin);
+        } while (forceMaxProgress && flushMin != 0 && output->pos < output->size);
+        DEBUGLOG(5, "completed ZSTD_compressStream2 delegating to ZSTDMT_compressStream_generic");
+        /* Either we don't require maximum forward progress, we've finished the
+         * flush, or we are out of output space.
+         */
+        assert(!forceMaxProgress || flushMin == 0 || output->pos == output->size);
+        return flushMin;
+    }
 #endif
     FORWARD_IF_ERROR( ZSTD_compressStream_generic(cctx, output, input, endOp) );
     DEBUGLOG(5, "completed ZSTD_compressStream2");
diff --git a/lib/zstd.h b/lib/zstd.h
index dc6348659bd..0c9ebe5b61f 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -577,6 +577,11 @@ typedef struct ZSTD_outBuffer_s {
 *  The caller must check if input has been entirely consumed.
 *  If not, the caller must make some room to receive more compressed data,
 *  and then present again remaining input data.
+*  note: ZSTD_e_continue is guaranteed to make some forward progress when called,
+*        but doesn't guarantee maximal forward progress. This is especially relevant
+*        when compressing with multiple threads. The call won't block if it can
+*        consume some input, but if it can't it will wait for some, but not all,
+*        output to be flushed.
 * @return : provides a minimum amount of data remaining to be flushed from internal buffers
 *           or an error code, which can be tested using ZSTD_isError().
 *
@@ -586,6 +591,8 @@ typedef struct ZSTD_outBuffer_s {
 *  In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush.
 *  You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the
 *  operation.
+*  note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will
+*        block until the flush is complete or the output buffer is full.
 *  @return : 0 if internal buffers are entirely flushed,
 *            >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
 *            or an error code, which can be tested using ZSTD_isError().
@@ -596,6 +603,8 @@ typedef struct ZSTD_outBuffer_s {
 *  flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush.
 *  You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to
 *  start a new frame.
+*  note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will
+*        block until the flush is complete or the output buffer is full.
 *  @return : 0 if frame fully completed and fully flushed,
 *            >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
 *            or an error code, which can be tested using ZSTD_isError().
@@ -613,11 +622,13 @@ typedef enum {
     ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */
     ZSTD_e_flush=1,    /* flush any data provided so far,
                         * it creates (at least) one new block, that can be decoded immediately on reception;
-                        * frame will continue: any future data can still reference previously compressed data, improving compression. */
+                        * frame will continue: any future data can still reference previously compressed data, improving compression.
+                        * note : multithreaded compression will block to flush as much output as possible. */
     ZSTD_e_end=2       /* flush any remaining data _and_ close current frame.
                         * note that frame is only closed after compressed data is fully flushed (return value == 0).
                         * After that point, any additional data starts a new frame.
-                        * note : each frame is independent (does not reference any content from previous frame). */
+                        * note : each frame is independent (does not reference any content from previous frame).
+                        : note : multithreaded compression will block to flush as much output as possible. */
 } ZSTD_EndDirective;
 
 /*! ZSTD_compressStream2() :
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 7bc2f10cbb2..cfb07eb1042 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -880,6 +880,19 @@ static int basicUnitTests(U32 seed, double compressibility)
     }
     DISPLAYLEVEL(3, "OK \n");
 
+    DISPLAYLEVEL(3, "test%3i : Multithreaded ZSTD_compress2() with rsyncable : ", testNb++)
+    {   ZSTD_CCtx* cctx = ZSTD_createCCtx();
+        /* Set rsyncable and don't give the ZSTD_compressBound(CNBuffSize) so
+         * ZSTDMT is forced to not take the shortcut.
+         */
+        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
+        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1) );
+        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_rsyncable, 1) );
+        CHECK( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize - 1, CNBuffer, CNBuffSize) );
+        ZSTD_freeCCtx(cctx);
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
     DISPLAYLEVEL(3, "test%3i : setting multithreaded parameters : ", testNb++)
     {   ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
         int value;

From c5d70b7dbb0a74272a5541d9e6ab523b3df0b31b Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Tue, 9 Apr 2019 16:47:59 -0700
Subject: [PATCH 143/178] [fuzzer] Sometimes fuzz with one less output byte

Zstd compression sometimes does different stuff when it has at least
`ZSTD_compressBound()` output bytes, or not. Half of the time fuzz with
`ZSTD_compressBound() - 1` output bytes. Ensure that we have at least
one byte of overhead by disabling either the dictionary ID or checksum.
---
 tests/fuzz/dictionary_round_trip.c | 30 +++++++++++++++---------------
 tests/fuzz/simple_round_trip.c     | 30 +++++++++++++++---------------
 2 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/tests/fuzz/dictionary_round_trip.c b/tests/fuzz/dictionary_round_trip.c
index 57a2eff3bf7..ceadbbc9dd3 100644
--- a/tests/fuzz/dictionary_round_trip.c
+++ b/tests/fuzz/dictionary_round_trip.c
@@ -24,9 +24,6 @@ static const int kMaxClevel = 19;
 
 static ZSTD_CCtx *cctx = NULL;
 static ZSTD_DCtx *dctx = NULL;
-static void* cBuf = NULL;
-static void* rBuf = NULL;
-static size_t bufSize = 0;
 static uint32_t seed;
 
 static size_t roundTripTest(void *result, size_t resultCapacity,
@@ -45,6 +42,8 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
                 cLevel);
     } else {
         FUZZ_setRandomParameters(cctx, srcSize, &seed);
+        /* Disable checksum so we can use sizes smaller than compress bound. */
+        FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 0));
         FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary(cctx, dict.buff, dict.size));
         cSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize);
     }
@@ -61,20 +60,19 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
 
 int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 {
-    size_t neededBufSize;
+    size_t const rBufSize = size;
+    void* rBuf = malloc(rBufSize);
+    size_t cBufSize = ZSTD_compressBound(size);
+    void* cBuf;
 
     seed = FUZZ_seed(&src, &size);
-    neededBufSize = ZSTD_compressBound(size);
+    /* Half of the time fuzz with a 1 byte smaller output size.
+     * This will still succeed because we force the checksum to be disabled,
+     * giving us 4 bytes of overhead.
+     */
+    cBufSize -= FUZZ_rand32(&seed, 0, 1);
+    cBuf = malloc(cBufSize);
 
-    /* Allocate all buffers and contexts if not already allocated */
-    if (neededBufSize > bufSize) {
-        free(cBuf);
-        free(rBuf);
-        cBuf = malloc(neededBufSize);
-        rBuf = malloc(neededBufSize);
-        bufSize = neededBufSize;
-        FUZZ_ASSERT(cBuf && rBuf);
-    }
     if (!cctx) {
         cctx = ZSTD_createCCtx();
         FUZZ_ASSERT(cctx);
@@ -86,11 +84,13 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 
     {
         size_t const result =
-            roundTripTest(rBuf, neededBufSize, cBuf, neededBufSize, src, size);
+            roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size);
         FUZZ_ZASSERT(result);
         FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size");
         FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!");
     }
+    free(rBuf);
+    free(cBuf);
 #ifndef STATEFUL_FUZZING
     ZSTD_freeCCtx(cctx); cctx = NULL;
     ZSTD_freeDCtx(dctx); dctx = NULL;
diff --git a/tests/fuzz/simple_round_trip.c b/tests/fuzz/simple_round_trip.c
index 21ab32c7f8d..7e3b6609822 100644
--- a/tests/fuzz/simple_round_trip.c
+++ b/tests/fuzz/simple_round_trip.c
@@ -25,9 +25,6 @@ static const int kMaxClevel = 19;
 
 static ZSTD_CCtx *cctx = NULL;
 static ZSTD_DCtx *dctx = NULL;
-static void* cBuf = NULL;
-static void* rBuf = NULL;
-static size_t bufSize = 0;
 static uint32_t seed;
 
 static size_t roundTripTest(void *result, size_t resultCapacity,
@@ -49,20 +46,21 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
 
 int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 {
-    size_t neededBufSize;
+    size_t const rBufSize = size;
+    void* rBuf = malloc(rBufSize);
+    size_t cBufSize = ZSTD_compressBound(size);
+    void* cBuf;
 
     seed = FUZZ_seed(&src, &size);
-    neededBufSize = ZSTD_compressBound(size);
+    /* Half of the time fuzz with a 1 byte smaller output size.
+     * This will still succeed because we don't use a dictionary, so the dictID
+     * field is empty, giving us 4 bytes of overhead.
+     */
+    cBufSize -= FUZZ_rand32(&seed, 0, 1);
+    cBuf = malloc(cBufSize);
+
+    FUZZ_ASSERT(cBuf && rBuf);
 
-    /* Allocate all buffers and contexts if not already allocated */
-    if (neededBufSize > bufSize) {
-        free(cBuf);
-        free(rBuf);
-        cBuf = malloc(neededBufSize);
-        rBuf = malloc(neededBufSize);
-        bufSize = neededBufSize;
-        FUZZ_ASSERT(cBuf && rBuf);
-    }
     if (!cctx) {
         cctx = ZSTD_createCCtx();
         FUZZ_ASSERT(cctx);
@@ -74,11 +72,13 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 
     {
         size_t const result =
-            roundTripTest(rBuf, neededBufSize, cBuf, neededBufSize, src, size);
+            roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size);
         FUZZ_ZASSERT(result);
         FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size");
         FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!");
     }
+    free(rBuf);
+    free(cBuf);
 #ifndef STATEFUL_FUZZING
     ZSTD_freeCCtx(cctx); cctx = NULL;
     ZSTD_freeDCtx(dctx); dctx = NULL;

From 824aaa695f811d081fb29ee0fb198e6cb0efdcfd Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Tue, 9 Apr 2019 17:59:27 -0700
Subject: [PATCH 144/178] [libzstd] Fix ZSTD_decompressDCtx() with a dictionary

* `ZSTD_decompressDCtx()` did not use the dictionary loaded by
  `ZSTD_DCtx_loadDictionary()`.
* Add a unit test.
* A stacked diff uses `ZSTD_decompressDCtx()` in the
  `dictionary_round_trip` and `dictionary_decompress` fuzzers.
---
 lib/decompress/zstd_decompress.c |  2 +-
 tests/fuzzer.c                   | 26 ++++++++++++++++++++++++++
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 14cc12a412f..d8f14882db8 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -788,7 +788,7 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
 
 size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
-    return ZSTD_decompress_usingDict(dctx, dst, dstCapacity, src, srcSize, NULL, 0);
+    return ZSTD_decompress_usingDDict(dctx, dst, dstCapacity, src, srcSize, dctx->ddict);
 }
 
 
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index cfb07eb1042..c38aef61083 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1438,6 +1438,32 @@ static int basicUnitTests(U32 seed, double compressibility)
         }
         DISPLAYLEVEL(3, "OK \n");
 
+        ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
+        CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, dictBuffer, dictSize) );
+        cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
+        CHECK_Z(cSize);
+        DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with dictionary : ", testNb++);
+        {
+            ZSTD_DCtx* dctx = ZSTD_createDCtx();
+            size_t ret;
+            /* We should fail to decompress without a dictionary. */
+            ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
+            ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
+            if (!ZSTD_isError(ret)) goto _output_error;
+            /* We should succeed to decompress with the dictionary. */
+            ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
+            CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictSize) );
+            CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
+            /* The dictionary should presist across calls. */
+            CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
+            /* When we reset the context the dictionary is cleared. */
+            ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
+            ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
+            if (!ZSTD_isError(ret)) goto _output_error;
+            ZSTD_freeDCtx(dctx);
+        }
+        DISPLAYLEVEL(3, "OK \n");
+
         DISPLAYLEVEL(3, "test%3i : Dictionary with non-default repcodes : ", testNb++);
         { U32 u; for (u=0; u
Date: Tue, 9 Apr 2019 18:01:49 -0700
Subject: [PATCH 145/178] [fuzzer] Make the regression_driver work while
 fuzzers are active

---
 tests/fuzz/regression_driver.c | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/tests/fuzz/regression_driver.c b/tests/fuzz/regression_driver.c
index 1553d436ce0..658c685f4f8 100644
--- a/tests/fuzz/regression_driver.c
+++ b/tests/fuzz/regression_driver.c
@@ -40,8 +40,13 @@ int main(int argc, char const **argv) {
     size_t readSize;
     FILE *file;
 
-    /* Check that it is a regular file, and that the fileSize is valid */
-    FUZZ_ASSERT_MSG(UTIL_isRegularFile(fileName), fileName);
+    /* Check that it is a regular file, and that the fileSize is valid.
+     * If it is not a regular file, then it may have been deleted since we
+     * constructed the list, so just skip it.
+     */
+    if (!UTIL_isRegularFile(fileName)) {
+      continue;
+    }
     FUZZ_ASSERT_MSG(fileSize <= kMaxFileSize, fileName);
     /* Ensure we have a large enough buffer allocated */
     if (fileSize > bufferSize) {

From c45dec12c59beafacd14507bee0150a69cac7726 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Tue, 9 Apr 2019 18:02:22 -0700
Subject: [PATCH 146/178] [fuzzer] Use ZSTD_DCtx_loadDictionary_advanced() half
 the time

---
 tests/fuzz/dictionary_decompress.c | 13 ++++++++++---
 tests/fuzz/dictionary_round_trip.c | 17 ++++++++++++-----
 2 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/tests/fuzz/dictionary_decompress.c b/tests/fuzz/dictionary_decompress.c
index bdecdef7ad3..7d3a7678adc 100644
--- a/tests/fuzz/dictionary_decompress.c
+++ b/tests/fuzz/dictionary_decompress.c
@@ -25,6 +25,7 @@ static size_t bufSize = 0;
 
 int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 {
+    FUZZ_dict_t dict;
     size_t neededBufSize;
 
     uint32_t seed = FUZZ_seed(&src, &size);
@@ -41,15 +42,21 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
         dctx = ZSTD_createDCtx();
         FUZZ_ASSERT(dctx);
     }
-    {
-        FUZZ_dict_t dict = FUZZ_train(src, size, &seed);
+    dict = FUZZ_train(src, size, &seed);
+    if (FUZZ_rand32(&seed, 0, 1) == 0) {
         ZSTD_decompress_usingDict(dctx,
                 rBuf, neededBufSize,
                 src, size,
                 dict.buff, dict.size);
-        free(dict.buff);
+    } else {
+        FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary_advanced(
+                dctx, dict.buff, dict.size,
+                (ZSTD_dictLoadMethod_e)FUZZ_rand32(&seed, 0, 1),
+                (ZSTD_dictContentType_e)FUZZ_rand32(&seed, 0, 2)));
+        ZSTD_decompressDCtx(dctx, rBuf, neededBufSize, src, size);
     }
 
+    free(dict.buff);
 #ifndef STATEFUL_FUZZING
     ZSTD_freeDCtx(dctx); dctx = NULL;
 #endif
diff --git a/tests/fuzz/dictionary_round_trip.c b/tests/fuzz/dictionary_round_trip.c
index ceadbbc9dd3..e28c65c98f0 100644
--- a/tests/fuzz/dictionary_round_trip.c
+++ b/tests/fuzz/dictionary_round_trip.c
@@ -30,6 +30,7 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
                             void *compressed, size_t compressedCapacity,
                             const void *src, size_t srcSize)
 {
+    ZSTD_dictContentType_e dictContentType = ZSTD_dct_auto;
     FUZZ_dict_t dict = FUZZ_train(src, srcSize, &seed);
     size_t cSize;
     if ((FUZZ_rand(&seed) & 15) == 0) {
@@ -41,18 +42,24 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
                 dict.buff, dict.size,
                 cLevel);
     } else {
+        dictContentType = FUZZ_rand32(&seed, 0, 2);
         FUZZ_setRandomParameters(cctx, srcSize, &seed);
         /* Disable checksum so we can use sizes smaller than compress bound. */
         FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 0));
-        FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary(cctx, dict.buff, dict.size));
+        FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary_advanced(
+                cctx, dict.buff, dict.size,
+                (ZSTD_dictLoadMethod_e)FUZZ_rand32(&seed, 0, 1),
+                dictContentType));
         cSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize);
     }
     FUZZ_ZASSERT(cSize);
+    FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary_advanced(
+        dctx, dict.buff, dict.size,
+        (ZSTD_dictLoadMethod_e)FUZZ_rand32(&seed, 0, 1),
+        dictContentType));
     {
-        size_t const ret = ZSTD_decompress_usingDict(dctx,
-                result, resultCapacity,
-                compressed, cSize,
-                dict.buff, dict.size);
+        size_t const ret = ZSTD_decompressDCtx(
+                dctx, result, resultCapacity, compressed, cSize);
         free(dict.buff);
         return ret;
     }

From bdeb4786b525d2b62dcd39205bf4e9f1fc40df4e Mon Sep 17 00:00:00 2001
From: Ori Peleg 
Date: Wed, 10 Apr 2019 13:37:41 +0300
Subject: [PATCH 147/178] Fixed `-r` typo

---
 programs/zstd.1    | 2 +-
 programs/zstd.1.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/programs/zstd.1 b/programs/zstd.1
index c93755f877e..cb4e1271a0f 100644
--- a/programs/zstd.1
+++ b/programs/zstd.1
@@ -164,7 +164,7 @@ keep source file(s) after successful compression or decompression\. This is the
 .
 .TP
 \fB\-r\fR
-operate recursively on dictionaries
+operate recursively on directories
 .
 .TP
 \fB\-\-format=FORMAT\fR
diff --git a/programs/zstd.1.md b/programs/zstd.1.md
index a029af5ff3c..93c6fa40010 100644
--- a/programs/zstd.1.md
+++ b/programs/zstd.1.md
@@ -178,7 +178,7 @@ the last one takes effect.
     keep source file(s) after successful compression or decompression.
     This is the default behavior.
 * `-r`:
-    operate recursively on dictionaries
+    operate recursively on directories
 * `--format=FORMAT`:
     compress and decompress in other formats. If compiled with
     support, zstd can compress to or decompress from other compression algorithm

From 90c0462d63768899d8a08a5ec0975294869e66f2 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 10 Apr 2019 10:03:06 -0700
Subject: [PATCH 148/178] minor presentation refactoring

and removed some // comment style
---
 programs/zstdcli.c | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index ef2fe2e91c7..f4452636bac 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -241,18 +241,20 @@ static void errorOut(const char* msg)
  * @return 1 if an overflow error occurs */
 static int readU32FromCharChecked(const char** stringPtr, unsigned* value)
 {
-    static unsigned const max = (((unsigned)(-1)) / 10) - 1;
     unsigned result = 0;
     while ((**stringPtr >='0') && (**stringPtr <='9')) {
-        if (result > max) return 1; // overflow error
-        result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
+        unsigned const max = (((unsigned)(-1)) / 10) - 1;
+        if (result > max) return 1; /* overflow error */
+        result *= 10;
+        result += (unsigned)(**stringPtr - '0');
+        (*stringPtr)++ ;
     }
     if ((**stringPtr=='K') || (**stringPtr=='M')) {
         unsigned const maxK = ((unsigned)(-1)) >> 10;
-        if (result > maxK) return 1; // overflow error
+        if (result > maxK) return 1; /* overflow error */
         result <<= 10;
         if (**stringPtr=='M') {
-            if (result > maxK) return 1; // overflow error
+            if (result > maxK) return 1; /* overflow error */
             result <<= 10;
         }
         (*stringPtr)++;  /* skip `K` or `M` */
@@ -483,7 +485,7 @@ static int init_cLevel(void) {
 
         if ((*ptr>='0') && (*ptr<='9')) {
             unsigned absLevel;
-            if (readU32FromCharChecked(&ptr, &absLevel)) { 
+            if (readU32FromCharChecked(&ptr, &absLevel)) {
                 DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large\n", ENV_CLEVEL, env);
                 return ZSTDCLI_CLEVEL_DEFAULT;
             } else if (*ptr == 0) {

From 59a7116cc2fb0118a0c4a238326b0549425b994e Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 10 Apr 2019 12:37:03 -0700
Subject: [PATCH 149/178] benchfn dependencies reduced to only timefn

benchfn used to rely on mem.h, and util,
which in turn relied on platform.h.
Using benchfn outside of zstd required to bring all these dependencies.

Now, dependency is reduced to timefn only.
This required to create a separate timefn from util,
and rewrite benchfn and timefn to no longer need mem.h.

Separating timefn from util has a wide effect accross the code base,
as usage of time functions is widespread.
A lot of build scripts had to be updated to also include timefn.
---
 contrib/adaptive-compression/Makefile |    4 +-
 contrib/adaptive-compression/adapt.c  |    1 +
 contrib/largeNbDicts/Makefile         |    8 +-
 doc/zstd_manual.html                  | 1071 ++++++++++++++-----------
 programs/Makefile                     |   12 +-
 programs/benchfn.c                    |   61 +-
 programs/benchzstd.c                  |    1 +
 programs/dibio.c                      |    1 +
 programs/fileio.c                     |    1 +
 programs/timefn.c                     |  149 ++++
 programs/timefn.h                     |   95 +++
 programs/util.c                       |  134 +---
 programs/util.h                       |   46 --
 tests/Makefile                        |   22 +-
 tests/decodecorpus.c                  |    1 +
 tests/fullbench.c                     |    1 +
 tests/fuzzer.c                        |    1 +
 tests/paramgrill.c                    |    1 +
 tests/zstreamtest.c                   |    1 +
 zlibWrapper/Makefile                  |    2 +-
 zlibWrapper/examples/zwrapbench.c     |    1 +
 21 files changed, 902 insertions(+), 712 deletions(-)
 create mode 100644 programs/timefn.c
 create mode 100644 programs/timefn.h

diff --git a/contrib/adaptive-compression/Makefile b/contrib/adaptive-compression/Makefile
index 2c6867f5cb7..2718e9d6e1b 100644
--- a/contrib/adaptive-compression/Makefile
+++ b/contrib/adaptive-compression/Makefile
@@ -22,10 +22,10 @@ FLAGS     = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MULTITHREAD_LDFLAGS)
 
 all: adapt datagen
 
-adapt: $(ZSTD_FILES) $(PRGDIR)/util.c adapt.c
+adapt: $(ZSTD_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c adapt.c
 	$(CC) $(FLAGS) $^ -o $@
 
-adapt-debug: $(ZSTD_FILES) $(PRGDIR)/util.c adapt.c
+adapt-debug: $(ZSTD_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c adapt.c
 	$(CC) $(FLAGS) -DDEBUG_MODE=2 $^ -o adapt
 
 datagen : $(PRGDIR)/datagen.c datagencli.c
diff --git a/contrib/adaptive-compression/adapt.c b/contrib/adaptive-compression/adapt.c
index 524d1378a80..8fb4047e996 100644
--- a/contrib/adaptive-compression/adapt.c
+++ b/contrib/adaptive-compression/adapt.c
@@ -13,6 +13,7 @@
 #include      /* memset */
 #include "zstd_internal.h"
 #include "util.h"
+#include "timefn.h"     /* UTIL_time_t, UTIL_getTime, UTIL_getSpanTimeMicro */
 
 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
 #define PRINT(...) fprintf(stdout, __VA_ARGS__)
diff --git a/contrib/largeNbDicts/Makefile b/contrib/largeNbDicts/Makefile
index 6fc38256379..4c055b0ed3f 100644
--- a/contrib/largeNbDicts/Makefile
+++ b/contrib/largeNbDicts/Makefile
@@ -28,14 +28,17 @@ default: largeNbDicts
 
 all : largeNbDicts
 
-largeNbDicts: util.o benchfn.o datagen.o xxhash.o largeNbDicts.c $(LIBZSTD)
+largeNbDicts: util.o timefn.o benchfn.o datagen.o xxhash.o largeNbDicts.c $(LIBZSTD)
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
 .PHONY: $(LIBZSTD)
 $(LIBZSTD):
 	$(MAKE) -C $(LIBDIR) libzstd.a CFLAGS="$(CFLAGS)"
 
-benchfn.o  : $(PROGDIR)/benchfn.c
+benchfn.o: $(PROGDIR)/benchfn.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+timefn.o: $(PROGDIR)/timefn.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
 
 datagen.o: $(PROGDIR)/datagen.c
@@ -48,6 +51,7 @@ util.o: $(PROGDIR)/util.c
 xxhash.o : $(LIBDIR)/common/xxhash.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
 
+
 clean:
 	$(RM) *.o
 	$(MAKE) -C $(LIBDIR) clean > /dev/null
diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index c7962e7de0c..cb101e1c62c 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -1,37 +1,46 @@
 
 
 
-zstd 1.3.8 Manual
+zstd 1.4.0 Manual
 
 
-
zstd 1.3.8 Manual

+
zstd 1.4.0 Manual

 

 
Contents

 
    
 
  1. Introduction
    2. 
 
  2. Version
    3. 
 
  3. Default constant
    4. 
-
  4. Simple API
    5. 
-
  5. Explicit context
    6. 
-
  6. Simple dictionary API
    7. 
-
  7. Bulk processing dictionary API
    8. 
-
  8. Streaming
    9. 
-
  9. Streaming compression - HowTo
    10. 
-
  10. Streaming decompression - HowTo
    11. 
-
  11. ADVANCED AND EXPERIMENTAL FUNCTIONS
    12. 
-
  12. Candidate API for promotion to stable status
    13. 
-
  13. Advanced compression API
    14. 
-
  14. experimental API (static linking only)
    15. 
-
  15. Frame size functions
    16. 
-
  16. Memory management
    17. 
-
  17. Advanced compression functions
    18. 
-
  18. Advanced decompression functions
    19. 
-
  19. Advanced streaming functions
    20. 
-
  20. Buffer-less and synchronous inner streaming functions
    21. 
-
  21. Buffer-less streaming compression (synchronous mode)
    22. 
-
  22. Buffer-less streaming decompression (synchronous mode)
    23. 
-
  23. ZSTD_getFrameHeader() :
    24. 
-
  24. Block level API
    25. 
+
  25. Constants
    26. 
+
  26. Simple API
    27. 
+
  27. Explicit context
    28. 
+
  28. Advanced compression API
    29. 
+
  29. Advanced decompression API
    30. 
+
  30. Streaming
    31. 
+
  31. Streaming compression - HowTo
    32. 
+
  32. This is a legacy streaming API, and can be replaced by ZSTD_CCtx_reset() and
    33. 
+
  33. Equivalent to:
    34. 
+
  34. Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).
    35. 
+
  35. Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush).
    36. 
+
  36. Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end).
    37. 
+
  37. Streaming decompression - HowTo
    38. 
+
  38. Simple dictionary API
    39. 
+
  39. Bulk processing dictionary API
    40. 
+
  40. Dictionary helper functions
    41. 
+
  41. Advanced dictionary and prefix API
    42. 
+
  42. ADVANCED AND EXPERIMENTAL FUNCTIONS
    43. 
+
  43. experimental API (static linking only)
    44. 
+
  44. Frame size functions
    45. 
+
  45. ZSTD_decompressBound() :
    46. 
+
  46. Memory management
    47. 
+
  47. Advanced compression functions
    48. 
+
  48. Advanced decompression functions
    49. 
+
  49. Advanced streaming functions
    50. 
+
  50. Buffer-less and synchronous inner streaming functions
    51. 
+
  51. Buffer-less streaming compression (synchronous mode)
    52. 
+
  52. Buffer-less streaming decompression (synchronous mode)
    53. 
+
  53. ZSTD_getFrameHeader() :
    54. 
+
  54. Block level API
    55. 
 

 

 
Introduction


 
Default constant

 
-
Simple API

+
Constants

+
+
Simple API

 
 
size_t ZSTD_compress( void* dst, size_t dstCapacity,
                 const void* src, size_t srcSize,
@@ -126,13 +137,22 @@ 
zstd 1.3.8 Manual

  @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. 
 


 
+
size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
+
 `src` should point to the start of a ZSTD frame or skippable frame.
+ `srcSize` must be >= first frame size
+ @return : the compressed size of the first frame starting at `src`,
+           suitable to pass as `srcSize` to `ZSTD_decompress` or similar,
+        or an error code if input is invalid 
+


+
 
Helper functions
#define ZSTD_COMPRESSBOUND(srcSize)   ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
 size_t      ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
 unsigned    ZSTD_isError(size_t code);          /*!< tells if a `size_t` function result is an error code */
 const char* ZSTD_getErrorName(size_t code);     /*!< provides readable string from an error code */
+int         ZSTD_minCLevel(void);               /*!< minimum negative compression level allowed */
 int         ZSTD_maxCLevel(void);               /*!< maximum compression level available */
 


-
Explicit context

+
Explicit context

 
 
Compression context
  When compressing many times,
   it is recommended to allocate a context just once, and re-use it for each successive compression operation.
@@ -169,228 +189,7 @@ 
Decompression context
  When decompressing many times,
  
 


 
-
Simple dictionary API

-
-
size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
-                               void* dst, size_t dstCapacity,
-                         const void* src, size_t srcSize,
-                         const void* dict,size_t dictSize,
-                               int compressionLevel);
-
  Compression at an explicit compression level using a Dictionary.
-  A dictionary can be any arbitrary data segment (also called a prefix),
-  or a buffer with specified information (see dictBuilder/zdict.h).
-  Note : This function loads the dictionary, resulting in significant startup delay.
-         It's intended for a dictionary used only once.
-  Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. 
-


-
-
size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
-                                 void* dst, size_t dstCapacity,
-                           const void* src, size_t srcSize,
-                           const void* dict,size_t dictSize);
-
  Decompression using a known Dictionary.
-  Dictionary must be identical to the one used during compression.
-  Note : This function loads the dictionary, resulting in significant startup delay.
-         It's intended for a dictionary used only once.
-  Note : When `dict == NULL || dictSize < 8` no dictionary is used. 
-


-
-
Bulk processing dictionary API

-
-
ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
-                             int compressionLevel);
-
  When compressing multiple messages / blocks using the same dictionary, it's recommended to load it only once.
-  ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup cost.
-  ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
- `dictBuffer` can be released after ZSTD_CDict creation, because its content is copied within CDict.
-  Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate `dictBuffer` content.
-  Note : A ZSTD_CDict can be created from an empty dictBuffer, but it is inefficient when used to compress small data. 
-


-
-
size_t      ZSTD_freeCDict(ZSTD_CDict* CDict);
-
  Function frees memory allocated by ZSTD_createCDict(). 
-


-
-
size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
-                                void* dst, size_t dstCapacity,
-                          const void* src, size_t srcSize,
-                          const ZSTD_CDict* cdict);
-
  Compression using a digested Dictionary.
-  Recommended when same dictionary is used multiple times.
-  Note : compression level is _decided at dictionary creation time_,
-     and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) 
-


-
-
ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
-
  Create a digested dictionary, ready to start decompression operation without startup delay.
-  dictBuffer can be released after DDict creation, as its content is copied inside DDict. 
-


-
-
size_t      ZSTD_freeDDict(ZSTD_DDict* ddict);
-
  Function frees memory allocated with ZSTD_createDDict() 
-


-
-
size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
-                                  void* dst, size_t dstCapacity,
-                            const void* src, size_t srcSize,
-                            const ZSTD_DDict* ddict);
-
  Decompression using a digested Dictionary.
-  Recommended when same dictionary is used multiple times. 
-


-
-
Streaming

-
-
typedef struct ZSTD_inBuffer_s {
-  const void* src;    /**< start of input buffer */
-  size_t size;        /**< size of input buffer */
-  size_t pos;         /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */
-} ZSTD_inBuffer;
-


-
typedef struct ZSTD_outBuffer_s {
-  void*  dst;         /**< start of output buffer */
-  size_t size;        /**< size of output buffer */
-  size_t pos;         /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */
-} ZSTD_outBuffer;
-


-
Streaming compression - HowTo
-  A ZSTD_CStream object is required to track streaming operation.
-  Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
-  ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
-  It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory.
-
-  For parallel execution, use one separate ZSTD_CStream per thread.
-
-  note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing.
-
-  Parameters are sticky : when starting a new compression on the same context,
-  it will re-use the same sticky parameters as previous compression session.
-  When in doubt, it's recommended to fully initialize the context before usage.
-  Use ZSTD_initCStream() to set the parameter to a selected compression level.
-  Use advanced API (ZSTD_CCtx_setParameter(), etc.) to set more specific parameters.
-
-  Use ZSTD_compressStream() as many times as necessary to consume input stream.
-  The function will automatically update both `pos` fields within `input` and `output`.
-  Note that the function may not consume the entire input,
-  for example, because the output buffer is already full,
-  in which case `input.pos < input.size`.
-  The caller must check if input has been entirely consumed.
-  If not, the caller must make some room to receive more compressed data,
-  and then present again remaining input data.
- @return : a size hint, preferred nb of bytes to use as input for next function call
-           or an error code, which can be tested using ZSTD_isError().
-           Note 1 : it's just a hint, to help latency a little, any value will work fine.
-           Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize()
-
-  At any moment, it's possible to flush whatever data might remain stuck within internal buffer,
-  using ZSTD_flushStream(). `output->pos` will be updated.
-  Note that, if `output->size` is too small, a single invocation of ZSTD_flushStream() might not be enough (return code > 0).
-  In which case, make some room to receive more compressed data, and call again ZSTD_flushStream().
-  @return : 0 if internal buffers are entirely flushed,
-            >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
-            or an error code, which can be tested using ZSTD_isError().
-
-  ZSTD_endStream() instructs to finish a frame.
-  It will perform a flush and write frame epilogue.
-  The epilogue is required for decoders to consider a frame completed.
-  flush() operation is the same, and follows same rules as ZSTD_flushStream().
-  @return : 0 if frame fully completed and fully flushed,
-            >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
-            or an error code, which can be tested using ZSTD_isError().
-
- 
-

-
-
typedef ZSTD_CCtx ZSTD_CStream;  /**< CCtx and CStream are now effectively same object (>= v1.3.0) */
-


-
ZSTD_CStream management functions
ZSTD_CStream* ZSTD_createCStream(void);
-size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
-


-
Streaming compression functions
size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
-size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
-size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
-size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
-


-
size_t ZSTD_CStreamInSize(void);    /**< recommended size for input buffer */
-


-
size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */
-


-
Streaming decompression - HowTo
-  A ZSTD_DStream object is required to track streaming operations.
-  Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
-  ZSTD_DStream objects can be re-used multiple times.
-
-  Use ZSTD_initDStream() to start a new decompression operation.
- @return : recommended first input size
-  Alternatively, use advanced API to set specific properties.
-
-  Use ZSTD_decompressStream() repetitively to consume your input.
-  The function will update both `pos` fields.
-  If `input.pos < input.size`, some input has not been consumed.
-  It's up to the caller to present again remaining data.
-  The function tries to flush all data decoded immediately, respecting output buffer size.
-  If `output.pos < output.size`, decoder has flushed everything it could.
-  But if `output.pos == output.size`, there might be some data left within internal buffers.,
-  In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer.
-  Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.
- @return : 0 when a frame is completely decoded and fully flushed,
-        or an error code, which can be tested using ZSTD_isError(),
-        or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
-                                the return value is a suggested next input size (just a hint for better latency)
-                                that will never request more than the remaining frame size.
- 
-

-
-
typedef ZSTD_DCtx ZSTD_DStream;  /**< DCtx and DStream are now effectively same object (>= v1.3.0) */
-


-
ZSTD_DStream management functions
ZSTD_DStream* ZSTD_createDStream(void);
-size_t ZSTD_freeDStream(ZSTD_DStream* zds);
-


-
Streaming decompression functions
size_t ZSTD_initDStream(ZSTD_DStream* zds);
-size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
-


-
size_t ZSTD_DStreamInSize(void);    /*!< recommended size for input buffer */
-


-
size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */
-


-
ADVANCED AND EXPERIMENTAL FUNCTIONS
- The definitions in the following section are considered experimental.
- They are provided for advanced scenarios.
- They should never be used with a dynamic library, as prototypes may change in the future.
- Use them only in association with static linking.
- 
-

-
-
Candidate API for promotion to stable status
- The following symbols and constants form the "staging area" :
- they are considered to join "stable API" by v1.4.0.
- The proposal is written so that it can be made stable "as is",
- though it's still possible to suggest improvements.
- Staging is in fact last chance for changes,
- the API is locked once reaching "stable" status.
- 
-

-
-
int ZSTD_minCLevel(void);  /*!< minimum negative compression level allowed */
-


-
size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
-
 `src` should point to the start of a ZSTD frame or skippable frame.
- `srcSize` must be >= first frame size
- @return : the compressed size of the first frame starting at `src`,
-           suitable to pass as `srcSize` to `ZSTD_decompress` or similar,
-        or an error code if input is invalid 
-


-
-
size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
-size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
-size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
-size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
-size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
-size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
-
  These functions give the _current_ memory usage of selected object.
-  Note that object memory usage can evolve (increase or decrease) over time. 
-


-
-
Advanced compression API

+
Advanced compression API

 
 
typedef enum { ZSTD_fast=1,
                ZSTD_dfast=2,
@@ -407,7 +206,10 @@ 
Streaming decompression functions
size_t ZSTD_initDSt
 


 
typedef enum {
 
-    /* compression parameters */
+    /* compression parameters
+     * Note: When compressing with a ZSTD_CDict these parameters are superseded
+     * by the parameters used to construct the ZSTD_CDict. See ZSTD_CCtx_refCDict()
+     * for more info (superseded-by-cdict). */
     ZSTD_c_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table
                               * Default level is ZSTD_CLEVEL_DEFAULT==3.
                               * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
@@ -522,68 +324,438 @@ 
Streaming decompression functions
size_t ZSTD_initDSt
                               * 9: full window;  8: w/2;  7: w/4;  6: w/8;  5:w/16;  4: w/32;  3:w/64;  2:w/128;  1:no overlap;  0:default
                               * default value varies between 6 and 9, depending on strategy */
 
-    /* note : additional experimental parameters are also available
-     * within the experimental section of the API.
-     * At the time of this writing, they include :
-     * ZSTD_c_rsyncable
-     * ZSTD_c_format
-     * ZSTD_c_forceMaxWindow
-     * ZSTD_c_forceAttachDict
-     * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
-     * note : never ever use experimentalParam? names directly;
-     *        also, the enums values themselves are unstable and can still change.
-     */
-     ZSTD_c_experimentalParam1=500,
-     ZSTD_c_experimentalParam2=10,
-     ZSTD_c_experimentalParam3=1000,
-     ZSTD_c_experimentalParam4=1001
-} ZSTD_cParameter;
-


-
typedef struct {
-    size_t error;
-    int lowerBound;
-    int upperBound;
-} ZSTD_bounds;
-


-
ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam);
-
  All parameters must belong to an interval with lower and upper bounds,
-  otherwise they will either trigger an error or be automatically clamped.
- @return : a structure, ZSTD_bounds, which contains
-         - an error status field, which must be tested using ZSTD_isError()
-         - lower and upper bounds, both inclusive
- 
+    /* note : additional experimental parameters are also available
+     * within the experimental section of the API.
+     * At the time of this writing, they include :
+     * ZSTD_c_rsyncable
+     * ZSTD_c_format
+     * ZSTD_c_forceMaxWindow
+     * ZSTD_c_forceAttachDict
+     * ZSTD_c_literalCompressionMode
+     * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
+     * note : never ever use experimentalParam? names directly;
+     *        also, the enums values themselves are unstable and can still change.
+     */
+     ZSTD_c_experimentalParam1=500,
+     ZSTD_c_experimentalParam2=10,
+     ZSTD_c_experimentalParam3=1000,
+     ZSTD_c_experimentalParam4=1001,
+     ZSTD_c_experimentalParam5=1002,
+} ZSTD_cParameter;
+


+
typedef struct {
+    size_t error;
+    int lowerBound;
+    int upperBound;
+} ZSTD_bounds;
+


+
ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam);
+
  All parameters must belong to an interval with lower and upper bounds,
+  otherwise they will either trigger an error or be automatically clamped.
+ @return : a structure, ZSTD_bounds, which contains
+         - an error status field, which must be tested using ZSTD_isError()
+         - lower and upper bounds, both inclusive
+ 
+


+
+
size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value);
+
  Set one compression parameter, selected by enum ZSTD_cParameter.
+  All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds().
+  Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).
+  Setting a parameter is generally only possible during frame initialization (before starting compression).
+  Exception : when using multi-threading mode (nbWorkers >= 1),
+              the following parameters can be updated _during_ compression (within same frame):
+              => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy.
+              new parameters will be active for next job only (after a flush()).
+ @return : an error code (which can be tested using ZSTD_isError()).
+ 
+


+
+
size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
+
  Total input data size to be compressed as a single frame.
+  Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag.
+  This value will also be controlled at end of frame, and trigger an error if not respected.
+ @result : 0, or an error code (which can be tested with ZSTD_isError()).
+  Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame.
+           In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
+           ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame.
+  Note 2 : pledgedSrcSize is only valid once, for the next frame.
+           It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN.
+  Note 3 : Whenever all input data is provided and consumed in a single round,
+           for example with ZSTD_compress2(),
+           or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end),
+           this value is automatically overriden by srcSize instead.
+ 
+


+
+
typedef enum {
+    ZSTD_reset_session_only = 1,
+    ZSTD_reset_parameters = 2,
+    ZSTD_reset_session_and_parameters = 3
+} ZSTD_ResetDirective;
+


+
size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset);
+
  There are 2 different things that can be reset, independently or jointly :
+  - The session : will stop compressing current frame, and make CCtx ready to start a new one.
+                  Useful after an error, or to interrupt any ongoing compression.
+                  Any internal data not yet flushed is cancelled.
+                  Compression parameters and dictionary remain unchanged.
+                  They will be used to compress next frame.
+                  Resetting session never fails.
+  - The parameters : changes all parameters back to "default".
+                  This removes any reference to any dictionary too.
+                  Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing)
+                  otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError())
+  - Both : similar to resetting the session, followed by resetting parameters.
+ 
+


+
+
size_t ZSTD_compress2( ZSTD_CCtx* cctx,
+                       void* dst, size_t dstCapacity,
+                 const void* src, size_t srcSize);
+
  Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API.
+  ZSTD_compress2() always starts a new frame.
+  Should cctx hold data from a previously unfinished frame, everything about it is forgotten.
+  - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
+  - The function is always blocking, returns when compression is completed.
+  Hint : compression runs faster if `dstCapacity` >=  `ZSTD_compressBound(srcSize)`.
+ @return : compressed size written into `dst` (<= `dstCapacity),
+           or an error code if it fails (which can be tested using ZSTD_isError()).
+ 
+


+
+
Advanced decompression API

+
+
typedef enum {
+
+    ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which
+                              * the streaming API will refuse to allocate memory buffer
+                              * in order to protect the host from unreasonable memory requirements.
+                              * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
+                              * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT).
+                              * Special: value 0 means "use default maximum windowLog". */
+
+    /* note : additional experimental parameters are also available
+     * within the experimental section of the API.
+     * At the time of this writing, they include :
+     * ZSTD_c_format
+     * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
+     * note : never ever use experimentalParam? names directly
+     */
+     ZSTD_d_experimentalParam1=1000
+
+} ZSTD_dParameter;
+


+
ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam);
+
  All parameters must belong to an interval with lower and upper bounds,
+  otherwise they will either trigger an error or be automatically clamped.
+ @return : a structure, ZSTD_bounds, which contains
+         - an error status field, which must be tested using ZSTD_isError()
+         - both lower and upper bounds, inclusive
+ 
+


+
+
size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value);
+
  Set one compression parameter, selected by enum ZSTD_dParameter.
+  All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds().
+  Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).
+  Setting a parameter is only possible during frame initialization (before starting decompression).
+ @return : 0, or an error code (which can be tested using ZSTD_isError()).
+ 
+


+
+
size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset);
+
  Return a DCtx to clean state.
+  Session and parameters can be reset jointly or separately.
+  Parameters can only be reset when no active frame is being decompressed.
+ @return : 0, or an error code, which can be tested with ZSTD_isError()
+ 
+


+
+
Streaming

+
+
typedef struct ZSTD_inBuffer_s {
+  const void* src;    /**< start of input buffer */
+  size_t size;        /**< size of input buffer */
+  size_t pos;         /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */
+} ZSTD_inBuffer;
+


+
typedef struct ZSTD_outBuffer_s {
+  void*  dst;         /**< start of output buffer */
+  size_t size;        /**< size of output buffer */
+  size_t pos;         /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */
+} ZSTD_outBuffer;
+


+
Streaming compression - HowTo
+  A ZSTD_CStream object is required to track streaming operation.
+  Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
+  ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
+  It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory.
+
+  For parallel execution, use one separate ZSTD_CStream per thread.
+
+  note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing.
+
+  Parameters are sticky : when starting a new compression on the same context,
+  it will re-use the same sticky parameters as previous compression session.
+  When in doubt, it's recommended to fully initialize the context before usage.
+  Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(),
+  ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to
+  set more specific parameters, the pledged source size, or load a dictionary.
+
+  Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to
+  consume input stream. The function will automatically update both `pos`
+  fields within `input` and `output`.
+  Note that the function may not consume the entire input, for example, because
+  the output buffer is already full, in which case `input.pos < input.size`.
+  The caller must check if input has been entirely consumed.
+  If not, the caller must make some room to receive more compressed data,
+  and then present again remaining input data.
+  note: ZSTD_e_continue is guaranteed to make some forward progress when called,
+        but doesn't guarantee maximal forward progress. This is especially relevant
+        when compressing with multiple threads. The call won't block if it can
+        consume some input, but if it can't it will wait for some, but not all,
+        output to be flushed.
+ @return : provides a minimum amount of data remaining to be flushed from internal buffers
+           or an error code, which can be tested using ZSTD_isError().
+
+  At any moment, it's possible to flush whatever data might remain stuck within internal buffer,
+  using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated.
+  Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0).
+  In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush.
+  You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the
+  operation.
+  note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will
+        block until the flush is complete or the output buffer is full.
+  @return : 0 if internal buffers are entirely flushed,
+            >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
+            or an error code, which can be tested using ZSTD_isError().
+
+  Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame.
+  It will perform a flush and write frame epilogue.
+  The epilogue is required for decoders to consider a frame completed.
+  flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush.
+  You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to
+  start a new frame.
+  note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will
+        block until the flush is complete or the output buffer is full.
+  @return : 0 if frame fully completed and fully flushed,
+            >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
+            or an error code, which can be tested using ZSTD_isError().
+
+ 
+

+
+
typedef ZSTD_CCtx ZSTD_CStream;  /**< CCtx and CStream are now effectively same object (>= v1.3.0) */
+


+
ZSTD_CStream management functions
ZSTD_CStream* ZSTD_createCStream(void);
+size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
+


+
Streaming compression functions
typedef enum {
+    ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */
+    ZSTD_e_flush=1,    /* flush any data provided so far,
+                        * it creates (at least) one new block, that can be decoded immediately on reception;
+                        * frame will continue: any future data can still reference previously compressed data, improving compression.
+                        * note : multithreaded compression will block to flush as much output as possible. */
+    ZSTD_e_end=2       /* flush any remaining data _and_ close current frame.
+                        * note that frame is only closed after compressed data is fully flushed (return value == 0).
+                        * After that point, any additional data starts a new frame.
+                        * note : each frame is independent (does not reference any content from previous frame).
+                        : note : multithreaded compression will block to flush as much output as possible. */
+} ZSTD_EndDirective;
+


+
size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
+                             ZSTD_outBuffer* output,
+                             ZSTD_inBuffer* input,
+                             ZSTD_EndDirective endOp);
+
  Behaves about the same as ZSTD_compressStream, with additional control on end directive.
+  - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
+  - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
+  - outpot->pos must be <= dstCapacity, input->pos must be <= srcSize
+  - outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
+  - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
+  - When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads, flush whatever is available,
+                                                  and then immediately returns, just indicating that there is some data remaining to be flushed.
+                                                  The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
+  - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking.
+  - @return provides a minimum amount of data remaining to be flushed from internal buffers
+            or an error code, which can be tested using ZSTD_isError().
+            if @return != 0, flush is not fully completed, there is still some data left within internal buffers.
+            This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers.
+            For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed.
+  - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0),
+            only ZSTD_e_end or ZSTD_e_flush operations are allowed.
+            Before starting a new compression job, or changing compression parameters,
+            it is required to fully flush internal buffers.
+ 
+


+
+
size_t ZSTD_CStreamInSize(void);    /**< recommended size for input buffer */
+


+
size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */
+


+
This is a legacy streaming API, and can be replaced by ZSTD_CCtx_reset() and
 ZSTD_compressStream2(). It is redundent, but is still fully supported.
+ Advanced parameters and dictionary compression can only be used through the
+ new API.
+

+
+
Equivalent to:
+     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+     ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
+     ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
+ 
+

+
+
Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).
 NOTE: The return value is different. ZSTD_compressStream() returns a hint for
+ the next read size (if non-zero and not an error). ZSTD_compressStream2()
+ returns the number of bytes left to flush (if non-zero and not an error).
+ 
+

+
+
Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush).

+
+
Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end).

+
+
Streaming decompression - HowTo
+  A ZSTD_DStream object is required to track streaming operations.
+  Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
+  ZSTD_DStream objects can be re-used multiple times.
+
+  Use ZSTD_initDStream() to start a new decompression operation.
+ @return : recommended first input size
+  Alternatively, use advanced API to set specific properties.
+
+  Use ZSTD_decompressStream() repetitively to consume your input.
+  The function will update both `pos` fields.
+  If `input.pos < input.size`, some input has not been consumed.
+  It's up to the caller to present again remaining data.
+  The function tries to flush all data decoded immediately, respecting output buffer size.
+  If `output.pos < output.size`, decoder has flushed everything it could.
+  But if `output.pos == output.size`, there might be some data left within internal buffers.,
+  In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer.
+  Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.
+ @return : 0 when a frame is completely decoded and fully flushed,
+        or an error code, which can be tested using ZSTD_isError(),
+        or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
+                                the return value is a suggested next input size (just a hint for better latency)
+                                that will never request more than the remaining frame size.
+ 
+

+
+
typedef ZSTD_DCtx ZSTD_DStream;  /**< DCtx and DStream are now effectively same object (>= v1.3.0) */
+


+
ZSTD_DStream management functions
ZSTD_DStream* ZSTD_createDStream(void);
+size_t ZSTD_freeDStream(ZSTD_DStream* zds);
+


+
Streaming decompression functions
size_t ZSTD_initDStream(ZSTD_DStream* zds);
+size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+


+
size_t ZSTD_DStreamInSize(void);    /*!< recommended size for input buffer */
+


+
size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */
+


+
Simple dictionary API

+
+
size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
+                               void* dst, size_t dstCapacity,
+                         const void* src, size_t srcSize,
+                         const void* dict,size_t dictSize,
+                               int compressionLevel);
+
  Compression at an explicit compression level using a Dictionary.
+  A dictionary can be any arbitrary data segment (also called a prefix),
+  or a buffer with specified information (see dictBuilder/zdict.h).
+  Note : This function loads the dictionary, resulting in significant startup delay.
+         It's intended for a dictionary used only once.
+  Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. 
+


+
+
size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
+                                 void* dst, size_t dstCapacity,
+                           const void* src, size_t srcSize,
+                           const void* dict,size_t dictSize);
+
  Decompression using a known Dictionary.
+  Dictionary must be identical to the one used during compression.
+  Note : This function loads the dictionary, resulting in significant startup delay.
+         It's intended for a dictionary used only once.
+  Note : When `dict == NULL || dictSize < 8` no dictionary is used. 
+


+
+
Bulk processing dictionary API

+
+
ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
+                             int compressionLevel);
+
  When compressing multiple messages / blocks using the same dictionary, it's recommended to load it only once.
+  ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup cost.
+  ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
+ `dictBuffer` can be released after ZSTD_CDict creation, because its content is copied within CDict.
+  Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate `dictBuffer` content.
+  Note : A ZSTD_CDict can be created from an empty dictBuffer, but it is inefficient when used to compress small data. 
+


+
+
size_t      ZSTD_freeCDict(ZSTD_CDict* CDict);
+
  Function frees memory allocated by ZSTD_createCDict(). 
+


+
+
size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
+                                void* dst, size_t dstCapacity,
+                          const void* src, size_t srcSize,
+                          const ZSTD_CDict* cdict);
+
  Compression using a digested Dictionary.
+  Recommended when same dictionary is used multiple times.
+  Note : compression level is _decided at dictionary creation time_,
+     and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) 
+


+
+
ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
+
  Create a digested dictionary, ready to start decompression operation without startup delay.
+  dictBuffer can be released after DDict creation, as its content is copied inside DDict. 
 


 
-
size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value);
-
  Set one compression parameter, selected by enum ZSTD_cParameter.
-  All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds().
-  Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).
-  Setting a parameter is generally only possible during frame initialization (before starting compression).
-  Exception : when using multi-threading mode (nbWorkers >= 1),
-              the following parameters can be updated _during_ compression (within same frame):
-              => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy.
-              new parameters will be active for next job only (after a flush()).
- @return : an error code (which can be tested using ZSTD_isError()).
- 
+
size_t      ZSTD_freeDDict(ZSTD_DDict* ddict);
+
  Function frees memory allocated with ZSTD_createDDict() 
 


 
-
size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
-
  Total input data size to be compressed as a single frame.
-  Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag.
-  This value will also be controlled at end of frame, and trigger an error if not respected.
- @result : 0, or an error code (which can be tested with ZSTD_isError()).
-  Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame.
-           In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
-           ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame.
-  Note 2 : pledgedSrcSize is only valid once, for the next frame.
-           It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN.
-  Note 3 : Whenever all input data is provided and consumed in a single round,
-           for example with ZSTD_compress2(),
-           or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end),
-           this value is automatically overriden by srcSize instead.
- 
+
size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
+                                  void* dst, size_t dstCapacity,
+                            const void* src, size_t srcSize,
+                            const ZSTD_DDict* ddict);
+
  Decompression using a digested Dictionary.
+  Recommended when same dictionary is used multiple times. 
+


+
+
Dictionary helper functions

+
+
unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
+
  Provides the dictID stored within dictionary.
+  if @return == 0, the dictionary is not conformant with Zstandard specification.
+  It can still be loaded, but as a content-only dictionary. 
+


+
+
unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
+
  Provides the dictID of the dictionary loaded into `ddict`.
+  If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
+  Non-conformant dictionaries can still be loaded, but as content-only dictionaries. 
+


+
+
unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
+
  Provides the dictID required to decompressed the frame stored within `src`.
+  If @return == 0, the dictID could not be decoded.
+  This could for one of the following reasons :
+  - The frame does not require a dictionary to be decoded (most common case).
+  - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.
+    Note : this use case also happens when using a non-conformant dictionary.
+  - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
+  - This is not a Zstandard frame.
+  When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. 
 


 
+
Advanced dictionary and prefix API
+ This API allows dictionaries to be used with ZSTD_compress2(),
+ ZSTD_compressStream2(), and ZSTD_decompress(). Dictionaries are sticky, and
+ only reset with the context is reset with ZSTD_reset_parameters or
+ ZSTD_reset_session_and_parameters. Prefixes are single-use.
+

+
 
size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
 
  Create an internal CDict from `dict` buffer.
   Decompression will have to use same dictionary.
@@ -606,7 +778,9 @@ 
Streaming decompression functions
size_t ZSTD_initDSt
 
size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
 
  Reference a prepared dictionary, to be used for all next compressed frames.
   Note that compression parameters are enforced from within CDict,
-  and supercede any compression parameter previously set within CCtx.
+  and supersede any compression parameter previously set within CCtx.
+  The parameters ignored are labled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
+  The ignored parameters will be used again if the CCtx is returned to no-dictionary mode.
   The dictionary will remain valid for future compressed frames using same CCtx.
  @result : 0, or an error code (which can be tested with ZSTD_isError()).
   Special : Referencing a NULL CDict means "return to no-dictionary mode".
@@ -636,116 +810,6 @@ 
Streaming decompression functions
size_t ZSTD_initDSt
            Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. 
 


 
-
typedef enum {
-    ZSTD_reset_session_only = 1,
-    ZSTD_reset_parameters = 2,
-    ZSTD_reset_session_and_parameters = 3
-} ZSTD_ResetDirective;
-


-
size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset);
-
  There are 2 different things that can be reset, independently or jointly :
-  - The session : will stop compressing current frame, and make CCtx ready to start a new one.
-                  Useful after an error, or to interrupt any ongoing compression.
-                  Any internal data not yet flushed is cancelled.
-                  Compression parameters and dictionary remain unchanged.
-                  They will be used to compress next frame.
-                  Resetting session never fails.
-  - The parameters : changes all parameters back to "default".
-                  This removes any reference to any dictionary too.
-                  Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing)
-                  otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError())
-  - Both : similar to resetting the session, followed by resetting parameters.
- 
-


-
-
size_t ZSTD_compress2( ZSTD_CCtx* cctx,
-                       void* dst, size_t dstCapacity,
-                 const void* src, size_t srcSize);
-
  Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API.
-  ZSTD_compress2() always starts a new frame.
-  Should cctx hold data from a previously unfinished frame, everything about it is forgotten.
-  - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
-  - The function is always blocking, returns when compression is completed.
-  Hint : compression runs faster if `dstCapacity` >=  `ZSTD_compressBound(srcSize)`.
- @return : compressed size written into `dst` (<= `dstCapacity),
-           or an error code if it fails (which can be tested using ZSTD_isError()).
- 
-


-
-
typedef enum {
-    ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */
-    ZSTD_e_flush=1,    /* flush any data provided so far,
-                        * it creates (at least) one new block, that can be decoded immediately on reception;
-                        * frame will continue: any future data can still reference previously compressed data, improving compression. */
-    ZSTD_e_end=2       /* flush any remaining data _and_ close current frame.
-                        * note that frame is only closed after compressed data is fully flushed (return value == 0).
-                        * After that point, any additional data starts a new frame.
-                        * note : each frame is independent (does not reference any content from previous frame). */
-} ZSTD_EndDirective;
-


-
size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
-                             ZSTD_outBuffer* output,
-                             ZSTD_inBuffer* input,
-                             ZSTD_EndDirective endOp);
-
  Behaves about the same as ZSTD_compressStream, with additional control on end directive.
-  - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
-  - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
-  - outpot->pos must be <= dstCapacity, input->pos must be <= srcSize
-  - outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
-  - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
-  - When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads, flush whatever is available,
-                                                  and then immediately returns, just indicating that there is some data remaining to be flushed.
-                                                  The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
-  - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking.
-  - @return provides a minimum amount of data remaining to be flushed from internal buffers
-            or an error code, which can be tested using ZSTD_isError().
-            if @return != 0, flush is not fully completed, there is still some data left within internal buffers.
-            This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers.
-            For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed.
-  - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0),
-            only ZSTD_e_end or ZSTD_e_flush operations are allowed.
-            Before starting a new compression job, or changing compression parameters,
-            it is required to fully flush internal buffers.
- 
-


-
-
typedef enum {
-
-    ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which
-                              * the streaming API will refuse to allocate memory buffer
-                              * in order to protect the host from unreasonable memory requirements.
-                              * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
-                              * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) */
-
-    /* note : additional experimental parameters are also available
-     * within the experimental section of the API.
-     * At the time of this writing, they include :
-     * ZSTD_c_format
-     * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
-     * note : never ever use experimentalParam? names directly
-     */
-     ZSTD_d_experimentalParam1=1000
-
-} ZSTD_dParameter;
-


-
ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam);
-
  All parameters must belong to an interval with lower and upper bounds,
-  otherwise they will either trigger an error or be automatically clamped.
- @return : a structure, ZSTD_bounds, which contains
-         - an error status field, which must be tested using ZSTD_isError()
-         - both lower and upper bounds, inclusive
- 
-


-
-
size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value);
-
  Set one compression parameter, selected by enum ZSTD_dParameter.
-  All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds().
-  Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).
-  Setting a parameter is only possible during frame initialization (before starting decompression).
- @return : 0, or an error code (which can be tested using ZSTD_isError()).
- 
-


-
 
size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
 
  Create an internal DDict from dict buffer,
   to be used to decompress next frames.
@@ -793,15 +857,25 @@ 
Streaming decompression functions
size_t ZSTD_initDSt
  
 


 
-
size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset);
-
  Return a DCtx to clean state.
-  Session and parameters can be reset jointly or separately.
-  Parameters can only be reset when no active frame is being decompressed.
- @return : 0, or an error code, which can be tested with ZSTD_isError()
- 
+
size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
+size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
+size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
+size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
+size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
+size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
+
  These functions give the _current_ memory usage of selected object.
+  Note that object memory usage can evolve (increase or decrease) over time. 
 


 
-
experimental API (static linking only)
+
ADVANCED AND EXPERIMENTAL FUNCTIONS
+ The definitions in the following section are considered experimental.
+ They are provided for advanced scenarios.
+ They should never be used with a dynamic library, as prototypes may change in the future.
+ Use them only in association with static linking.
+ 
+

+
+
experimental API (static linking only)
  The following symbols and constants
  are not planned to join "stable API" status in the near future.
  They can still change in future versions.
@@ -890,12 +964,21 @@ 
Streaming decompression functions
size_t ZSTD_initDSt
     ZSTD_dictForceCopy     = 2, /* Always copy the dictionary. */
 } ZSTD_dictAttachPref_e;
 


-
Frame size functions

+
typedef enum {
+  ZSTD_lcm_auto = 0,          /**< Automatically determine the compression mode based on the compression level.
+                               *   Negative compression levels will be uncompressed, and positive compression
+                               *   levels will be compressed. */
+  ZSTD_lcm_huffman = 1,       /**< Always attempt Huffman compression. Uncompressed literals will still be
+                               *   emitted if Huffman compression is not profitable. */
+  ZSTD_lcm_uncompressed = 2,  /**< Always emit uncompressed literals. */
+} ZSTD_literalCompressionMode_e;
+


+
Frame size functions

 
 
unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
-
  `src` should point the start of a series of ZSTD encoded and/or skippable frames
+
  `src` should point to the start of a series of ZSTD encoded and/or skippable frames
   `srcSize` must be the _exact_ size of this series
-       (i.e. there should be a frame boundary exactly at `srcSize` bytes after `src`)
+       (i.e. there should be a frame boundary at `src + srcSize`)
   @return : - decompressed size of all data in all successive frames
             - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
             - if an error occurred: ZSTD_CONTENTSIZE_ERROR
@@ -915,13 +998,27 @@ 
Streaming decompression functions
size_t ZSTD_initDSt
             however it does mean that all frame data must be present and valid. 
 


 
+
ZSTD_decompressBound() :
  `src` should point to the start of a series of ZSTD encoded and/or skippable frames
+  `srcSize` must be the _exact_ size of this series
+       (i.e. there should be a frame boundary at `src + srcSize`)
+  @return : - upper-bound for the decompressed size of all data in all successive frames
+            - if an error occured: ZSTD_CONTENTSIZE_ERROR
+
+  note 1  : an error can occur if `src` contains an invalid or incorrectly formatted frame.
+  note 2  : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`.
+            in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value.
+  note 3  : when the decompressed size field isn't available, the upper-bound for that frame is calculated by:
+              upper-bound = # blocks * min(128 KB, Window_Size)
+ 
+

+
 
size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
 
  srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX.
  @return : size of the Frame Header,
            or an error code (if srcSize is too small) 
 


 
-
Memory management

+
Memory management

 
 
size_t ZSTD_estimateCCtxSize(int compressionLevel);
 size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
@@ -933,7 +1030,7 @@ 
Streaming decompression functions
size_t ZSTD_initDSt
   It will also consider src size to be arbitrarily "large", which is worst case.
   If srcSize is known to always be small, ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation.
   ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
-  ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
+  ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
   Note : CCtx size estimation is only correct for single-threaded compression. 
 


 
@@ -946,7 +1043,7 @@ 
Streaming decompression functions
size_t ZSTD_initDSt
   It will also consider src size to be arbitrarily "large", which is worst case.
   If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation.
   ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
-  ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
+  ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
   Note : CStream size estimation is only correct for single-threaded compression.
   ZSTD_DStream memory budget depends on window Size.
   This information can be passed manually, using ZSTD_estimateDStreamSize,
@@ -1001,7 +1098,7 @@ 
Streaming decompression functions
size_t ZSTD_initDSt
  
 


 
-
Advanced compression functions

+
Advanced compression functions

 
 
ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
 
  Create a digested dictionary for compression
@@ -1076,10 +1173,10 @@ 
Streaming decompression functions
size_t ZSTD_initDSt
 size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
 
  Quick howto :
   - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure
-  - ZSTD_CCtxParam_setParameter() : Push parameters one by one into
-                                    an existing ZSTD_CCtx_params structure.
-                                    This is similar to
-                                    ZSTD_CCtx_setParameter().
+  - ZSTD_CCtxParams_setParameter() : Push parameters one by one into
+                                     an existing ZSTD_CCtx_params structure.
+                                     This is similar to
+                                     ZSTD_CCtx_setParameter().
   - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to
                                     an existing CCtx.
                                     These parameters will be applied to
@@ -1109,7 +1206,7 @@ 
Streaming decompression functions
size_t ZSTD_initDSt
  
 


 
-
size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
+
size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
 
  Similar to ZSTD_CCtx_setParameter.
   Set one compression parameter, selected by enum ZSTD_cParameter.
   Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams().
@@ -1117,7 +1214,7 @@ 
Streaming decompression functions
size_t ZSTD_initDSt
  
 


 
-
size_t ZSTD_CCtxParam_getParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
+
size_t ZSTD_CCtxParams_getParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
 
 Similar to ZSTD_CCtx_getParameter.
  Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
  @result : 0, or an error code (which can be tested with ZSTD_isError()).
@@ -1146,7 +1243,7 @@ 
Streaming decompression functions
size_t ZSTD_initDSt
  
 


 
-
Advanced decompression functions

+
Advanced decompression functions

 
 
unsigned ZSTD_isFrame(const void* buffer, size_t size);
 
  Tells if the content of `buffer` starts with a valid Frame Identifier.
@@ -1162,30 +1259,6 @@ 
Streaming decompression functions
size_t ZSTD_initDSt
   it must remain read accessible throughout the lifetime of DDict 
 


 
-
unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
-
  Provides the dictID stored within dictionary.
-  if @return == 0, the dictionary is not conformant with Zstandard specification.
-  It can still be loaded, but as a content-only dictionary. 
-


-
-
unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
-
  Provides the dictID of the dictionary loaded into `ddict`.
-  If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
-  Non-conformant dictionaries can still be loaded, but as content-only dictionaries. 
-


-
-
unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
-
  Provides the dictID required to decompressed the frame stored within `src`.
-  If @return == 0, the dictID could not be decoded.
-  This could for one of the following reasons :
-  - The frame does not require a dictionary to be decoded (most common case).
-  - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.
-    Note : this use case also happens when using a non-conformant dictionary.
-  - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
-  - This is not a Zstandard frame.
-  When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. 
-


-
 
size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
 
  Same as ZSTD_DCtx_loadDictionary(),
   but references `dict` content instead of copying it into `dctx`.
@@ -1232,20 +1305,74 @@ 
Streaming decompression functions
size_t ZSTD_initDSt
  
 


 
-
Advanced streaming functions
  Warning : most of these functions are now redundant with the Advanced API.
+
Advanced streaming functions
  Warning : most of these functions are now redundant with the Advanced API.
   Once Advanced API reaches "stable" status,
   redundant functions will be deprecated, and then at some point removed.
 

 
-
Advanced Streaming compression functions
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   /**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, "0" also disables frame content size field. It may be enabled in the future. */
-size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/
+
Advanced Streaming compression functions
/**! ZSTD_initCStream_srcSize() :
+ * This function is deprecated, and equivalent to:
+ *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ *     ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
+ *     ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
+ *     ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
+ *
+ * pledgedSrcSize must be correct. If it is not known at init time, use
+ * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs,
+ * "0" also disables frame content size field. It may be enabled in the future.
+ */
+size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);
+/**! ZSTD_initCStream_usingDict() :
+ * This function is deprecated, and is equivalent to:
+ *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ *     ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
+ *     ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
+ *
+ * Creates of an internal CDict (incompatible with static CCtx), except if
+ * dict == NULL or dictSize < 8, in which case no dict is used.
+ * Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if
+ * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
+ */
+size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
+/**! ZSTD_initCStream_advanced() :
+ * This function is deprecated, and is approximately equivalent to:
+ *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ *     ZSTD_CCtx_setZstdParams(zcs, params); // Set the zstd params and leave the rest as-is
+ *     ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
+ *     ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
+ *
+ * pledgedSrcSize must be correct. If srcSize is not known at init time, use
+ * value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy.
+ */
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
-                                             ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */
-size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);  /**< note : cdict will just be referenced, and must outlive compression session */
-size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize);  /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters. pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. */
+                                             ZSTD_parameters params, unsigned long long pledgedSrcSize);
+/**! ZSTD_initCStream_usingCDict() :
+ * This function is deprecated, and equivalent to:
+ *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ *     ZSTD_CCtx_refCDict(zcs, cdict);
+ *
+ * note : cdict will just be referenced, and must outlive compression session
+ */
+size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
+/**! ZSTD_initCStream_usingCDict_advanced() :
+ * This function is deprecated, and is approximately equivalent to:
+ *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ *     ZSTD_CCtx_setZstdFrameParams(zcs, fParams); // Set the zstd frame params and leave the rest as-is
+ *     ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
+ *     ZSTD_CCtx_refCDict(zcs, cdict);
+ *
+ * same as ZSTD_initCStream_usingCDict(), with control over frame parameters.
+ * pledgedSrcSize must be correct. If srcSize is not known at init time, use
+ * value ZSTD_CONTENTSIZE_UNKNOWN.
+ */
+size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize);
 


 
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
-
  start a new frame, using same parameters from previous frame.
+
 This function is deprecated, and is equivalent to:
+     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+     ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
+
+  start a new frame, using same parameters from previous frame.
   This is typically useful to skip dictionary loading stage, since it will re-use it in-place.
   Note that zcs must be init at least once before using ZSTD_resetCStream().
   If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
@@ -1284,14 +1411,14 @@ 
Advanced Streaming decompression functions
size_t ZST
 size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);  /**< note : ddict is referenced, it must outlive decompression session */
 size_t ZSTD_resetDStream(ZSTD_DStream* zds);  /**< re-use decompression parameters from previous init; saves dictionary loading */
 


-
Buffer-less and synchronous inner streaming functions
+
Buffer-less and synchronous inner streaming functions
   This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
   But it's also a complex one, with several restrictions, documented below.
   Prefer normal streaming API for an easier experience.
  
 

 
-
Buffer-less streaming compression (synchronous mode)
+
Buffer-less streaming compression (synchronous mode)
   A ZSTD_CCtx object is required to track streaming operations.
   Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
   ZSTD_CCtx object can be re-used multiple times within successive compression operations.
@@ -1327,7 +1454,7 @@ 
Buffer-less streaming compression functions
size_t ZS
 size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize);   /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
 size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**<  note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
 


-
Buffer-less streaming decompression (synchronous mode)
+
Buffer-less streaming decompression (synchronous mode)
   A ZSTD_DCtx object is required to track streaming operations.
   Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
   A ZSTD_DCtx object can be re-used multiple times.
@@ -1409,7 +1536,7 @@ 
Buffer-less streaming decompression functions
typedef
     unsigned checksumFlag;
 } ZSTD_frameHeader;
 


-
ZSTD_getFrameHeader() :
  decode Frame Header, or requires larger `srcSize`.
+
ZSTD_getFrameHeader() :
  decode Frame Header, or requires larger `srcSize`.
  @return : 0, `zfhPtr` is correctly filled,
           >0, `srcSize` is too small, value is wanted `srcSize` amount,
            or an error code, which can be tested using ZSTD_isError() 
@@ -1425,7 +1552,7 @@ 
Buffer-less streaming decompression functions
typedef
 
 
typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
 


-
Block level API

+
Block level API

 
 
    Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
     User will have to take in charge required information to regenerate data, such as compressed and content sizes.
diff --git a/programs/Makefile b/programs/Makefile
index 692980e4fee..5ceeff1504f 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -165,7 +165,7 @@ $(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP)
 zstd : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) $(LZ4CPP)
 zstd : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD)
 zstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
-zstd : $(ZSTDLIB_FILES) zstdcli.o util.o fileio.o benchfn.o benchzstd.o datagen.o dibio.o
+zstd : $(ZSTDLIB_FILES) zstdcli.o util.o timefn.o fileio.o benchfn.o benchzstd.o datagen.o dibio.o
 	@echo "$(THREAD_MSG)"
 	@echo "$(ZLIB_MSG)"
 	@echo "$(LZMA_MSG)"
@@ -183,13 +183,13 @@ zstd-release: zstd
 zstd32 : CPPFLAGS += $(THREAD_CPP)
 zstd32 : LDFLAGS  += $(THREAD_LD)
 zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
-zstd32 : $(ZSTDLIB_FILES) zstdcli.c util.c fileio.c benchfn.c benchzstd.c datagen.c dibio.c
+zstd32 : $(ZSTDLIB_FILES) zstdcli.c util.c timefn.c fileio.c benchfn.c benchzstd.c datagen.c dibio.c
 ifneq (,$(filter Windows%,$(OS)))
 	windres/generate_res.bat
 endif
 	$(CC) -m32 $(FLAGS) $^ $(RES32_FILE) -o $@$(EXT)
 
-zstd-nolegacy : $(ZSTD_FILES) $(ZDICT_FILES) zstdcli.o util.o fileio.c benchfn.o benchzstd.o datagen.o dibio.o
+zstd-nolegacy : $(ZSTD_FILES) $(ZDICT_FILES) zstdcli.o util.o fileio.c benchfn.o benchzstd.o timefn.o datagen.o dibio.o
 	$(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS)
 
 zstd-nomt : THREAD_CPP :=
@@ -222,13 +222,13 @@ zstd-pgo :
 
 # minimal target, with only zstd compression and decompression. no bench. no legacy.
 zstd-small: CFLAGS = -Os -s
-zstd-frugal zstd-small: $(ZSTD_FILES) zstdcli.c util.c fileio.c
+zstd-frugal zstd-small: $(ZSTD_FILES) zstdcli.c util.c timefn.c fileio.c
 	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT $^ -o $@$(EXT)
 
-zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c util.c fileio.c
+zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c util.c timefn.c fileio.c
 	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS $^ -o $@$(EXT)
 
-zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c util.c fileio.c
+zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c util.c timefn.c fileio.c
 	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT)
 
 zstdmt: zstd
diff --git a/programs/benchfn.c b/programs/benchfn.c
index 318f4d63b12..8ae619536a4 100644
--- a/programs/benchfn.c
+++ b/programs/benchfn.c
@@ -13,25 +13,20 @@
 /* *************************************
 *  Includes
 ***************************************/
-#include "platform.h"    /* Large Files support */
-#include "util.h"        /* UTIL_getFileSize, UTIL_sleep */
 #include       /* malloc, free */
 #include       /* memset */
-#include        /* fprintf, fopen */
 #undef NDEBUG            /* assert must not be disabled */
 #include       /* assert */
 
-#include "mem.h"
+#include "timefn.h"        /* UTIL_time_t, UTIL_getTime */
 #include "benchfn.h"
 
 
 /* *************************************
 *  Constants
 ***************************************/
-#define TIMELOOP_MICROSEC     (1*1000000ULL) /* 1 second */
+#define TIMELOOP_MICROSEC     SEC_TO_MICRO      /* 1 second */
 #define TIMELOOP_NANOSEC      (1*1000000000ULL) /* 1 second */
-#define ACTIVEPERIOD_MICROSEC (70*TIMELOOP_MICROSEC) /* 70 seconds */
-#define COOLPERIOD_SEC        10
 
 #define KB *(1 <<10)
 #define MB *(1 <<20)
@@ -39,14 +34,16 @@
 
 
 /* *************************************
-*  Errors
+*  Debug errors
 ***************************************/
-#ifndef DEBUG
-#  define DEBUG 0
+#if defined(DEBUG) && (DEBUG >= 1)
+#  include        /* fprintf */
+#  define DISPLAY(...)       fprintf(stderr, __VA_ARGS__)
+#  define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); }
+#else
+#  define DEBUGOUTPUT(...)
 #endif
 
-#define DISPLAY(...)       fprintf(stderr, __VA_ARGS__)
-#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); }
 
 /* error without displaying */
 #define RETURN_QUIET_ERROR(retValue, ...) {           \
@@ -116,15 +113,7 @@ BMK_runOutcome_t BMK_benchFunction(BMK_benchParams_t p,
     {   size_t i;
         for(i = 0; i < p.blockCount; i++) {
             memset(p.dstBuffers[i], 0xE5, p.dstCapacities[i]);  /* warm up and erase result buffer */
-        }
-#if 0
-        /* based on testing these seem to lower accuracy of multiple calls of 1 nbLoops vs 1 call of multiple nbLoops
-         * (Makes former slower)
-         */
-        UTIL_sleepMilli(5);  /* give processor time to other processes */
-        UTIL_waitForNextTick();
-#endif
-    }
+    }   }
 
     /* benchmark */
     {   UTIL_time_t const clockStart = UTIL_getTime();
@@ -146,7 +135,7 @@ BMK_runOutcome_t BMK_benchFunction(BMK_benchParams_t p,
             }   }
         }  /* for (loopNb = 0; loopNb < nbLoops; loopNb++) */
 
-        {   U64 const totalTime = UTIL_clockSpanNano(clockStart);
+        {   PTime const totalTime = UTIL_clockSpanNano(clockStart);
             BMK_runTime_t rt;
             rt.nanoSecPerRun = (double)totalTime / nbLoops;
             rt.sumOfReturn = dstSize;
@@ -158,9 +147,9 @@ BMK_runOutcome_t BMK_benchFunction(BMK_benchParams_t p,
 /* ====  Benchmarking any function, providing intermediate results  ==== */
 
 struct BMK_timedFnState_s {
-    U64 timeSpent_ns;
-    U64 timeBudget_ns;
-    U64 runBudget_ns;
+    PTime timeSpent_ns;
+    PTime timeBudget_ns;
+    PTime runBudget_ns;
     BMK_runTime_t fastestRun;
     unsigned nbLoops;
     UTIL_time_t coolTime;
@@ -196,8 +185,8 @@ void BMK_resetTimedFnState(BMK_timedFnState_t* timedFnState, unsigned total_ms,
     if (!run_ms) run_ms = 1;
     if (run_ms > total_ms) run_ms = total_ms;
     timedFnState->timeSpent_ns = 0;
-    timedFnState->timeBudget_ns = (U64)total_ms * TIMELOOP_NANOSEC / 1000;
-    timedFnState->runBudget_ns = (U64)run_ms * TIMELOOP_NANOSEC / 1000;
+    timedFnState->timeBudget_ns = (PTime)total_ms * TIMELOOP_NANOSEC / 1000;
+    timedFnState->runBudget_ns = (PTime)run_ms * TIMELOOP_NANOSEC / 1000;
     timedFnState->fastestRun.nanoSecPerRun = (double)TIMELOOP_NANOSEC * 2000000000;  /* hopefully large enough : must be larger than any potential measurement */
     timedFnState->fastestRun.sumOfReturn = (size_t)(-1LL);
     timedFnState->nbLoops = 1;
@@ -220,23 +209,13 @@ int BMK_isCompleted_TimedFn(const BMK_timedFnState_t* timedFnState)
 BMK_runOutcome_t BMK_benchTimedFn(BMK_timedFnState_t* cont,
                                   BMK_benchParams_t p)
 {
-    U64 const runBudget_ns = cont->runBudget_ns;
-    U64 const runTimeMin_ns = runBudget_ns / 2;
+    PTime const runBudget_ns = cont->runBudget_ns;
+    PTime const runTimeMin_ns = runBudget_ns / 2;
     int completed = 0;
     BMK_runTime_t bestRunTime = cont->fastestRun;
 
     while (!completed) {
-        BMK_runOutcome_t runResult;
-
-        /* Overheat protection */
-        if (UTIL_clockSpanMicro(cont->coolTime) > ACTIVEPERIOD_MICROSEC) {
-            DEBUGOUTPUT("\rcooling down ...    \r");
-            UTIL_sleep(COOLPERIOD_SEC);
-            cont->coolTime = UTIL_getTime();
-        }
-
-        /* reinitialize capacity */
-        runResult = BMK_benchFunction(p, cont->nbLoops);
+        BMK_runOutcome_t const runResult = BMK_benchFunction(p, cont->nbLoops);
 
         if(!BMK_isSuccessful_runOutcome(runResult)) { /* error : move out */
             return runResult;
@@ -250,7 +229,7 @@ BMK_runOutcome_t BMK_benchTimedFn(BMK_timedFnState_t* cont,
             /* estimate nbLoops for next run to last approximately 1 second */
             if (loopDuration_ns > (runBudget_ns / 50)) {
                 double const fastestRun_ns = MIN(bestRunTime.nanoSecPerRun, newRunTime.nanoSecPerRun);
-                cont->nbLoops = (U32)(runBudget_ns / fastestRun_ns) + 1;
+                cont->nbLoops = (unsigned)(runBudget_ns / fastestRun_ns) + 1;
             } else {
                 /* previous run was too short : blindly increase workload by x multiplier */
                 const unsigned multiplier = 10;
diff --git a/programs/benchzstd.c b/programs/benchzstd.c
index 8f3511919bc..94ec5f25104 100644
--- a/programs/benchzstd.c
+++ b/programs/benchzstd.c
@@ -28,6 +28,7 @@
 #include 
 #include       /* assert */
 
+#include "timefn.h"      /* UTIL_time_t */
 #include "benchfn.h"
 #include "mem.h"
 #define ZSTD_STATIC_LINKING_ONLY
diff --git a/programs/dibio.c b/programs/dibio.c
index c9d214e754e..12eb3268085 100644
--- a/programs/dibio.c
+++ b/programs/dibio.c
@@ -29,6 +29,7 @@
 #include           /* errno */
 #include 
 
+#include "timefn.h"         /* UTIL_time_t, UTIL_clockSpanMicro, UTIL_getTime */
 #include "mem.h"            /* read */
 #include "error_private.h"
 #include "dibio.h"
diff --git a/programs/fileio.c b/programs/fileio.c
index 412ef4762c7..b0cea5eacd4 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -31,6 +31,7 @@
 #include 
 #include       /* errno */
 #include 
+#include "timefn.h"     /* UTIL_getTime, UTIL_clockSpanMicro */
 
 #if defined (_MSC_VER)
 #  include 
diff --git a/programs/timefn.c b/programs/timefn.c
new file mode 100644
index 00000000000..ad24769582f
--- /dev/null
+++ b/programs/timefn.c
@@ -0,0 +1,149 @@
+/*
+ * Copyright (c) 2019-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+/* ===  Dependencies  === */
+
+#include "timefn.h"
+
+
+/*-****************************************
+*  Time functions
+******************************************/
+
+#if defined(_WIN32)   /* Windows */
+
+UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; }
+
+PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
+{
+    static LARGE_INTEGER ticksPerSecond;
+    static int init = 0;
+    if (!init) {
+        if (!QueryPerformanceFrequency(&ticksPerSecond))
+            UTIL_DISPLAYLEVEL(1, "ERROR: QueryPerformanceFrequency() failure\n");
+        init = 1;
+    }
+    return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
+}
+
+PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
+{
+    static LARGE_INTEGER ticksPerSecond;
+    static int init = 0;
+    if (!init) {
+        if (!QueryPerformanceFrequency(&ticksPerSecond))
+            UTIL_DISPLAYLEVEL(1, "ERROR: QueryPerformanceFrequency() failure\n");
+        init = 1;
+    }
+    return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
+}
+
+#elif defined(__APPLE__) && defined(__MACH__)
+
+UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); }
+
+PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
+{
+    static mach_timebase_info_data_t rate;
+    static int init = 0;
+    if (!init) {
+        mach_timebase_info(&rate);
+        init = 1;
+    }
+    return (((clockEnd - clockStart) * (PTime)rate.numer) / ((PTime)rate.denom))/1000ULL;
+}
+
+PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
+{
+    static mach_timebase_info_data_t rate;
+    static int init = 0;
+    if (!init) {
+        mach_timebase_info(&rate);
+        init = 1;
+    }
+    return ((clockEnd - clockStart) * (PTime)rate.numer) / ((PTime)rate.denom);
+}
+
+#elif (PLATFORM_POSIX_VERSION >= 200112L) \
+   && (defined(__UCLIBC__)                \
+      || (defined(__GLIBC__)              \
+          && ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 17) \
+             || (__GLIBC__ > 2))))
+
+UTIL_time_t UTIL_getTime(void)
+{
+    UTIL_time_t time;
+    if (clock_gettime(CLOCK_MONOTONIC, &time))
+        UTIL_DISPLAYLEVEL(1, "ERROR: Failed to get time\n");   /* we could also exit() */
+    return time;
+}
+
+UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end)
+{
+    UTIL_time_t diff;
+    if (end.tv_nsec < begin.tv_nsec) {
+        diff.tv_sec = (end.tv_sec - 1) - begin.tv_sec;
+        diff.tv_nsec = (end.tv_nsec + 1000000000ULL) - begin.tv_nsec;
+    } else {
+        diff.tv_sec = end.tv_sec - begin.tv_sec;
+        diff.tv_nsec = end.tv_nsec - begin.tv_nsec;
+    }
+    return diff;
+}
+
+PTime UTIL_getSpanTimeMicro(UTIL_time_t begin, UTIL_time_t end)
+{
+    UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
+    PTime micro = 0;
+    micro += 1000000ULL * diff.tv_sec;
+    micro += diff.tv_nsec / 1000ULL;
+    return micro;
+}
+
+PTime UTIL_getSpanTimeNano(UTIL_time_t begin, UTIL_time_t end)
+{
+    UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
+    PTime nano = 0;
+    nano += 1000000000ULL * diff.tv_sec;
+    nano += diff.tv_nsec;
+    return nano;
+}
+
+#else   /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */
+
+UTIL_time_t UTIL_getTime(void) { return clock(); }
+PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
+PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
+
+#endif
+
+/* returns time span in microseconds */
+PTime UTIL_clockSpanMicro(UTIL_time_t clockStart )
+{
+    UTIL_time_t const clockEnd = UTIL_getTime();
+    return UTIL_getSpanTimeMicro(clockStart, clockEnd);
+}
+
+/* returns time span in microseconds */
+PTime UTIL_clockSpanNano(UTIL_time_t clockStart )
+{
+    UTIL_time_t const clockEnd = UTIL_getTime();
+    return UTIL_getSpanTimeNano(clockStart, clockEnd);
+}
+
+void UTIL_waitForNextTick(void)
+{
+    UTIL_time_t const clockStart = UTIL_getTime();
+    UTIL_time_t clockEnd;
+    do {
+        clockEnd = UTIL_getTime();
+    } while (UTIL_getSpanTimeNano(clockStart, clockEnd) == 0);
+}
diff --git a/programs/timefn.h b/programs/timefn.h
new file mode 100644
index 00000000000..7892a692114
--- /dev/null
+++ b/programs/timefn.h
@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef TIME_FN_H_MODULE_287987
+#define TIME_FN_H_MODULE_287987
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+/*-****************************************
+*  Dependencies
+******************************************/
+#include     /* utime */
+#if defined(_MSC_VER)
+#  include   /* utime */
+#else
+#  include       /* utime */
+#endif
+#include          /* clock_t, clock, CLOCKS_PER_SEC */
+
+
+
+/*-****************************************
+*  Local Types
+******************************************/
+
+#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+# include 
+  typedef uint64_t PTime;  /* Precise Time */
+#else
+  typedef unsigned long long PTime;   /* does not support compilers without long long support */
+#endif
+
+
+
+/*-****************************************
+*  Time functions
+******************************************/
+#if defined(_WIN32)   /* Windows */
+
+    #define UTIL_TIME_INITIALIZER { { 0, 0 } }
+    typedef LARGE_INTEGER UTIL_time_t;
+
+#elif defined(__APPLE__) && defined(__MACH__)
+
+    #include 
+    #define UTIL_TIME_INITIALIZER 0
+    typedef PTime UTIL_time_t;
+
+#elif (PLATFORM_POSIX_VERSION >= 200112L) \
+   && (defined(__UCLIBC__)                \
+      || (defined(__GLIBC__)              \
+          && ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 17) \
+             || (__GLIBC__ > 2))))
+
+    #define UTIL_TIME_INITIALIZER { 0, 0 }
+    typedef struct timespec UTIL_freq_t;
+    typedef struct timespec UTIL_time_t;
+
+    UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end);
+
+#else   /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */
+
+    typedef clock_t UTIL_time_t;
+    #define UTIL_TIME_INITIALIZER 0
+
+#endif
+
+UTIL_time_t UTIL_getTime(void);
+PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd);
+PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd);
+
+
+#define SEC_TO_MICRO 1000000
+PTime UTIL_clockSpanMicro(UTIL_time_t clockStart);
+
+PTime UTIL_clockSpanNano(UTIL_time_t clockStart);
+
+void UTIL_waitForNextTick(void);
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* TIME_FN_H_MODULE_287987 */
diff --git a/programs/util.c b/programs/util.c
index 622e5025f37..7b827d4536a 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -352,146 +352,18 @@ UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
     return fileTable;
 }
 
+
 /*-****************************************
 *  Console log
 ******************************************/
 int g_utilDisplayLevel;
 
 
+
 /*-****************************************
-*  Time functions
+*  count the number of physical cores
 ******************************************/
-#if defined(_WIN32)   /* Windows */
-
-UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; }
-
-U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
-{
-    static LARGE_INTEGER ticksPerSecond;
-    static int init = 0;
-    if (!init) {
-        if (!QueryPerformanceFrequency(&ticksPerSecond))
-            UTIL_DISPLAYLEVEL(1, "ERROR: QueryPerformanceFrequency() failure\n");
-        init = 1;
-    }
-    return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
-}
-
-U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
-{
-    static LARGE_INTEGER ticksPerSecond;
-    static int init = 0;
-    if (!init) {
-        if (!QueryPerformanceFrequency(&ticksPerSecond))
-            UTIL_DISPLAYLEVEL(1, "ERROR: QueryPerformanceFrequency() failure\n");
-        init = 1;
-    }
-    return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
-}
-
-#elif defined(__APPLE__) && defined(__MACH__)
-
-UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); }
-
-U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
-{
-    static mach_timebase_info_data_t rate;
-    static int init = 0;
-    if (!init) {
-        mach_timebase_info(&rate);
-        init = 1;
-    }
-    return (((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom))/1000ULL;
-}
-
-U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
-{
-    static mach_timebase_info_data_t rate;
-    static int init = 0;
-    if (!init) {
-        mach_timebase_info(&rate);
-        init = 1;
-    }
-    return ((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom);
-}
-
-#elif (PLATFORM_POSIX_VERSION >= 200112L) \
-   && (defined(__UCLIBC__)                \
-      || (defined(__GLIBC__)              \
-          && ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 17) \
-             || (__GLIBC__ > 2))))
-
-UTIL_time_t UTIL_getTime(void)
-{
-    UTIL_time_t time;
-    if (clock_gettime(CLOCK_MONOTONIC, &time))
-        UTIL_DISPLAYLEVEL(1, "ERROR: Failed to get time\n");   /* we could also exit() */
-    return time;
-}
-
-UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end)
-{
-    UTIL_time_t diff;
-    if (end.tv_nsec < begin.tv_nsec) {
-        diff.tv_sec = (end.tv_sec - 1) - begin.tv_sec;
-        diff.tv_nsec = (end.tv_nsec + 1000000000ULL) - begin.tv_nsec;
-    } else {
-        diff.tv_sec = end.tv_sec - begin.tv_sec;
-        diff.tv_nsec = end.tv_nsec - begin.tv_nsec;
-    }
-    return diff;
-}
-
-U64 UTIL_getSpanTimeMicro(UTIL_time_t begin, UTIL_time_t end)
-{
-    UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
-    U64 micro = 0;
-    micro += 1000000ULL * diff.tv_sec;
-    micro += diff.tv_nsec / 1000ULL;
-    return micro;
-}
-
-U64 UTIL_getSpanTimeNano(UTIL_time_t begin, UTIL_time_t end)
-{
-    UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
-    U64 nano = 0;
-    nano += 1000000000ULL * diff.tv_sec;
-    nano += diff.tv_nsec;
-    return nano;
-}
-
-#else   /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */
-
-UTIL_time_t UTIL_getTime(void) { return clock(); }
-U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
-U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
-
-#endif
-
-/* returns time span in microseconds */
-U64 UTIL_clockSpanMicro(UTIL_time_t clockStart )
-{
-    UTIL_time_t const clockEnd = UTIL_getTime();
-    return UTIL_getSpanTimeMicro(clockStart, clockEnd);
-}
-
-/* returns time span in microseconds */
-U64 UTIL_clockSpanNano(UTIL_time_t clockStart )
-{
-    UTIL_time_t const clockEnd = UTIL_getTime();
-    return UTIL_getSpanTimeNano(clockStart, clockEnd);
-}
-
-void UTIL_waitForNextTick(void)
-{
-    UTIL_time_t const clockStart = UTIL_getTime();
-    UTIL_time_t clockEnd;
-    do {
-        clockEnd = UTIL_getTime();
-    } while (UTIL_getSpanTimeNano(clockStart, clockEnd) == 0);
-}
 
-/* count the number of physical cores */
 #if defined(_WIN32) || defined(WIN32)
 
 #include 
diff --git a/programs/util.h b/programs/util.h
index eee7ebfc33b..d6e5bb550ec 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -112,52 +112,6 @@ extern int g_utilDisplayLevel;
 #define UTIL_DISPLAYLEVEL(l, ...) { if (g_utilDisplayLevel>=l) { UTIL_DISPLAY(__VA_ARGS__); } }
 
 
-/*-****************************************
-*  Time functions
-******************************************/
-#if defined(_WIN32)   /* Windows */
-
-    #define UTIL_TIME_INITIALIZER { { 0, 0 } }
-    typedef LARGE_INTEGER UTIL_time_t;
-
-#elif defined(__APPLE__) && defined(__MACH__)
-
-    #include 
-    #define UTIL_TIME_INITIALIZER 0
-    typedef U64 UTIL_time_t;
-
-#elif (PLATFORM_POSIX_VERSION >= 200112L) \
-   && (defined(__UCLIBC__)                \
-      || (defined(__GLIBC__)              \
-          && ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 17) \
-             || (__GLIBC__ > 2))))
-
-    #define UTIL_TIME_INITIALIZER { 0, 0 }
-    typedef struct timespec UTIL_freq_t;
-    typedef struct timespec UTIL_time_t;
-
-    UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end);
-
-#else   /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */
-
-    typedef clock_t UTIL_time_t;
-    #define UTIL_TIME_INITIALIZER 0
-
-#endif
-
-UTIL_time_t UTIL_getTime(void);
-U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd);
-U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd);
-
-#define SEC_TO_MICRO 1000000
-
-/* returns time span in microseconds */
-U64 UTIL_clockSpanMicro(UTIL_time_t clockStart);
-
-/* returns time span in microseconds */
-U64 UTIL_clockSpanNano(UTIL_time_t clockStart);
-void UTIL_waitForNextTick(void);
-
 /*-****************************************
 *  File functions
 ******************************************/
diff --git a/tests/Makefile b/tests/Makefile
index 2a9cd3d8f64..695a49b9959 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -132,18 +132,18 @@ fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP)
 fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD)
 fullbench fullbench32 : DEBUGFLAGS = -DNDEBUG  # turn off assert() for speed measurements
 fullbench fullbench32 : $(ZSTD_FILES)
-fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c fullbench.c
+fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c fullbench.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
 fullbench-lib : CPPFLAGS += -DXXH_NAMESPACE=ZSTD_
 fullbench-lib : zstd-staticLib
-fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c fullbench.c
+fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c fullbench.c
 	$(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) $(ZSTDDIR)/libzstd.a
 
 # note : broken : requires unavailable symbols
 fullbench-dll : zstd-dll
 fullbench-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
-fullbench-dll: $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c fullbench.c
+fullbench-dll: $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c $(PRGDIR)/timefn.c fullbench.c
 #	$(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll
 	$(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT)
 
@@ -152,32 +152,32 @@ fuzzer  : LDFLAGS += $(MULTITHREAD_LD)
 fuzzer32: CFLAGS += -m32
 fuzzer  : $(ZSTDMT_OBJECTS)
 fuzzer32: $(ZSTD_FILES)
-fuzzer fuzzer32 : $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/datagen.c fuzzer.c
+fuzzer fuzzer32 : $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
 fuzzer-dll : zstd-dll
 fuzzer-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
-fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/datagen.c fuzzer.c
+fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
 
 zbufftest : CPPFLAGS += -I$(ZSTDDIR)/deprecated
 zbufftest : CFLAGS += -Wno-deprecated-declarations   # required to silence deprecation warnings
-zbufftest : $(ZSTD_OBJECTS) $(ZBUFF_FILES) $(PRGDIR)/util.c $(PRGDIR)/datagen.c zbufftest.c
+zbufftest : $(ZSTD_OBJECTS) $(ZBUFF_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c zbufftest.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
 zbufftest32 : CPPFLAGS += -I$(ZSTDDIR)/deprecated
 zbufftest32 : CFLAGS += -Wno-deprecated-declarations -m32
-zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/util.c $(PRGDIR)/datagen.c zbufftest.c
+zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c zbufftest.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
 zbufftest-dll : zstd-dll
 zbufftest-dll : CPPFLAGS += -I$(ZSTDDIR)/deprecated
 zbufftest-dll : CFLAGS += -Wno-deprecated-declarations   # required to silence deprecation warnings
 zbufftest-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
-zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/datagen.c zbufftest.c
+zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c zbufftest.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
 
-ZSTREAM_LOCAL_FILES := $(PRGDIR)/datagen.c $(PRGDIR)/util.c seqgen.c zstreamtest.c
+ZSTREAM_LOCAL_FILES := $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c seqgen.c zstreamtest.c
 ZSTREAM_PROPER_FILES := $(ZDICT_FILES) $(ZSTREAM_LOCAL_FILES)
 ZSTREAMFILES := $(ZSTD_FILES) $(ZSTREAM_PROPER_FILES)
 zstreamtest32 : CFLAGS += -m32
@@ -203,7 +203,7 @@ zstreamtest-dll : $(ZSTREAM_LOCAL_FILES)
 	$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
 
 paramgrill : DEBUGFLAGS =  # turn off assert() by default for speed measurements
-paramgrill : $(ZSTD_FILES) $(PRGDIR)/util.c $(PRGDIR)/benchfn.c $(PRGDIR)/benchzstd.c $(PRGDIR)/datagen.c paramgrill.c
+paramgrill : $(ZSTD_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c $(PRGDIR)/benchzstd.c $(PRGDIR)/datagen.c paramgrill.c
 	$(CC) $(FLAGS) $^ -lm -o $@$(EXT)
 
 datagen : $(PRGDIR)/datagen.c datagencli.c
@@ -222,7 +222,7 @@ legacy : CPPFLAGS += -I$(ZSTDDIR)/legacy -DZSTD_LEGACY_SUPPORT=4
 legacy : $(ZSTD_FILES) $(wildcard $(ZSTDDIR)/legacy/*.c) legacy.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-decodecorpus : $(filter-out zstdc_zstd_compress.o, $(ZSTD_OBJECTS)) $(ZDICT_FILES) $(PRGDIR)/util.c decodecorpus.c
+decodecorpus : $(filter-out zstdc_zstd_compress.o, $(ZSTD_OBJECTS)) $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c decodecorpus.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT) -lm
 
 symbols  : symbols.c zstd-dll
diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c
index b03dc55eabf..d8b332479ed 100644
--- a/tests/decodecorpus.c
+++ b/tests/decodecorpus.c
@@ -16,6 +16,7 @@
 #include 
 
 #include "util.h"
+#include "timefn.h"   /* UTIL_clockSpanMicro, SEC_TO_MICRO, UTIL_TIME_INITIALIZER */
 #include "zstd.h"
 #include "zstd_internal.h"
 #include "mem.h"
diff --git a/tests/fullbench.c b/tests/fullbench.c
index c46535178e3..47494d84f24 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -17,6 +17,7 @@
 #include        /* fprintf, fopen, ftello64 */
 #include       /* assert */
 
+#include "timefn.h"      /* UTIL_clockSpanNano, UTIL_getTime */
 #include "mem.h"         /* U32 */
 #ifndef ZSTD_DLL_IMPORT
     #include "zstd_internal.h"   /* ZSTD_decodeSeqHeaders, ZSTD_blockHeaderSize, blockType_e, KB, MB */
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index c38aef61083..a6d78e6e405 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -38,6 +38,7 @@
 #define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
 #include "xxhash.h"       /* XXH64 */
 #include "util.h"
+#include "timefn.h"       /* SEC_TO_MICRO, UTIL_time_t, UTIL_TIME_INITIALIZER, UTIL_clockSpanMicro, UTIL_getTime */
 
 
 /*-************************************
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 88b7259c865..fb3c776bb21 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -19,6 +19,7 @@
 #include       /* log */
 #include 
 
+#include "timefn.h"    /* SEC_TO_MICRO, UTIL_time_t, UTIL_clockSpanMicro, UTIL_clockSpanNano, UTIL_getTime */
 #include "mem.h"
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_parameters, ZSTD_estimateCCtxSize */
 #include "zstd.h"
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 51cb27b5466..34076f4d629 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -37,6 +37,7 @@
 #include "xxhash.h"       /* XXH64_* */
 #include "seqgen.h"
 #include "util.h"
+#include "timefn.h"       /* UTIL_time_t, UTIL_clockSpanMicro, UTIL_getTime */
 
 
 /*-************************************
diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile
index 0c19107b3e6..d4fc33b51e7 100644
--- a/zlibWrapper/Makefile
+++ b/zlibWrapper/Makefile
@@ -88,7 +88,7 @@ fitblk: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLI
 fitblk_zstd: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(ZSTDLIBRARY)
 	$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@
 
-zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(PROGRAMS_PATH)/util.o $(PROGRAMS_PATH)/datagen.o $(ZSTDLIBRARY)
+zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(PROGRAMS_PATH)/util.o $(PROGRAMS_PATH)/timefn.o $(PROGRAMS_PATH)/datagen.o $(ZSTDLIBRARY)
 	$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@
 
 
diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c
index e071c309852..99f9e11b2be 100644
--- a/zlibWrapper/examples/zwrapbench.c
+++ b/zlibWrapper/examples/zwrapbench.c
@@ -19,6 +19,7 @@
 #include        /* toupper */
 #include        /* errno */
 
+#include "timefn.h"      /* UTIL_time_t, UTIL_getTime, UTIL_clockSpanMicro, UTIL_waitForNextTick */
 #include "mem.h"
 #define ZSTD_STATIC_LINKING_ONLY
 #include "zstd.h"

From 50b9c41196f77c3ff89cfd1d4cd4a81372b80c64 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Wed, 10 Apr 2019 12:34:21 -0700
Subject: [PATCH 150/178] [libzstd] Fix decompression dictionary bugs and clean
 up initialization

Bugs:

* `ZSTD_DCtx_refPrefix()` didn't clear the dictionary after the first
  use. Fix and add a test case.
* `ZSTD_DCtx_reset()` always cleared the dictionary. Fix and add a test
  case.
* After calling `ZSTD_resetDStream()` you could no longer load a
  dictionary, since the stage was set to `zdss_loadHeader`. Fix and add
  a test case.

Cleanup:

* Make `ZSTD_initDStream*()` and `ZSTD_resetDStream()` wrap the new
 advanced API, and add test cases.
* Document the equivalent of these functions in the advanced API and
  document the unstable functions as deprecated.
---
 lib/decompress/zstd_decompress.c          |  79 ++++++++----
 lib/decompress/zstd_decompress_internal.h |   3 +
 lib/zstd.h                                |  36 +++++-
 tests/fuzzer.c                            |  35 +++++
 tests/zstreamtest.c                       | 150 ++++++++++++++++++++++
 5 files changed, 274 insertions(+), 29 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index d8f14882db8..bd955d6ce1a 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -106,6 +106,7 @@ static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
     dctx->ddictLocal  = NULL;
     dctx->dictEnd     = NULL;
     dctx->ddictIsCold = 0;
+    dctx->dictUsesRemaining = 0;
     dctx->inBuff      = NULL;
     dctx->inBuffSize  = 0;
     dctx->outBuffSize = 0;
@@ -147,13 +148,20 @@ ZSTD_DCtx* ZSTD_createDCtx(void)
     return ZSTD_createDCtx_advanced(ZSTD_defaultCMem);
 }
 
+static void ZSTD_clearDict(ZSTD_DCtx* dctx)
+{
+    ZSTD_freeDDict(dctx->ddictLocal);
+    dctx->ddictLocal = NULL;
+    dctx->ddict = NULL;
+    dctx->dictUsesRemaining = 0;
+}
+
 size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
 {
     if (dctx==NULL) return 0;   /* support free on NULL */
     RETURN_ERROR_IF(dctx->staticSize, memory_allocation, "not compatible with static DCtx");
     {   ZSTD_customMem const cMem = dctx->customMem;
-        ZSTD_freeDDict(dctx->ddictLocal);
-        dctx->ddictLocal = NULL;
+        ZSTD_clearDict(dctx);
         ZSTD_free(dctx->inBuff, cMem);
         dctx->inBuff = NULL;
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
@@ -786,9 +794,22 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
 }
 
 
+static ZSTD_DDict const* ZSTD_getDDict(ZSTD_DCtx* dctx)
+{
+    if (dctx->dictUsesRemaining == 0) {
+        ZSTD_clearDict(dctx);
+        return NULL;
+    }
+    if (dctx->dictUsesRemaining < 0) {
+        return dctx->ddict;
+    }
+    --dctx->dictUsesRemaining;
+    return dctx->ddict;
+}
+
 size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
-    return ZSTD_decompress_usingDDict(dctx, dst, dstCapacity, src, srcSize, dctx->ddict);
+    return ZSTD_decompress_usingDDict(dctx, dst, dstCapacity, src, srcSize, ZSTD_getDDict(dctx));
 }
 
 
@@ -1235,14 +1256,13 @@ size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx,
                                          ZSTD_dictContentType_e dictContentType)
 {
     RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong);
-    ZSTD_freeDDict(dctx->ddictLocal);
+    ZSTD_clearDict(dctx);
     if (dict && dictSize >= 8) {
         dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem);
         RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation);
-    } else {
-        dctx->ddictLocal = NULL;
+        dctx->ddict = dctx->ddictLocal;
+        dctx->dictUsesRemaining = -1;
     }
-    dctx->ddict = dctx->ddictLocal;
     return 0;
 }
 
@@ -1258,7 +1278,9 @@ size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSi
 
 size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType)
 {
-    return ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType);
+    FORWARD_IF_ERROR(ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType));
+    dctx->dictUsesRemaining = 1;
+    return 0;
 }
 
 size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize)
@@ -1273,8 +1295,7 @@ size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSiz
 size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize)
 {
     DEBUGLOG(4, "ZSTD_initDStream_usingDict");
-    zds->streamStage = zdss_init;
-    zds->noForwardProgress = 0;
+    FORWARD_IF_ERROR( ZSTD_DCtx_reset(zds, ZSTD_reset_session_only) );
     FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) );
     return ZSTD_FRAMEHEADERSIZE_PREFIX;
 }
@@ -1283,7 +1304,7 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
 size_t ZSTD_initDStream(ZSTD_DStream* zds)
 {
     DEBUGLOG(4, "ZSTD_initDStream");
-    return ZSTD_initDStream_usingDict(zds, NULL, 0);
+    return ZSTD_initDStream_usingDDict(zds, NULL);
 }
 
 /* ZSTD_initDStream_usingDDict() :
@@ -1291,9 +1312,9 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds)
  * this function cannot fail */
 size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict)
 {
-    size_t const initResult = ZSTD_initDStream(dctx);
-    dctx->ddict = ddict;
-    return initResult;
+    FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) );
+    FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) );
+    return ZSTD_FRAMEHEADERSIZE_PREFIX;
 }
 
 /* ZSTD_resetDStream() :
@@ -1301,11 +1322,7 @@ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict)
  * this function cannot fail */
 size_t ZSTD_resetDStream(ZSTD_DStream* dctx)
 {
-    DEBUGLOG(4, "ZSTD_resetDStream");
-    dctx->streamStage = zdss_loadHeader;
-    dctx->lhSize = dctx->inPos = dctx->outStart = dctx->outEnd = 0;
-    dctx->legacyVersion = 0;
-    dctx->hostageByte = 0;
+    FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only));
     return ZSTD_FRAMEHEADERSIZE_PREFIX;
 }
 
@@ -1313,7 +1330,11 @@ size_t ZSTD_resetDStream(ZSTD_DStream* dctx)
 size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
 {
     RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong);
-    dctx->ddict = ddict;
+    ZSTD_clearDict(dctx);
+    if (ddict) {
+        dctx->ddict = ddict;
+        dctx->dictUsesRemaining = -1;
+    }
     return 0;
 }
 
@@ -1394,11 +1415,13 @@ size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset)
 {
     if ( (reset == ZSTD_reset_session_only)
       || (reset == ZSTD_reset_session_and_parameters) ) {
-        (void)ZSTD_initDStream(dctx);
+        dctx->streamStage = zdss_init;
+        dctx->noForwardProgress = 0;
     }
     if ( (reset == ZSTD_reset_parameters)
       || (reset == ZSTD_reset_session_and_parameters) ) {
         RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong);
+        ZSTD_clearDict(dctx);
         dctx->format = ZSTD_f_zstd1;
         dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
     }
@@ -1481,7 +1504,10 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
         {
         case zdss_init :
             DEBUGLOG(5, "stage zdss_init => transparent reset ");
-            ZSTD_resetDStream(zds);   /* transparent reset on starting decoding a new frame */
+            zds->streamStage = zdss_loadHeader;
+            zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
+            zds->legacyVersion = 0;
+            zds->hostageByte = 0;
             /* fall-through */
 
         case zdss_loadHeader :
@@ -1501,8 +1527,9 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
                     U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
                     if (legacyVersion) {
-                        const void* const dict = zds->ddict ? ZSTD_DDict_dictContent(zds->ddict) : NULL;
-                        size_t const dictSize = zds->ddict ? ZSTD_DDict_dictSize(zds->ddict) : 0;
+                        ZSTD_DDict const* const ddict = ZSTD_getDDict(zds);
+                        const void* const dict = ddict ? ZSTD_DDict_dictContent(ddict) : NULL;
+                        size_t const dictSize = ddict ? ZSTD_DDict_dictSize(ddict) : 0;
                         DEBUGLOG(5, "ZSTD_decompressStream: detected legacy version v0.%u", legacyVersion);
                         RETURN_ERROR_IF(zds->staticSize, memory_allocation,
                             "legacy support is incompatible with static dctx");
@@ -1540,7 +1567,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                 size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart);
                 if (cSize <= (size_t)(iend-istart)) {
                     /* shortcut : using single-pass mode */
-                    size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, oend-op, istart, cSize, zds->ddict);
+                    size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, oend-op, istart, cSize, ZSTD_getDDict(zds));
                     if (ZSTD_isError(decompressedSize)) return decompressedSize;
                     DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()")
                     ip = istart + cSize;
@@ -1553,7 +1580,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
 
             /* Consume header (see ZSTDds_decodeFrameHeader) */
             DEBUGLOG(4, "Consume header");
-            FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, zds->ddict));
+            FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds)));
 
             if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {  /* skippable frame */
                 zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE);
diff --git a/lib/decompress/zstd_decompress_internal.h b/lib/decompress/zstd_decompress_internal.h
index abd0030519d..ea07243832c 100644
--- a/lib/decompress/zstd_decompress_internal.h
+++ b/lib/decompress/zstd_decompress_internal.h
@@ -123,6 +123,9 @@ struct ZSTD_DCtx_s
     const ZSTD_DDict* ddict;     /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */
     U32 dictID;
     int ddictIsCold;             /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */
+    int dictUsesRemaining;       /* if == 1 : dictionary should be used once.
+                                  * if == 0 : dictionary should be forgotten now.
+                                  * if < 0 : dictionary should be used indefinitely. */
 
     /* streaming */
     ZSTD_dStreamStage streamStage;
diff --git a/lib/zstd.h b/lib/zstd.h
index 0c9ebe5b61f..cc87628b164 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -722,7 +722,14 @@ ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
 ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
 
 /*===== Streaming decompression functions =====*/
+
+/* This function is redundant with the advanced API and equivalent to:
+ *
+ *     ZSTD_DCtx_reset(zds);
+ *     ZSTD_DCtx_refDDict(zds, NULL);
+ */
 ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
+
 ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
 
 ZSTDLIB_API size_t ZSTD_DStreamInSize(void);    /*!< recommended size for input buffer */
@@ -1672,9 +1679,32 @@ ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
 
 
 /*=====   Advanced Streaming decompression functions  =====*/
-ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: no dictionary will be used if dict == NULL or dictSize < 8 */
-ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);  /**< note : ddict is referenced, it must outlive decompression session */
-ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);  /**< re-use decompression parameters from previous init; saves dictionary loading */
+/**
+ * This function is deprecated, and is equivalent to:
+ *
+ *     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
+ *     ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
+ *
+ * note: no dictionary will be used if dict == NULL or dictSize < 8
+ */
+ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
+/**
+ * This function is deprecated, and is equivalent to:
+ *
+ *     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
+ *     ZSTD_DCtx_refDDict(zds, ddict);
+ *
+ * note : ddict is referenced, it must outlive decompression session
+ */
+ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
+/**
+ * This function is deprecated, and is equivalent to:
+ *
+ *     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
+ *
+ * re-use decompression parameters from previous init; saves dictionary loading
+ */
+ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
 
 
 /*********************************************************************
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index c38aef61083..01f87e6edc8 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1464,6 +1464,41 @@ static int basicUnitTests(U32 seed, double compressibility)
         }
         DISPLAYLEVEL(3, "OK \n");
 
+        DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with ddict : ", testNb++);
+        {
+            ZSTD_DCtx* dctx = ZSTD_createDCtx();
+            ZSTD_DDict* ddict = ZSTD_createDDict(dictBuffer, dictSize);
+            size_t ret;
+            /* We should succeed to decompress with the ddict. */
+            ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
+            CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddict) );
+            CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
+            /* The ddict should presist across calls. */
+            CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
+            /* When we reset the context the ddict is cleared. */
+            ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
+            ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
+            if (!ZSTD_isError(ret)) goto _output_error;
+            ZSTD_freeDCtx(dctx);
+            ZSTD_freeDDict(ddict);
+        }
+        DISPLAYLEVEL(3, "OK \n");
+
+        DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with prefix : ", testNb++);
+        {
+            ZSTD_DCtx* dctx = ZSTD_createDCtx();
+            size_t ret;
+            /* We should succeed to decompress with the prefix. */
+            ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
+            CHECK_Z( ZSTD_DCtx_refPrefix_advanced(dctx, dictBuffer, dictSize, ZSTD_dct_auto) );
+            CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
+            /* The prefix should be cleared after the first compression. */
+            ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
+            if (!ZSTD_isError(ret)) goto _output_error;
+            ZSTD_freeDCtx(dctx);
+        }
+        DISPLAYLEVEL(3, "OK \n");
+
         DISPLAYLEVEL(3, "test%3i : Dictionary with non-default repcodes : ", testNb++);
         { U32 u; for (u=0; u
Date: Wed, 10 Apr 2019 13:26:27 -0700
Subject: [PATCH 151/178] tried a blindfix for unix + c11

---
 programs/platform.h |  5 +++--
 programs/timefn.c   |  9 +++------
 programs/timefn.h   | 27 ++++++++++-----------------
 3 files changed, 16 insertions(+), 25 deletions(-)

diff --git a/programs/platform.h b/programs/platform.h
index 1a8f97bc4df..38ded872743 100644
--- a/programs/platform.h
+++ b/programs/platform.h
@@ -87,8 +87,8 @@ extern "C" {
  * The following list of build macros tries to "guess" if target OS is likely unix-like, and therefore can #include 
  */
 #  elif !defined(_WIN32) \
-     && (defined(__unix__) || defined(__unix) \
-     || defined(__midipix__) || defined(__VMS) || defined(__HAIKU__))
+     && ( defined(__unix__) || defined(__unix) \
+       || defined(__midipix__) || defined(__VMS) || defined(__HAIKU__) )
 
 #    if defined(__linux__) || defined(__linux)
 #      ifndef _POSIX_C_SOURCE
@@ -108,6 +108,7 @@ extern "C" {
 
 #endif   /* PLATFORM_POSIX_VERSION */
 
+
 /*-*********************************************
 *  Detect if isatty() and fileno() are available
 ************************************************/
diff --git a/programs/timefn.c b/programs/timefn.c
index ad24769582f..efb8156b6f7 100644
--- a/programs/timefn.c
+++ b/programs/timefn.c
@@ -72,11 +72,8 @@ PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
     return ((clockEnd - clockStart) * (PTime)rate.numer) / ((PTime)rate.denom);
 }
 
-#elif (PLATFORM_POSIX_VERSION >= 200112L) \
-   && (defined(__UCLIBC__)                \
-      || (defined(__GLIBC__)              \
-          && ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 17) \
-             || (__GLIBC__ > 2))))
+#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \
+    && defined (CLOCK_MONOTONIC)
 
 UTIL_time_t UTIL_getTime(void)
 {
@@ -86,7 +83,7 @@ UTIL_time_t UTIL_getTime(void)
     return time;
 }
 
-UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end)
+static UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end)
 {
     UTIL_time_t diff;
     if (end.tv_nsec < begin.tv_nsec) {
diff --git a/programs/timefn.h b/programs/timefn.h
index 7892a692114..290da884e48 100644
--- a/programs/timefn.h
+++ b/programs/timefn.h
@@ -35,9 +35,9 @@ extern "C" {
 
 #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
 # include 
-  typedef uint64_t PTime;  /* Precise Time */
+  typedef uint64_t           PTime;  /* Precise Time */
 #else
-  typedef unsigned long long PTime;   /* does not support compilers without long long support */
+  typedef unsigned long long PTime;  /* does not support compilers without long long support */
 #endif
 
 
@@ -47,42 +47,35 @@ extern "C" {
 ******************************************/
 #if defined(_WIN32)   /* Windows */
 
-    #define UTIL_TIME_INITIALIZER { { 0, 0 } }
     typedef LARGE_INTEGER UTIL_time_t;
+    #define UTIL_TIME_INITIALIZER { { 0, 0 } }
 
 #elif defined(__APPLE__) && defined(__MACH__)
 
     #include 
-    #define UTIL_TIME_INITIALIZER 0
     typedef PTime UTIL_time_t;
+    #define UTIL_TIME_INITIALIZER 0
 
-#elif (PLATFORM_POSIX_VERSION >= 200112L) \
-   && (defined(__UCLIBC__)                \
-      || (defined(__GLIBC__)              \
-          && ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 17) \
-             || (__GLIBC__ > 2))))
+#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \
+    && defined (CLOCK_MONOTONIC)
 
+    typedef struct timespec UTIL_time_t;   /* C11 defines struct timespes within time.h */
     #define UTIL_TIME_INITIALIZER { 0, 0 }
-    typedef struct timespec UTIL_freq_t;
-    typedef struct timespec UTIL_time_t;
 
-    UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end);
-
-#else   /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */
+#else   /* relies on standard C90 (note : clock_t measurements can be wrong when using multi-threading) */
 
     typedef clock_t UTIL_time_t;
     #define UTIL_TIME_INITIALIZER 0
 
 #endif
 
+
 UTIL_time_t UTIL_getTime(void);
 PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd);
 PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd);
 
-
-#define SEC_TO_MICRO 1000000
+#define SEC_TO_MICRO ((PTime)1000000)
 PTime UTIL_clockSpanMicro(UTIL_time_t clockStart);
-
 PTime UTIL_clockSpanNano(UTIL_time_t clockStart);
 
 void UTIL_waitForNextTick(void);

From 70802cde6dbddde9d6fbb2586a091a2f3e7f86db Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 10 Apr 2019 14:01:18 -0700
Subject: [PATCH 152/178] fixed error message

using stdlib's perror()
---
 programs/timefn.c | 30 +++++++++++++++++++++++-------
 1 file changed, 23 insertions(+), 7 deletions(-)

diff --git a/programs/timefn.c b/programs/timefn.c
index efb8156b6f7..4b3b23839b1 100644
--- a/programs/timefn.c
+++ b/programs/timefn.c
@@ -11,6 +11,8 @@
 
 /* ===  Dependencies  === */
 
+#include    /* perror, abort */
+
 #include "timefn.h"
 
 
@@ -27,8 +29,10 @@ PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
     static LARGE_INTEGER ticksPerSecond;
     static int init = 0;
     if (!init) {
-        if (!QueryPerformanceFrequency(&ticksPerSecond))
-            UTIL_DISPLAYLEVEL(1, "ERROR: QueryPerformanceFrequency() failure\n");
+        if (!QueryPerformanceFrequency(&ticksPerSecond)) {
+            perror("timefn::QueryPerformanceFrequency");
+            abort();
+        }
         init = 1;
     }
     return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
@@ -39,13 +43,17 @@ PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
     static LARGE_INTEGER ticksPerSecond;
     static int init = 0;
     if (!init) {
-        if (!QueryPerformanceFrequency(&ticksPerSecond))
-            UTIL_DISPLAYLEVEL(1, "ERROR: QueryPerformanceFrequency() failure\n");
+        if (!QueryPerformanceFrequency(&ticksPerSecond)) {
+            perror("timefn::QueryPerformanceFrequency");
+            abort();
+        }
         init = 1;
     }
     return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
 }
 
+
+
 #elif defined(__APPLE__) && defined(__MACH__)
 
 UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); }
@@ -72,14 +80,18 @@ PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
     return ((clockEnd - clockStart) * (PTime)rate.numer) / ((PTime)rate.denom);
 }
 
+
+
 #elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \
     && defined (CLOCK_MONOTONIC)
 
 UTIL_time_t UTIL_getTime(void)
 {
     UTIL_time_t time;
-    if (clock_gettime(CLOCK_MONOTONIC, &time))
-        UTIL_DISPLAYLEVEL(1, "ERROR: Failed to get time\n");   /* we could also exit() */
+    if (clock_gettime(CLOCK_MONOTONIC, &time)) {
+        perror("timefb::clock_gettime");
+        abort();
+    }
     return time;
 }
 
@@ -114,7 +126,9 @@ PTime UTIL_getSpanTimeNano(UTIL_time_t begin, UTIL_time_t end)
     return nano;
 }
 
-#else   /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */
+
+
+#else   /* relies on standard C90 (note : clock_t measurements can be wrong when using multi-threading) */
 
 UTIL_time_t UTIL_getTime(void) { return clock(); }
 PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
@@ -122,6 +136,8 @@ PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) { retur
 
 #endif
 
+
+
 /* returns time span in microseconds */
 PTime UTIL_clockSpanMicro(UTIL_time_t clockStart )
 {

From 4765929271fbd3a2e60b3558225d0b7fe6527095 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 10 Apr 2019 14:04:11 -0700
Subject: [PATCH 153/178] fixed perror include

---
 programs/timefn.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/programs/timefn.c b/programs/timefn.c
index 4b3b23839b1..cb545ad67d9 100644
--- a/programs/timefn.c
+++ b/programs/timefn.c
@@ -11,7 +11,8 @@
 
 /* ===  Dependencies  === */
 
-#include    /* perror, abort */
+#include    /* abort */
+#include     /* perror */
 
 #include "timefn.h"
 

From 36d2dfd846e1e33cc149d4f6da78e785faa6a865 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 10 Apr 2019 14:15:11 -0700
Subject: [PATCH 154/178] moved C11 code path to timespec_get

---
 programs/timefn.c | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/programs/timefn.c b/programs/timefn.c
index cb545ad67d9..8c6010c9035 100644
--- a/programs/timefn.c
+++ b/programs/timefn.c
@@ -11,9 +11,6 @@
 
 /* ===  Dependencies  === */
 
-#include    /* abort */
-#include     /* perror */
-
 #include "timefn.h"
 
 
@@ -23,6 +20,9 @@
 
 #if defined(_WIN32)   /* Windows */
 
+#include    /* abort */
+#include     /* perror */
+
 UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; }
 
 PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
@@ -83,14 +83,16 @@ PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
 
 
 
-#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \
-    && defined (CLOCK_MONOTONIC)
+#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */)
+
+#include    /* abort */
+#include     /* perror */
 
 UTIL_time_t UTIL_getTime(void)
 {
     UTIL_time_t time;
-    if (clock_gettime(CLOCK_MONOTONIC, &time)) {
-        perror("timefb::clock_gettime");
+    if (timespec_get(&time, TIME_UTC) == 0) {
+        perror("timefn::timespec_get");
         abort();
     }
     return time;

From 3d346579d8f622c65a2b0523c64563759403c4e3 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 10 Apr 2019 14:16:39 -0700
Subject: [PATCH 155/178] no more need for CLOCK_MONOTONIC

---
 programs/timefn.h | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/programs/timefn.h b/programs/timefn.h
index 290da884e48..6e5d456d7bb 100644
--- a/programs/timefn.h
+++ b/programs/timefn.h
@@ -56,8 +56,7 @@ extern "C" {
     typedef PTime UTIL_time_t;
     #define UTIL_TIME_INITIALIZER 0
 
-#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \
-    && defined (CLOCK_MONOTONIC)
+#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) 
 
     typedef struct timespec UTIL_time_t;   /* C11 defines struct timespes within time.h */
     #define UTIL_TIME_INITIALIZER { 0, 0 }

From 2c6b14ed222b02db35096c2575b7f6b7652ad4ca Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 10 Apr 2019 14:54:13 -0700
Subject: [PATCH 156/178] fixed Windows header

cmake build script: added timefn
---
 build/cmake/programs/CMakeLists.txt | 4 ++--
 programs/timefn.h                   | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/build/cmake/programs/CMakeLists.txt b/build/cmake/programs/CMakeLists.txt
index 4c5146f8d2f..f6f7a3616f4 100644
--- a/build/cmake/programs/CMakeLists.txt
+++ b/build/cmake/programs/CMakeLists.txt
@@ -26,7 +26,7 @@ if (MSVC)
     set(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstd.rc)
 endif ()
 
-add_executable(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources})
+add_executable(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources})
 target_link_libraries(zstd libzstd_static)
 if (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)")
     target_link_libraries(zstd rt)
@@ -63,7 +63,7 @@ if (UNIX)
     install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdgrep.1 DESTINATION "${MAN_INSTALL_DIR}")
     install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdless.1 DESTINATION "${MAN_INSTALL_DIR}")
 
-    add_executable(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/fileio.c)
+    add_executable(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c)
     target_link_libraries(zstd-frugal libzstd_static)
     set_property(TARGET zstd-frugal APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT")
 endif ()
diff --git a/programs/timefn.h b/programs/timefn.h
index 6e5d456d7bb..810d6b2623f 100644
--- a/programs/timefn.h
+++ b/programs/timefn.h
@@ -47,6 +47,7 @@ extern "C" {
 ******************************************/
 #if defined(_WIN32)   /* Windows */
 
+    #include    /* LARGE_INTEGER */
     typedef LARGE_INTEGER UTIL_time_t;
     #define UTIL_TIME_INITIALIZER { { 0, 0 } }
 
@@ -56,7 +57,7 @@ extern "C" {
     typedef PTime UTIL_time_t;
     #define UTIL_TIME_INITIALIZER 0
 
-#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) 
+#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */)
 
     typedef struct timespec UTIL_time_t;   /* C11 defines struct timespes within time.h */
     #define UTIL_TIME_INITIALIZER { 0, 0 }

From 2fa4f2e246b0e4e3a07570a31e89ddd9410bec6a Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 10 Apr 2019 15:07:36 -0700
Subject: [PATCH 157/178] updated Visual projects

added timefn
---
 build/VS2008/fullbench/fullbench.vcproj          | 4 ++++
 build/VS2008/fuzzer/fuzzer.vcproj                | 4 ++++
 build/VS2008/zstd/zstd.vcproj                    | 4 ++++
 build/VS2010/fullbench-dll/fullbench-dll.vcxproj | 1 +
 build/VS2010/fullbench/fullbench.vcxproj         | 3 ++-
 build/VS2010/fuzzer/fuzzer.vcxproj               | 1 +
 build/VS2010/zstd/zstd.vcxproj                   | 1 +
 7 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/build/VS2008/fullbench/fullbench.vcproj b/build/VS2008/fullbench/fullbench.vcproj
index 996ee26a916..2246262df80 100644
--- a/build/VS2008/fullbench/fullbench.vcproj
+++ b/build/VS2008/fullbench/fullbench.vcproj
@@ -408,6 +408,10 @@
 				RelativePath="..\..\..\programs\util.c"
 				>
 			
+			
+			
 			
diff --git a/build/VS2008/fuzzer/fuzzer.vcproj b/build/VS2008/fuzzer/fuzzer.vcproj
index 25924c8b7f8..46cc4d4f244 100644
--- a/build/VS2008/fuzzer/fuzzer.vcproj
+++ b/build/VS2008/fuzzer/fuzzer.vcproj
@@ -332,6 +332,10 @@
 				RelativePath="..\..\..\programs\util.c"
 				>
 			
+			
+			
 			
diff --git a/build/VS2008/zstd/zstd.vcproj b/build/VS2008/zstd/zstd.vcproj
index d738be565f2..745f2e875fa 100644
--- a/build/VS2008/zstd/zstd.vcproj
+++ b/build/VS2008/zstd/zstd.vcproj
@@ -336,6 +336,10 @@
 				RelativePath="..\..\..\programs\util.c"
 				>
 			
+			
+			
 			
diff --git a/build/VS2010/fullbench-dll/fullbench-dll.vcxproj b/build/VS2010/fullbench-dll/fullbench-dll.vcxproj
index 29762860fe1..befdc044513 100644
--- a/build/VS2010/fullbench-dll/fullbench-dll.vcxproj
+++ b/build/VS2010/fullbench-dll/fullbench-dll.vcxproj
@@ -167,6 +167,7 @@
   
     
     
+    
     
     
     
diff --git a/build/VS2010/fullbench/fullbench.vcxproj b/build/VS2010/fullbench/fullbench.vcxproj
index a3a884a7b0f..57ee337160c 100644
--- a/build/VS2010/fullbench/fullbench.vcxproj
+++ b/build/VS2010/fullbench/fullbench.vcxproj
@@ -156,7 +156,6 @@
   
   
     
-    
     
     
     
@@ -178,6 +177,8 @@
     
     
     
+    
+    
     
     
     
diff --git a/build/VS2010/fuzzer/fuzzer.vcxproj b/build/VS2010/fuzzer/fuzzer.vcxproj
index 106dcf99c70..53881c194b8 100644
--- a/build/VS2010/fuzzer/fuzzer.vcxproj
+++ b/build/VS2010/fuzzer/fuzzer.vcxproj
@@ -182,6 +182,7 @@
     
     
     
+    
     
     
   
diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj
index 6681e581d74..6e7ddca1e33 100644
--- a/build/VS2010/zstd/zstd.vcxproj
+++ b/build/VS2010/zstd/zstd.vcxproj
@@ -53,6 +53,7 @@
     
     
     
+    
     
     
     

From 885476fb5b30e5bf3adac6bb35bbd07ebe15d1e2 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 10 Apr 2019 15:22:18 -0700
Subject: [PATCH 158/178] FreeBSD_11 specific fix

C11 mandates the definition of timespec_get() and TIME_UTC.
However, FreeBSD11 announce C11 compliance, but does not provifr timespec_get(),
breaking link stage for benchfn.
Since it does not provide TIME_UTC either, which is also required by C11,
test this macro: this will automatically rule out FreeBSD 11 for this code path
(it will use the backup C90 path instead, based on clock_t).

The issue seeems fixed in FreeBSD 12.
---
 programs/timefn.c | 3 ++-
 programs/timefn.h | 5 +++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/programs/timefn.c b/programs/timefn.c
index 8c6010c9035..21f2a9618b4 100644
--- a/programs/timefn.c
+++ b/programs/timefn.c
@@ -83,7 +83,8 @@ PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
 
 
 
-#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */)
+#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \
+    && defined(TIME_UTC) /* C11 requires timespec_get, but FreeBSD 11 lacks it, while still claiming C11 compliance */
 
 #include    /* abort */
 #include     /* perror */
diff --git a/programs/timefn.h b/programs/timefn.h
index 810d6b2623f..d1ddd31b1c0 100644
--- a/programs/timefn.h
+++ b/programs/timefn.h
@@ -57,9 +57,10 @@ extern "C" {
     typedef PTime UTIL_time_t;
     #define UTIL_TIME_INITIALIZER 0
 
-#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */)
+#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \
+    && defined(TIME_UTC) /* C11 requires timespec_get, but FreeBSD 11 lacks it, while still claiming C11 compliance */
 
-    typedef struct timespec UTIL_time_t;   /* C11 defines struct timespes within time.h */
+    typedef struct timespec UTIL_time_t;
     #define UTIL_TIME_INITIALIZER { 0, 0 }
 
 #else   /* relies on standard C90 (note : clock_t measurements can be wrong when using multi-threading) */

From 9703a5912173982f9e9f14681f428afe18a4d828 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 10 Apr 2019 15:54:55 -0700
Subject: [PATCH 159/178] fixed minor conversion warning

---
 programs/benchfn.c | 3 ++-
 tests/fullbench.c  | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/programs/benchfn.c b/programs/benchfn.c
index 8ae619536a4..fda09935673 100644
--- a/programs/benchfn.c
+++ b/programs/benchfn.c
@@ -167,7 +167,8 @@ void BMK_freeTimedFnState(BMK_timedFnState_t* state) {
     free(state);
 }
 
-BMK_timedFnState_t* BMK_initStatic_timedFnState(void* buffer, size_t size, unsigned total_ms, unsigned run_ms)
+BMK_timedFnState_t*
+BMK_initStatic_timedFnState(void* buffer, size_t size, unsigned total_ms, unsigned run_ms)
 {
     enum { timedFnState_staticSize_isLargeEnough=(1/(sizeof(BMK_timedFnState_shell) >= sizeof(struct BMK_timedFnState_s))) };  /* static assert */
     typedef struct { char c; long long ll; } ll_align;  /* this will force ll to be aligned at its next best position */
diff --git a/tests/fullbench.c b/tests/fullbench.c
index 47494d84f24..6e42d2103fb 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -497,7 +497,7 @@ static int benchMem(unsigned benchNb,
         BMK_benchParams_t bp;
         BMK_runTime_t bestResult;
         bestResult.sumOfReturn = 0;
-        bestResult.nanoSecPerRun = (unsigned long long)(-1LL);
+        bestResult.nanoSecPerRun = (double)(-1);
         assert(tfs != NULL);
 
         bp.benchFn = benchFunction;

From 526ec646b72e56f11af3bffa5c59265245a3579c Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 10 Apr 2019 16:05:02 -0700
Subject: [PATCH 160/178] alternate static assert

to circumvent Visual's C4804 warning
---
 programs/benchfn.c | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/programs/benchfn.c b/programs/benchfn.c
index fda09935673..0932d155de4 100644
--- a/programs/benchfn.c
+++ b/programs/benchfn.c
@@ -163,19 +163,18 @@ BMK_timedFnState_t* BMK_createTimedFnState(unsigned total_ms, unsigned run_ms)
     return r;
 }
 
-void BMK_freeTimedFnState(BMK_timedFnState_t* state) {
-    free(state);
-}
+void BMK_freeTimedFnState(BMK_timedFnState_t* state) { free(state); }
 
 BMK_timedFnState_t*
 BMK_initStatic_timedFnState(void* buffer, size_t size, unsigned total_ms, unsigned run_ms)
 {
-    enum { timedFnState_staticSize_isLargeEnough=(1/(sizeof(BMK_timedFnState_shell) >= sizeof(struct BMK_timedFnState_s))) };  /* static assert */
-    typedef struct { char c; long long ll; } ll_align;  /* this will force ll to be aligned at its next best position */
-    size_t const ll_alignment = offsetof(ll_align, ll); /* provides the minimal alignment restriction for long long */
+    typedef char check_size[ 2 * (sizeof(BMK_timedFnState_shell) >= sizeof(struct BMK_timedFnState_s)) - 1];  /* static assert : a compilation failure indicates that BMK_timedFnState_shell is not large enough */
+    typedef struct { check_size c; BMK_timedFnState_t tfs; } tfs_align;  /* force tfs to be aligned at its next best position */
+    size_t const tfs_alignment = offsetof(tfs_align, tfs); /* provides the minimal alignment restriction for BMK_timedFnState_t */
     BMK_timedFnState_t* const r = (BMK_timedFnState_t*)buffer;
+    if (buffer == NULL) return NULL;
     if (size < sizeof(struct BMK_timedFnState_s)) return NULL;
-    if ((size_t)buffer % ll_alignment) return NULL;  /* must be aligned to satisfy `long long` alignment requirement */
+    if ((size_t)buffer % tfs_alignment) return NULL;  /* buffer must be properly aligned */
     BMK_resetTimedFnState(r, total_ms, run_ms);
     return r;
 }

From aafe97b67d30ec79609fac5ab4b228e89cda016b Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Wed, 10 Apr 2019 16:50:35 -0700
Subject: [PATCH 161/178] [libzstd] Switch dictUses to an enum

---
 lib/decompress/zstd_decompress.c          | 24 +++++++++++++----------
 lib/decompress/zstd_decompress_internal.h | 10 +++++++---
 2 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index bd955d6ce1a..4f2922790e2 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -106,7 +106,7 @@ static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
     dctx->ddictLocal  = NULL;
     dctx->dictEnd     = NULL;
     dctx->ddictIsCold = 0;
-    dctx->dictUsesRemaining = 0;
+    dctx->dictUses = ZSTD_dont_use;
     dctx->inBuff      = NULL;
     dctx->inBuffSize  = 0;
     dctx->outBuffSize = 0;
@@ -153,7 +153,7 @@ static void ZSTD_clearDict(ZSTD_DCtx* dctx)
     ZSTD_freeDDict(dctx->ddictLocal);
     dctx->ddictLocal = NULL;
     dctx->ddict = NULL;
-    dctx->dictUsesRemaining = 0;
+    dctx->dictUses = ZSTD_dont_use;
 }
 
 size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
@@ -796,15 +796,19 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
 
 static ZSTD_DDict const* ZSTD_getDDict(ZSTD_DCtx* dctx)
 {
-    if (dctx->dictUsesRemaining == 0) {
+    switch (dctx->dictUses) {
+    default:
+        assert(0 /* Impossible */);
+        /* fall-through */
+    case ZSTD_dont_use:
         ZSTD_clearDict(dctx);
         return NULL;
-    }
-    if (dctx->dictUsesRemaining < 0) {
+    case ZSTD_use_indefinitely:
+        return dctx->ddict;
+    case ZSTD_use_once:
+        dctx->dictUses = ZSTD_dont_use;
         return dctx->ddict;
     }
-    --dctx->dictUsesRemaining;
-    return dctx->ddict;
 }
 
 size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
@@ -1261,7 +1265,7 @@ size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx,
         dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem);
         RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation);
         dctx->ddict = dctx->ddictLocal;
-        dctx->dictUsesRemaining = -1;
+        dctx->dictUses = ZSTD_use_indefinitely;
     }
     return 0;
 }
@@ -1279,7 +1283,7 @@ size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSi
 size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType)
 {
     FORWARD_IF_ERROR(ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType));
-    dctx->dictUsesRemaining = 1;
+    dctx->dictUses = ZSTD_use_once;
     return 0;
 }
 
@@ -1333,7 +1337,7 @@ size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
     ZSTD_clearDict(dctx);
     if (ddict) {
         dctx->ddict = ddict;
-        dctx->dictUsesRemaining = -1;
+        dctx->dictUses = ZSTD_use_indefinitely;
     }
     return 0;
 }
diff --git a/lib/decompress/zstd_decompress_internal.h b/lib/decompress/zstd_decompress_internal.h
index ea07243832c..ccbdfa090fa 100644
--- a/lib/decompress/zstd_decompress_internal.h
+++ b/lib/decompress/zstd_decompress_internal.h
@@ -89,6 +89,12 @@ typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
 typedef enum { zdss_init=0, zdss_loadHeader,
                zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage;
 
+typedef enum {
+    ZSTD_use_indefinitely = -1,  /* Use the dictionary indefinitely */
+    ZSTD_dont_use = 0,           /* Do not use the dictionary (if one exists free it) */
+    ZSTD_use_once = 1            /* Use the dictionary once and set to ZSTD_dont_use */
+} ZSTD_dictUses_e;
+
 struct ZSTD_DCtx_s
 {
     const ZSTD_seqSymbol* LLTptr;
@@ -123,9 +129,7 @@ struct ZSTD_DCtx_s
     const ZSTD_DDict* ddict;     /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */
     U32 dictID;
     int ddictIsCold;             /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */
-    int dictUsesRemaining;       /* if == 1 : dictionary should be used once.
-                                  * if == 0 : dictionary should be forgotten now.
-                                  * if < 0 : dictionary should be used indefinitely. */
+    ZSTD_dictUses_e dictUses;
 
     /* streaming */
     ZSTD_dStreamStage streamStage;

From fbdd30d68ee2b41f51ee71bcbbf3066290084a6c Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 10 Apr 2019 17:47:01 -0700
Subject: [PATCH 162/178] fixed cmake build script for test programs

---
 build/cmake/tests/CMakeLists.txt | 6 +++---
 programs/Makefile                | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/build/cmake/tests/CMakeLists.txt b/build/cmake/tests/CMakeLists.txt
index 69316e28003..077d824b52d 100644
--- a/build/cmake/tests/CMakeLists.txt
+++ b/build/cmake/tests/CMakeLists.txt
@@ -43,13 +43,13 @@ include_directories(${TESTS_DIR} ${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/c
 add_executable(datagen ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/datagencli.c)
 target_link_libraries(datagen libzstd_static)
 
-add_executable(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${TESTS_DIR}/fullbench.c)
+add_executable(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${TESTS_DIR}/fullbench.c)
 target_link_libraries(fullbench libzstd_static)
 
-add_executable(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${TESTS_DIR}/fuzzer.c)
+add_executable(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/fuzzer.c)
 target_link_libraries(fuzzer libzstd_static)
 
 if (UNIX)
-    add_executable(paramgrill ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${TESTS_DIR}/paramgrill.c)
+    add_executable(paramgrill ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/paramgrill.c)
     target_link_libraries(paramgrill libzstd_static m) #m is math library
 endif ()
diff --git a/programs/Makefile b/programs/Makefile
index 5ceeff1504f..64dcae0028e 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -265,9 +265,9 @@ man: zstd.1 zstdgrep.1 zstdless.1
 
 .PHONY: clean-man
 clean-man:
-	rm zstd.1
-	rm zstdgrep.1
-	rm zstdless.1
+	$(RM) zstd.1
+	$(RM) zstdgrep.1
+	$(RM) zstdless.1
 
 .PHONY: preview-man
 preview-man: clean-man man

From 30c26ab7262b37a2825ff41b54209c69b5ab134b Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 10 Apr 2019 17:48:56 -0700
Subject: [PATCH 163/178] fixed minor warning

unused variable when assert() turned off in fileio.c
---
 programs/fileio.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index b0cea5eacd4..30514d41bf1 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1547,10 +1547,12 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
     return storedSkips;
 }
 
-static void FIO_fwriteSparseEnd(FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips)
+static void
+FIO_fwriteSparseEnd(FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips)
 {
     if (storedSkips>0) {
         assert(prefs->sparseFileSupport > 0);  /* storedSkips>0 implies sparse support is enabled */
+        (void)prefs;   /* assert can be disabled, in which case prefs becomes unused */
         if (LONG_SEEK(file, storedSkips-1, SEEK_CUR) != 0)
             EXM_THROW(69, "Final skip error (sparse file support)");
         /* last zero must be explicitly written,

From 8f56fa2f586e7cc9869f6e9cfdb795eaa501ee32 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 11 Apr 2019 09:50:39 -0700
Subject: [PATCH 164/178] fixed poolTests

added poolTests to all
---
 tests/Makefile    | 4 ++--
 tests/poolTests.c | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/Makefile b/tests/Makefile
index 695a49b9959..f11b731835e 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -82,7 +82,7 @@ default: fullbench
 	@echo $(ZSTDMT_OBJECTS)
 
 all: fullbench fuzzer zstreamtest paramgrill datagen decodecorpus roundTripCrash \
-     fullbench-lib
+     fullbench-lib poolTests
 
 all32: fullbench32 fuzzer32 zstreamtest32
 
@@ -233,7 +233,7 @@ else
 	$(CC) $(FLAGS) $< -o $@$(EXT) -Wl,-rpath=$(ZSTDDIR) $(ZSTDDIR)/libzstd.so   # broken on Mac
 endif
 
-poolTests : $(PRGDIR)/util.c poolTests.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/error_private.c
+poolTests : $(PRGDIR)/util.c $(PRGDIR)/timefn.c poolTests.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/error_private.c
 	$(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT)
 
 .PHONY: versionsTest
diff --git a/tests/poolTests.c b/tests/poolTests.c
index 8b9a4700229..dbfe5096b17 100644
--- a/tests/poolTests.c
+++ b/tests/poolTests.c
@@ -12,6 +12,7 @@
 #include "pool.h"
 #include "threading.h"
 #include "util.h"
+#include "timefn.h"
 #include 
 #include 
 

From f8e9bec73abf10a5e886305356f16e7dd8e5857c Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 11 Apr 2019 12:03:42 -0700
Subject: [PATCH 165/178] fixed poolTests on Windows

must use ZSTD_ prefix in front of pthread types
so that they get properly translated for Windows.
---
 tests/poolTests.c | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/tests/poolTests.c b/tests/poolTests.c
index dbfe5096b17..9b244afed09 100644
--- a/tests/poolTests.c
+++ b/tests/poolTests.c
@@ -26,12 +26,13 @@
 #define ASSERT_EQ(lhs, rhs) ASSERT_TRUE((lhs) == (rhs))
 
 struct data {
-  pthread_mutex_t mutex;
+  ZSTD_pthread_mutex_t mutex;
   unsigned data[16];
   size_t i;
 };
 
-static void fn(void *opaque) {
+static void fn(void *opaque)
+{
   struct data *data = (struct data *)opaque;
   ZSTD_pthread_mutex_lock(&data->mutex);
   data->data[data->i] = data->i;
@@ -39,12 +40,13 @@ static void fn(void *opaque) {
   ZSTD_pthread_mutex_unlock(&data->mutex);
 }
 
-static int testOrder(size_t numThreads, size_t queueSize) {
+static int testOrder(size_t numThreads, size_t queueSize)
+{
   struct data data;
-  POOL_ctx *ctx = POOL_create(numThreads, queueSize);
+  POOL_ctx* const ctx = POOL_create(numThreads, queueSize);
   ASSERT_TRUE(ctx);
   data.i = 0;
-  ZSTD_pthread_mutex_init(&data.mutex, NULL);
+  (void)ZSTD_pthread_mutex_init(&data.mutex, NULL);
   { size_t i;
     for (i = 0; i < 16; ++i) {
       POOL_add(ctx, &fn, &data);
@@ -72,7 +74,7 @@ static void waitFn(void *opaque) {
 /* Tests for deadlock */
 static int testWait(size_t numThreads, size_t queueSize) {
   struct data data;
-  POOL_ctx *ctx = POOL_create(numThreads, queueSize);
+  POOL_ctx* const ctx = POOL_create(numThreads, queueSize);
   ASSERT_TRUE(ctx);
   { size_t i;
     for (i = 0; i < 16; ++i) {
@@ -94,7 +96,7 @@ typedef struct {
 } poolTest_t;
 
 static void waitLongFn(void *opaque) {
-  poolTest_t* test = (poolTest_t*) opaque;
+  poolTest_t* const test = (poolTest_t*) opaque;
   UTIL_sleepMilli(10);
   ZSTD_pthread_mutex_lock(&test->mut);
   test->val = test->val + 1;

From 058da605cb37dd5cdbe24c9c404020be52a9e6ab Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 11 Apr 2019 12:25:27 -0700
Subject: [PATCH 166/178] fixed minor conversion warning

---
 tests/poolTests.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/poolTests.c b/tests/poolTests.c
index 9b244afed09..272e961d380 100644
--- a/tests/poolTests.c
+++ b/tests/poolTests.c
@@ -35,7 +35,7 @@ static void fn(void *opaque)
 {
   struct data *data = (struct data *)opaque;
   ZSTD_pthread_mutex_lock(&data->mutex);
-  data->data[data->i] = data->i;
+  data->data[data->i] = (unsigned)(data->i);
   ++data->i;
   ZSTD_pthread_mutex_unlock(&data->mutex);
 }

From 1e01560b83389e1fdf91eb9a8311d9a1c69abdce Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 11 Apr 2019 13:46:30 -0700
Subject: [PATCH 167/178] fixed timespec_get() initialization bug on some
 targets

not sure why, but msan fires an "unitialized variable" error
when time gets properly initialized by timespec_get().
Maybe in some cases, not all bytes of the structure are initialized ?
Or maybe msan fails to detect the initialization ?

Anyway, pre-initializing the variable before passing it to timespec_get() works.
---
 programs/timefn.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/programs/timefn.c b/programs/timefn.c
index 21f2a9618b4..096e1910bf4 100644
--- a/programs/timefn.c
+++ b/programs/timefn.c
@@ -91,8 +91,10 @@ PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
 
 UTIL_time_t UTIL_getTime(void)
 {
-    UTIL_time_t time;
-    if (timespec_get(&time, TIME_UTC) == 0) {
+    /* time must be initialized, othersize it may fail msan test.
+     * No good reason, likely a limitation of timespec_get() for some target */
+    UTIL_time_t time = UTIL_TIME_INITIALIZER;
+    if (timespec_get(&time, TIME_UTC) != TIME_UTC) {
         perror("timefn::timespec_get");
         abort();
     }

From fb6901b2c0fc4f67954e4f67db4ec34aaf0c4866 Mon Sep 17 00:00:00 2001
From: Lzu Tao 
Date: Fri, 12 Apr 2019 21:28:17 +0700
Subject: [PATCH 168/178] tests: Add missing header timefn.h

---
 tests/zbufftest.c   | 1 +
 tests/zstreamtest.c | 1 +
 2 files changed, 2 insertions(+)

diff --git a/tests/zbufftest.c b/tests/zbufftest.c
index 9b6f7bad6dc..8cbde3f4f38 100644
--- a/tests/zbufftest.c
+++ b/tests/zbufftest.c
@@ -25,6 +25,7 @@
 #include        /* free */
 #include         /* fgets, sscanf */
 #include        /* strcmp */
+#include "timefn.h"       /* UTIL_time_t */
 #include "mem.h"
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_maxCLevel */
 #include "zstd.h"         /* ZSTD_compressBound */
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index e6f69d78ba7..f07477ae5b2 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -26,6 +26,7 @@
 #include         /* fgets, sscanf */
 #include        /* strcmp */
 #include        /* assert */
+#include "timefn.h"       /* UTIL_time_t, UTIL_getTime */
 #include "mem.h"
 #define ZSTD_STATIC_LINKING_ONLY  /* ZSTD_maxCLevel, ZSTD_customMem, ZSTD_getDictID_fromFrame */
 #include "zstd.h"         /* ZSTD_compressBound */

From 6c9b023f3e30d59b8c7f4f05621e69113e98bc34 Mon Sep 17 00:00:00 2001
From: Lzu Tao 
Date: Fri, 12 Apr 2019 21:28:36 +0700
Subject: [PATCH 169/178] meson: Fix build

---
 build/meson/programs/meson.build | 2 ++
 build/meson/tests/meson.build    | 7 +++++++
 2 files changed, 9 insertions(+)

diff --git a/build/meson/programs/meson.build b/build/meson/programs/meson.build
index d91dd3368dd..363818f9d64 100644
--- a/build/meson/programs/meson.build
+++ b/build/meson/programs/meson.build
@@ -12,6 +12,7 @@ zstd_rootdir = '../../..'
 
 zstd_programs_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'),
   join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'programs/timefn.c'),
   join_paths(zstd_rootdir, 'programs/fileio.c'),
   join_paths(zstd_rootdir, 'programs/benchfn.c'),
   join_paths(zstd_rootdir, 'programs/benchzstd.c'),
@@ -63,6 +64,7 @@ zstd = executable('zstd',
   install: true)
 
 zstd_frugal_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'),
+  join_paths(zstd_rootdir, 'programs/timefn.c'),
   join_paths(zstd_rootdir, 'programs/util.c'),
   join_paths(zstd_rootdir, 'programs/fileio.c')]
 
diff --git a/build/meson/tests/meson.build b/build/meson/tests/meson.build
index aa9cd86f4eb..fd813ea8805 100644
--- a/build/meson/tests/meson.build
+++ b/build/meson/tests/meson.build
@@ -40,6 +40,7 @@ datagen = executable('datagen',
 
 fullbench_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
   join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'programs/timefn.c'),
   join_paths(zstd_rootdir, 'programs/benchfn.c'),
   join_paths(zstd_rootdir, 'programs/benchzstd.c'),
   join_paths(zstd_rootdir, 'tests/fullbench.c')]
@@ -51,6 +52,7 @@ fullbench = executable('fullbench',
 
 fuzzer_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
   join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'programs/timefn.c'),
   join_paths(zstd_rootdir, 'tests/fuzzer.c')]
 fuzzer = executable('fuzzer',
   fuzzer_sources,
@@ -60,6 +62,7 @@ fuzzer = executable('fuzzer',
 
 zbufftest_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
   join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'programs/timefn.c'),
   join_paths(zstd_rootdir, 'tests/zbufftest.c')]
 zbufftest = executable('zbufftest',
   zbufftest_sources,
@@ -70,6 +73,7 @@ zbufftest = executable('zbufftest',
 
 zstreamtest_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
   join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'programs/timefn.c'),
   join_paths(zstd_rootdir, 'tests/seqgen.c'),
   join_paths(zstd_rootdir, 'tests/zstreamtest.c')]
 zstreamtest = executable('zstreamtest',
@@ -79,6 +83,7 @@ zstreamtest = executable('zstreamtest',
   install: false)
 
 paramgrill_sources = [join_paths(zstd_rootdir, 'programs/benchfn.c'),
+  join_paths(zstd_rootdir, 'programs/timefn.c'),
   join_paths(zstd_rootdir, 'programs/benchzstd.c'),
   join_paths(zstd_rootdir, 'programs/datagen.c'),
   join_paths(zstd_rootdir, 'programs/util.c'),
@@ -116,6 +121,7 @@ legacy = executable('legacy',
   install: false)
 
 decodecorpus_sources = [join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'programs/timefn.c'),
   join_paths(zstd_rootdir, 'tests/decodecorpus.c')]
 decodecorpus = executable('decodecorpus',
   decodecorpus_sources,
@@ -132,6 +138,7 @@ symbols = executable('symbols',
   install: false)
 
 poolTests_sources = [join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'programs/timefn.c'),
   join_paths(zstd_rootdir, 'tests/poolTests.c'),
   join_paths(zstd_rootdir, 'lib/common/pool.c'),
   join_paths(zstd_rootdir, 'lib/common/threading.c'),

From e31e7ca9ed56c6a28e6f038afdf58c25ed5de8a1 Mon Sep 17 00:00:00 2001
From: Lzu Tao 
Date: Fri, 12 Apr 2019 23:43:34 +0700
Subject: [PATCH 170/178] travis: Make Mesonbuild non-optional

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index b3a654378d8..ad2b4292a27 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -178,7 +178,7 @@ matrix:
 
     # meson dedicated test
     - name: Xenial (Meson + clang)
-      env: ALLOW_FAILURES=true
+      # env: ALLOW_FAILURES=true
       dist: xenial
       language: cpp
       compiler: clang

From a880ca239b447968493dd2fed3850e766d6305cc Mon Sep 17 00:00:00 2001
From: Josh Soref 
Date: Fri, 12 Apr 2019 14:18:11 -0400
Subject: [PATCH 171/178] Spelling (#1582)

* spelling: accidentally

* spelling: across

* spelling: additionally

* spelling: addresses

* spelling: appropriate

* spelling: assumed

* spelling: available

* spelling: builder

* spelling: capacity

* spelling: compiler

* spelling: compressibility

* spelling: compressor

* spelling: compression

* spelling: contract

* spelling: convenience

* spelling: decompress

* spelling: description

* spelling: deflate

* spelling: deterministically

* spelling: dictionary

* spelling: display

* spelling: eliminate

* spelling: preemptively

* spelling: exclude

* spelling: failure

* spelling: independence

* spelling: independent

* spelling: intentionally

* spelling: matching

* spelling: maximum

* spelling: meaning

* spelling: mishandled

* spelling: memory

* spelling: occasionally

* spelling: occurrence

* spelling: official

* spelling: offsets

* spelling: original

* spelling: output

* spelling: overflow

* spelling: overridden

* spelling: parameter

* spelling: performance

* spelling: probability

* spelling: receives

* spelling: redundant

* spelling: recompression

* spelling: resources

* spelling: sanity

* spelling: segment

* spelling: series

* spelling: specified

* spelling: specify

* spelling: subtracted

* spelling: successful

* spelling: return

* spelling: translation

* spelling: update

* spelling: unrelated

* spelling: useless

* spelling: variables

* spelling: variety

* spelling: verbatim

* spelling: verification

* spelling: visited

* spelling: warming

* spelling: workers

* spelling: with
---
 CHANGELOG                                        |  2 +-
 build/cmake/README.md                            |  2 +-
 contrib/docker/README.md                         |  2 +-
 .../benchmarkDictBuilder/benchmark.c             |  4 ++--
 .../fastCover/fastCover.c                        |  4 ++--
 contrib/largeNbDicts/largeNbDicts.c              |  2 +-
 .../linux-kernel/0002-lib-Add-zstd-modules.patch | 16 ++++++++--------
 .../0006-squashfs-tools-Add-zstd-support.patch   |  4 ++--
 contrib/linux-kernel/lib/zstd/compress.c         |  4 ++--
 contrib/linux-kernel/lib/zstd/fse_compress.c     |  4 ++--
 contrib/linux-kernel/lib/zstd/huf.h              |  2 +-
 .../linux-kernel/test/include/linux/compiler.h   |  6 +++---
 contrib/pzstd/Pzstd.cpp                          |  2 +-
 contrib/pzstd/utils/Range.h                      |  2 +-
 contrib/pzstd/utils/ResourcePool.h               |  2 +-
 doc/README.md                                    |  4 ++--
 doc/educational_decoder/zstd_decompress.c        |  2 +-
 doc/zstd_manual.html                             |  6 +++---
 examples/multiple_simple_compression.c           |  2 +-
 lib/common/compiler.h                            |  2 +-
 lib/common/fse.h                                 |  2 +-
 lib/common/threading.c                           |  4 ++--
 lib/common/xxhash.c                              |  4 ++--
 lib/compress/fse_compress.c                      |  4 ++--
 lib/compress/zstd_compress.c                     | 14 +++++++-------
 lib/compress/zstd_compress_internal.h            |  6 +++---
 lib/compress/zstd_lazy.h                         |  2 +-
 lib/compress/zstd_ldm.c                          |  2 +-
 lib/compress/zstd_opt.c                          |  4 ++--
 lib/compress/zstdmt_compress.c                   |  6 +++---
 lib/compress/zstdmt_compress.h                   |  2 +-
 lib/decompress/zstd_decompress.c                 |  2 +-
 lib/decompress/zstd_decompress_block.c           |  4 ++--
 lib/dictBuilder/cover.c                          |  4 ++--
 lib/dictBuilder/cover.h                          |  4 ++--
 lib/dictBuilder/fastcover.c                      |  4 ++--
 lib/legacy/zstd_v01.c                            |  2 +-
 lib/legacy/zstd_v02.c                            |  2 +-
 lib/legacy/zstd_v03.c                            |  2 +-
 lib/legacy/zstd_v04.c                            |  2 +-
 lib/legacy/zstd_v05.c                            |  2 +-
 lib/legacy/zstd_v06.c                            |  2 +-
 lib/legacy/zstd_v07.c                            |  2 +-
 lib/zstd.h                                       | 10 +++++-----
 programs/benchzstd.c                             |  2 +-
 programs/benchzstd.h                             |  2 +-
 programs/fileio.c                                |  2 +-
 programs/zstdcli.c                               |  2 +-
 tests/README.md                                  |  4 ++--
 tests/decodecorpus.c                             |  2 +-
 tests/fuzz/Makefile                              |  2 +-
 tests/fuzz/README.md                             |  4 ++--
 tests/fuzz/fuzz.py                               |  6 +++---
 tests/fuzz/fuzz_helpers.h                        |  2 +-
 tests/fuzz/zstd_helpers.c                        |  2 +-
 tests/fuzzer.c                                   |  2 +-
 tests/paramgrill.c                               | 10 +++++-----
 tests/regression/test.c                          |  2 +-
 tests/zstreamtest.c                              |  6 +++---
 zlibWrapper/README.md                            |  6 +++---
 zlibWrapper/examples/fitblk.c                    |  4 ++--
 zlibWrapper/examples/fitblk_original.c           |  4 ++--
 zlibWrapper/examples/zwrapbench.c                |  2 +-
 63 files changed, 116 insertions(+), 116 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index 0c09b4ad363..cf82ccd9d11 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -243,7 +243,7 @@ v1.0.0
 Change Licensing, all project is now BSD, Copyright Facebook
 Small decompression speed improvement
 API : Streaming API supports legacy format
-API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}(), ZSTD_setDStreamParamter()
+API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}(), ZSTD_setDStreamParameter()
 CLI supports legacy formats v0.4+
 Fixed : compression fails on certain huge files, reported by Jesse McGrew
 Enhanced documentation, by Przemyslaw Skibinski
diff --git a/build/cmake/README.md b/build/cmake/README.md
index 0c71a54ad5f..681b14cef39 100644
--- a/build/cmake/README.md
+++ b/build/cmake/README.md
@@ -3,7 +3,7 @@
 Contributions to the cmake build configurations are welcome. Please
 use case sensitivity that matches modern (ie. cmake version 2.6 and above)
 conventions of using lower-case for commands, and upper-case for
-varibles.
+variables.
 
 # CMake Style Recommendations
 
diff --git a/contrib/docker/README.md b/contrib/docker/README.md
index 299117787f8..43f6d7a1ae1 100644
--- a/contrib/docker/README.md
+++ b/contrib/docker/README.md
@@ -5,7 +5,7 @@ The `Dockerfile` script requires a version of `docker` >= 17.05
 
 ## Installing docker
 
-The officiel docker install docs use a ppa with a modern version available:
+The official docker install docs use a ppa with a modern version available:
 https://docs.docker.com/install/linux/docker-ce/ubuntu/
 
 ## How to run
diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c
index b1934569255..cd943797bde 100644
--- a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c
+++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c
@@ -127,7 +127,7 @@ dictInfo* createDictFromFiles(sampleInfo *info, unsigned maxDictSize,
 
 
 /** compressWithDict() :
- *  Compress samples from sample buffer given dicionary stored on dictionary buffer and compression level
+ *  Compress samples from sample buffer given dictionary stored on dictionary buffer and compression level
  *  @return compression ratio
  */
 double compressWithDict(sampleInfo *srcInfo, dictInfo* dInfo, int compressionLevel, int displayLevel) {
@@ -194,7 +194,7 @@ double compressWithDict(sampleInfo *srcInfo, dictInfo* dInfo, int compressionLev
     totalCompressedSize += compressedSize;
   }
 
-  /* Sum orignal sizes */
+  /* Sum original sizes */
   for (i = 0; inbSamples; i++) {
     totalOriginalSize += srcInfo->samplesSizes[i];
   }
diff --git a/contrib/experimental_dict_builders/fastCover/fastCover.c b/contrib/experimental_dict_builders/fastCover/fastCover.c
index 02c155a81e4..0a338bde2b2 100644
--- a/contrib/experimental_dict_builders/fastCover/fastCover.c
+++ b/contrib/experimental_dict_builders/fastCover/fastCover.c
@@ -125,7 +125,7 @@ typedef struct {
  *
  *     Score(S) = F(S_1) + F(S_2) + ... + F(S_{k-d+1})
  *
- * Once the dmer with hash value d is in the dictionay we set F(d) = F(d)/2.
+ * Once the dmer with hash value d is in the dictionary we set F(d) = F(d)/2.
  */
 static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx,
                                                   U32 *freqs, U32 begin,U32 end,
@@ -149,7 +149,7 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx,
     while (activeSegment.end < end) {
       /* Get hash value of current dmer */
       const size_t index = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.end, parameters.f, ctx->d);
-      /* Add frequency of this index to score if this is the first occurence of index in active segment */
+      /* Add frequency of this index to score if this is the first occurrence of index in active segment */
       if (ctx->segmentFreqs[index] == 0) {
         activeSegment.score += freqs[index];
       }
diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c
index 18953ec283f..dcc186b4501 100644
--- a/contrib/largeNbDicts/largeNbDicts.c
+++ b/contrib/largeNbDicts/largeNbDicts.c
@@ -424,7 +424,7 @@ static ddict_collection_t createDDictCollection(const void* dictBuffer, size_t d
 }
 
 
-/* mess with adresses, so that linear scanning dictionaries != linear address scanning */
+/* mess with addresses, so that linear scanning dictionaries != linear address scanning */
 void shuffleDictionaries(ddict_collection_t dicts)
 {
     size_t const nbDicts = dicts.nbDDict;
diff --git a/contrib/linux-kernel/0002-lib-Add-zstd-modules.patch b/contrib/linux-kernel/0002-lib-Add-zstd-modules.patch
index c3bbaed7367..0232d2d4ab5 100644
--- a/contrib/linux-kernel/0002-lib-Add-zstd-modules.patch
+++ b/contrib/linux-kernel/0002-lib-Add-zstd-modules.patch
@@ -4,7 +4,7 @@ Date: Mon, 17 Jul 2017 17:08:19 -0700
 Subject: [PATCH v5 2/5] lib: Add zstd modules
 
 Add zstd compression and decompression kernel modules.
-zstd offers a wide varity of compression speed and quality trade-offs.
+zstd offers a wide variety of compression speed and quality trade-offs.
 It can compress at speeds approaching lz4, and quality approaching lzma.
 zstd decompressions at speeds more than twice as fast as zlib, and
 decompression speed remains roughly the same across all compression levels.
@@ -21,7 +21,7 @@ will be easier to keep the kernel zstd up to date.
 I benchmarked zstd compression as a special character device. I ran zstd
 and zlib compression at several levels, as well as performing no
 compression, which measure the time spent copying the data to kernel space.
-Data is passed to the compresser 4096 B at a time. The benchmark file is
+Data is passed to the compressor 4096 B at a time. The benchmark file is
 located in the upstream zstd source repository under
 `contrib/linux-kernel/zstd_compress_test.c` [2].
 
@@ -86,7 +86,7 @@ Tested in userland using the test-suite in the zstd repo under
 `contrib/linux-kernel/test/UserlandTest.cpp` [5] by mocking the kernel
 functions. Fuzz tested using libfuzzer [6] with the fuzz harnesses under
 `contrib/linux-kernel/test/{RoundTripCrash.c,DecompressCrash.c}` [7] [8]
-with ASAN, UBSAN, and MSAN. Additionaly, it was tested while testing the
+with ASAN, UBSAN, and MSAN. Additionally, it was tested while testing the
 BtrFS and SquashFS patches coming next.
 
 [1] https://clang.llvm.org/docs/ClangFormat.html
@@ -4200,14 +4200,14 @@ index 0000000..ff18ae6
 +	BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
 +	U32 const fcsCode =
 +	    params.fParams.contentSizeFlag ? (pledgedSrcSize >= 256) + (pledgedSrcSize >= 65536 + 256) + (pledgedSrcSize >= 0xFFFFFFFFU) : 0; /* 0-3 */
-+	BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag << 2) + (singleSegment << 5) + (fcsCode << 6));
++	BYTE const frameHeaderDescriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag << 2) + (singleSegment << 5) + (fcsCode << 6));
 +	size_t pos;
 +
 +	if (dstCapacity < ZSTD_frameHeaderSize_max)
 +		return ERROR(dstSize_tooSmall);
 +
 +	ZSTD_writeLE32(dst, ZSTD_MAGICNUMBER);
-+	op[4] = frameHeaderDecriptionByte;
++	op[4] = frameHeaderDescriptionByte;
 +	pos = 5;
 +	if (!singleSegment)
 +		op[pos++] = windowLogByte;
@@ -8812,8 +8812,8 @@ index 0000000..ef3d174
 +		U32 position = 0;
 +		U32 symbol;
 +		for (symbol = 0; symbol <= maxSymbolValue; symbol++) {
-+			int nbOccurences;
-+			for (nbOccurences = 0; nbOccurences < normalizedCounter[symbol]; nbOccurences++) {
++			int nbOccurrences;
++			for (nbOccurrences = 0; nbOccurrences < normalizedCounter[symbol]; nbOccurrences++) {
 +				tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
 +				position = (position + step) & tableMask;
 +				while (position > highThreshold)
@@ -9944,7 +9944,7 @@ index 0000000..2143da2
 +	HUF_repeat_none,  /**< Cannot use the previous table */
 +	HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1,
 +			     4}X_repeat */
-+	HUF_repeat_valid  /**< Can use the previous table and it is asumed to be valid */
++	HUF_repeat_valid  /**< Can use the previous table and it is assumed to be valid */
 +} HUF_repeat;
 +/** HUF_compress4X_repeat() :
 +*   Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
diff --git a/contrib/linux-kernel/0006-squashfs-tools-Add-zstd-support.patch b/contrib/linux-kernel/0006-squashfs-tools-Add-zstd-support.patch
index ca638f2644b..00d24e2b5a8 100644
--- a/contrib/linux-kernel/0006-squashfs-tools-Add-zstd-support.patch
+++ b/contrib/linux-kernel/0006-squashfs-tools-Add-zstd-support.patch
@@ -11,7 +11,7 @@ Signed-off-by: Nick Terrell 
 ---
 v4 -> v5:
 - Fix patch documentation to reflect that Sean Purcell is the author
-- Don't strip trailing whitespace of unreleated code
+- Don't strip trailing whitespace of unrelated code
 - Make zstd_display_options() static
 
 v5 -> v6:
@@ -224,7 +224,7 @@ index 0000000..dcab75a
 + * set the default options, this is to ensure any user supplied
 + * -X options on the appending mksquashfs command line are over-ridden.
 + *
-+ * This function returns 0 on sucessful extraction of options, and -1 on error.
++ * This function returns 0 on successful extraction of options, and -1 on error.
 + */
 +static int zstd_extract_options(int block_size, void *buffer, int size)
 +{
diff --git a/contrib/linux-kernel/lib/zstd/compress.c b/contrib/linux-kernel/lib/zstd/compress.c
index ff18ae6d6e8..43535b8db55 100644
--- a/contrib/linux-kernel/lib/zstd/compress.c
+++ b/contrib/linux-kernel/lib/zstd/compress.c
@@ -2436,14 +2436,14 @@ static size_t ZSTD_writeFrameHeader(void *dst, size_t dstCapacity, ZSTD_paramete
 	BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
 	U32 const fcsCode =
 	    params.fParams.contentSizeFlag ? (pledgedSrcSize >= 256) + (pledgedSrcSize >= 65536 + 256) + (pledgedSrcSize >= 0xFFFFFFFFU) : 0; /* 0-3 */
-	BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag << 2) + (singleSegment << 5) + (fcsCode << 6));
+	BYTE const frameHeaderDescriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag << 2) + (singleSegment << 5) + (fcsCode << 6));
 	size_t pos;
 
 	if (dstCapacity < ZSTD_frameHeaderSize_max)
 		return ERROR(dstSize_tooSmall);
 
 	ZSTD_writeLE32(dst, ZSTD_MAGICNUMBER);
-	op[4] = frameHeaderDecriptionByte;
+	op[4] = frameHeaderDescriptionByte;
 	pos = 5;
 	if (!singleSegment)
 		op[pos++] = windowLogByte;
diff --git a/contrib/linux-kernel/lib/zstd/fse_compress.c b/contrib/linux-kernel/lib/zstd/fse_compress.c
index ef3d1741d53..0fe468edf6f 100644
--- a/contrib/linux-kernel/lib/zstd/fse_compress.c
+++ b/contrib/linux-kernel/lib/zstd/fse_compress.c
@@ -141,8 +141,8 @@ size_t FSE_buildCTable_wksp(FSE_CTable *ct, const short *normalizedCounter, unsi
 		U32 position = 0;
 		U32 symbol;
 		for (symbol = 0; symbol <= maxSymbolValue; symbol++) {
-			int nbOccurences;
-			for (nbOccurences = 0; nbOccurences < normalizedCounter[symbol]; nbOccurences++) {
+			int nbOccurrences;
+			for (nbOccurrences = 0; nbOccurrences < normalizedCounter[symbol]; nbOccurrences++) {
 				tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
 				position = (position + step) & tableMask;
 				while (position > highThreshold)
diff --git a/contrib/linux-kernel/lib/zstd/huf.h b/contrib/linux-kernel/lib/zstd/huf.h
index 2143da28d95..923218d12e2 100644
--- a/contrib/linux-kernel/lib/zstd/huf.h
+++ b/contrib/linux-kernel/lib/zstd/huf.h
@@ -134,7 +134,7 @@ typedef enum {
 	HUF_repeat_none,  /**< Cannot use the previous table */
 	HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1,
 			     4}X_repeat */
-	HUF_repeat_valid  /**< Can use the previous table and it is asumed to be valid */
+	HUF_repeat_valid  /**< Can use the previous table and it is assumed to be valid */
 } HUF_repeat;
 /** HUF_compress4X_repeat() :
 *   Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
diff --git a/contrib/linux-kernel/test/include/linux/compiler.h b/contrib/linux-kernel/test/include/linux/compiler.h
index 7991b8b29d5..4fb4f42e29c 100644
--- a/contrib/linux-kernel/test/include/linux/compiler.h
+++ b/contrib/linux-kernel/test/include/linux/compiler.h
@@ -1,5 +1,5 @@
-#ifndef LINUX_COMIPLER_H_
-#define LINUX_COMIPLER_H_
+#ifndef LINUX_COMPILER_H_
+#define LINUX_COMPILER_H_
 
 #ifndef __always_inline
 #  define __always_inline inline
@@ -9,4 +9,4 @@
 #  define noinline __attribute__((__noinline__))
 #endif
 
-#endif // LINUX_COMIPLER_H_
+#endif // LINUX_COMPILER_H_
diff --git a/contrib/pzstd/Pzstd.cpp b/contrib/pzstd/Pzstd.cpp
index 6c580b3bccc..652187c3bd0 100644
--- a/contrib/pzstd/Pzstd.cpp
+++ b/contrib/pzstd/Pzstd.cpp
@@ -55,7 +55,7 @@ static std::uint64_t handleOneInput(const Options &options,
                              SharedState& state) {
   auto inputSize = fileSizeOrZero(inputFile);
   // WorkQueue outlives ThreadPool so in the case of error we are certain
-  // we don't accidently try to call push() on it after it is destroyed
+  // we don't accidentally try to call push() on it after it is destroyed
   WorkQueue> outs{options.numThreads + 1};
   std::uint64_t bytesRead;
   std::uint64_t bytesWritten;
diff --git a/contrib/pzstd/utils/Range.h b/contrib/pzstd/utils/Range.h
index 7e2559cc9e3..fedb5d786c6 100644
--- a/contrib/pzstd/utils/Range.h
+++ b/contrib/pzstd/utils/Range.h
@@ -9,7 +9,7 @@
  
 /**
  * A subset of `folly/Range.h`.
- * All code copied verbatiam modulo formatting
+ * All code copied verbatim modulo formatting
  */
 #pragma once
 
diff --git a/contrib/pzstd/utils/ResourcePool.h b/contrib/pzstd/utils/ResourcePool.h
index a6ff5ffc5db..8dfcdd76590 100644
--- a/contrib/pzstd/utils/ResourcePool.h
+++ b/contrib/pzstd/utils/ResourcePool.h
@@ -54,7 +54,7 @@ class ResourcePool {
 
   /**
    * @returns  A unique pointer to a resource.  The resource is null iff
-   *           there are no avaiable resources and `factory()` returns null.
+   *           there are no available resources and `factory()` returns null.
    */
   UniquePtr get() {
     std::lock_guard lock(mutex_);
diff --git a/doc/README.md b/doc/README.md
index 1f01fa47d4b..bb7a3e4902e 100644
--- a/doc/README.md
+++ b/doc/README.md
@@ -12,8 +12,8 @@ __`zstd_compression_format.md`__ : This document defines the Zstandard compressi
 Compliant decoders must adhere to this document,
 and compliant encoders must generate data that follows it.
 
-Should you look for ressources to develop your own port of Zstandard algorithm,
-you may find the following ressources useful :
+Should you look for resources to develop your own port of Zstandard algorithm,
+you may find the following resources useful :
 
 __`educational_decoder`__ : This directory contains an implementation of a Zstandard decoder,
 compliant with the Zstandard compression format.
diff --git a/doc/educational_decoder/zstd_decompress.c b/doc/educational_decoder/zstd_decompress.c
index bea0e0ce1f4..8e231bbb5a9 100644
--- a/doc/educational_decoder/zstd_decompress.c
+++ b/doc/educational_decoder/zstd_decompress.c
@@ -358,7 +358,7 @@ static u32 copy_literals(const size_t seq, istream_t *litstream,
                          ostream_t *const out);
 
 // Given an offset code from a sequence command (either an actual offset value
-// or an index for previous offset), computes the correct offset and udpates
+// or an index for previous offset), computes the correct offset and updates
 // the offset history
 static size_t compute_offset(sequence_command_t seq, u64 *const offset_hist);
 
diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index cb101e1c62c..f1628b50d08 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -384,7 +384,7 @@ 
Decompression context
  When decompressing many times,
   Note 3 : Whenever all input data is provided and consumed in a single round,
            for example with ZSTD_compress2(),
            or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end),
-           this value is automatically overriden by srcSize instead.
+           this value is automatically overridden by srcSize instead.
  
 


 
@@ -571,8 +571,8 @@ 
Streaming compression functions
typedef enum {
 
  Behaves about the same as ZSTD_compressStream, with additional control on end directive.
   - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
   - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
-  - outpot->pos must be <= dstCapacity, input->pos must be <= srcSize
-  - outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
+  - output->pos must be <= dstCapacity, input->pos must be <= srcSize
+  - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
   - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
   - When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads, flush whatever is available,
                                                   and then immediately returns, just indicating that there is some data remaining to be flushed.
diff --git a/examples/multiple_simple_compression.c b/examples/multiple_simple_compression.c
index 51c9ec725d3..a44ac8b442f 100644
--- a/examples/multiple_simple_compression.c
+++ b/examples/multiple_simple_compression.c
@@ -107,7 +107,7 @@ int main(int argc, const char** argv)
         compressFile_orDie(ress, inFilename, outFilename);
     }
 
-    /* free momery */
+    /* free memory */
     freeResources(ress,outFilename);
 
     printf("compressed %i files \n", argc-1);
diff --git a/lib/common/compiler.h b/lib/common/compiler.h
index 7f561282ca3..0836e3ed27a 100644
--- a/lib/common/compiler.h
+++ b/lib/common/compiler.h
@@ -40,7 +40,7 @@
 
 /**
  * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant
- * parameters. They must be inlined for the compiler to elimininate the constant
+ * parameters. They must be inlined for the compiler to eliminate the constant
  * branches.
  */
 #define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR
diff --git a/lib/common/fse.h b/lib/common/fse.h
index f72c519b259..811c670bddc 100644
--- a/lib/common/fse.h
+++ b/lib/common/fse.h
@@ -358,7 +358,7 @@ size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size
 typedef enum {
    FSE_repeat_none,  /**< Cannot use the previous table */
    FSE_repeat_check, /**< Can use the previous table but it must be checked */
-   FSE_repeat_valid  /**< Can use the previous table and it is asumed to be valid */
+   FSE_repeat_valid  /**< Can use the previous table and it is assumed to be valid */
  } FSE_repeat;
 
 /* *****************************************
diff --git a/lib/common/threading.c b/lib/common/threading.c
index 8be8c8da948..f3d4fa84184 100644
--- a/lib/common/threading.c
+++ b/lib/common/threading.c
@@ -14,8 +14,8 @@
  * This file will hold wrapper for systems, which do not support pthreads
  */
 
-/* create fake symbol to avoid empty trnaslation unit warning */
-int g_ZSTD_threading_useles_symbol;
+/* create fake symbol to avoid empty translation unit warning */
+int g_ZSTD_threading_useless_symbol;
 
 #if defined(ZSTD_MULTITHREAD) && defined(_WIN32)
 
diff --git a/lib/common/xxhash.c b/lib/common/xxhash.c
index 532b8161929..30599aaae41 100644
--- a/lib/common/xxhash.c
+++ b/lib/common/xxhash.c
@@ -66,10 +66,10 @@
 /* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */
 
 /*!XXH_FORCE_NATIVE_FORMAT :
- * By default, xxHash library provides endian-independant Hash values, based on little-endian convention.
+ * By default, xxHash library provides endian-independent Hash values, based on little-endian convention.
  * Results are therefore identical for little-endian and big-endian CPU.
  * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format.
- * Should endian-independance be of no importance for your application, you may set the #define below to 1,
+ * Should endian-independence be of no importance for your application, you may set the #define below to 1,
  * to improve speed for Big-endian CPU.
  * This option has no impact on Little_Endian CPU.
  */
diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c
index 60f357bbd24..68b47e10935 100644
--- a/lib/compress/fse_compress.c
+++ b/lib/compress/fse_compress.c
@@ -129,9 +129,9 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
     {   U32 position = 0;
         U32 symbol;
         for (symbol=0; symbol<=maxSymbolValue; symbol++) {
-            int nbOccurences;
+            int nbOccurrences;
             int const freq = normalizedCounter[symbol];
-            for (nbOccurences=0; nbOccurences highThreshold)
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 4a9f6b7c892..2e163c8bf3d 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2046,7 +2046,7 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
  * If x == 0: Return 0
  * Else: Return floor(-log2(x / 256) * 256)
  */
-static unsigned const kInverseProbabiltyLog256[256] = {
+static unsigned const kInverseProbabilityLog256[256] = {
     0,    2048, 1792, 1642, 1536, 1453, 1386, 1329, 1280, 1236, 1197, 1162,
     1130, 1100, 1073, 1047, 1024, 1001, 980,  960,  941,  923,  906,  889,
     874,  859,  844,  830,  817,  804,  791,  779,  768,  756,  745,  734,
@@ -2085,7 +2085,7 @@ static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t
         if (count[s] != 0 && norm == 0)
             norm = 1;
         assert(count[s] < total);
-        cost += count[s] * kInverseProbabiltyLog256[norm];
+        cost += count[s] * kInverseProbabilityLog256[norm];
     }
     return cost >> 8;
 }
@@ -2108,7 +2108,7 @@ static size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog,
         unsigned const norm256 = normAcc << shift;
         assert(norm256 > 0);
         assert(norm256 < 256);
-        cost += count[s] * kInverseProbabiltyLog256[norm256];
+        cost += count[s] * kInverseProbabilityLog256[norm256];
     }
     return cost >> 8;
 }
@@ -2611,7 +2611,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
         FORWARD_IF_ERROR(bitstreamSize);
         op += bitstreamSize;
         /* zstd versions <= 1.3.4 mistakenly report corruption when
-         * FSE_readNCount() recieves a buffer < 4 bytes.
+         * FSE_readNCount() receives a buffer < 4 bytes.
          * Fixed by https://github.com/facebook/zstd/pull/1146.
          * This can happen when the last set_compressed table present is 2
          * bytes and the bitstream is only one byte.
@@ -2914,7 +2914,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
     BYTE  const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
     U32   const fcsCode = params.fParams.contentSizeFlag ?
                      (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : 0;  /* 0-3 */
-    BYTE  const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) );
+    BYTE  const frameHeaderDescriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) );
     size_t pos=0;
 
     assert(!(params.fParams.contentSizeFlag && pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN));
@@ -2926,7 +2926,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
         MEM_writeLE32(dst, ZSTD_MAGICNUMBER);
         pos = 4;
     }
-    op[pos++] = frameHeaderDecriptionByte;
+    op[pos++] = frameHeaderDescriptionByte;
     if (!singleSegment) op[pos++] = windowLogByte;
     switch(dictIDSizeCode)
     {
@@ -2950,7 +2950,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
 /* ZSTD_writeLastEmptyBlock() :
  * output an empty Block with end-of-frame mark to complete a frame
  * @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h))
- *           or an error code if `dstCapcity` is too small (window.nextSrc >= (BYTE const*)src + srcSize);
     /* The input could be very large (in zstdmt), so it must be broken up into
-     * chunks to enforce the maximmum distance and handle overflow correction.
+     * chunks to enforce the maximum distance and handle overflow correction.
      */
     assert(sequences->pos <= sequences->size);
     assert(sequences->size <= sequences->capacity);
diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index cf2f70b11f4..efb69d3267d 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -885,7 +885,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
             /* large match -> immediate encoding */
             {   U32 const maxML = matches[nbMatches-1].len;
                 U32 const maxOffset = matches[nbMatches-1].off;
-                DEBUGLOG(6, "found %u matches of maxLength=%u and maxOffCode=%u at cPos=%u => start new serie",
+                DEBUGLOG(6, "found %u matches of maxLength=%u and maxOffCode=%u at cPos=%u => start new series",
                             nbMatches, maxML, maxOffset, (U32)(ip-prefixStart));
 
                 if (maxML > sufficient_len) {
@@ -1133,7 +1133,7 @@ MEM_STATIC void ZSTD_upscaleStats(optState_t* optPtr)
 /* ZSTD_initStats_ultra():
  * make a first compression pass, just to seed stats with more accurate starting values.
  * only works on first block, with no dictionary and no ldm.
- * this function cannot error, hence its constract must be respected.
+ * this function cannot error, hence its contract must be respected.
  */
 static void
 ZSTD_initStats_ultra(ZSTD_matchState_t* ms,
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 7e2c78984eb..4b29fb3e4ee 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -456,7 +456,7 @@ typedef struct {
      * Must be acquired after the main mutex when acquiring both.
      */
     ZSTD_pthread_mutex_t ldmWindowMutex;
-    ZSTD_pthread_cond_t ldmWindowCond;  /* Signaled when ldmWindow is udpated */
+    ZSTD_pthread_cond_t ldmWindowCond;  /* Signaled when ldmWindow is updated */
     ZSTD_window_t ldmWindow;  /* A thread-safe copy of ldmState.window */
 } serialState_t;
 
@@ -647,7 +647,7 @@ static void ZSTDMT_compressionJob(void* jobDescription)
     buffer_t dstBuff = job->dstBuff;
     size_t lastCBlockSize = 0;
 
-    /* ressources */
+    /* resources */
     if (cctx==NULL) JOB_ERROR(ERROR(memory_allocation));
     if (dstBuff.start == NULL) {   /* streaming job : doesn't provide a dstBuffer */
         dstBuff = ZSTDMT_getBuffer(job->bufPool);
@@ -1527,7 +1527,7 @@ size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel) {
 /* ZSTDMT_writeLastEmptyBlock()
  * Write a single empty block with an end-of-frame to finish a frame.
  * Job must be created from streaming variant.
- * This function is always successfull if expected conditions are fulfilled.
+ * This function is always successful if expected conditions are fulfilled.
  */
 static void ZSTDMT_writeLastEmptyBlock(ZSTDMT_jobDescription* job)
 {
diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h
index bae830e280b..8d274ce8d3c 100644
--- a/lib/compress/zstdmt_compress.h
+++ b/lib/compress/zstdmt_compress.h
@@ -103,7 +103,7 @@ ZSTDMT_API size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,
  * List of parameters that can be set using ZSTDMT_setMTCtxParameter() */
 typedef enum {
     ZSTDMT_p_jobSize,     /* Each job is compressed in parallel. By default, this value is dynamically determined depending on compression parameters. Can be set explicitly here. */
-    ZSTDMT_p_overlapLog,  /* Each job may reload a part of previous job to enhance compressionr ratio; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window. This is a "sticky" parameter : its value will be re-used on next compression job */
+    ZSTDMT_p_overlapLog,  /* Each job may reload a part of previous job to enhance compression ratio; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window. This is a "sticky" parameter : its value will be re-used on next compression job */
     ZSTDMT_p_rsyncable    /* Enables rsyncable mode. */
 } ZSTDMT_parameter;
 
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 4f2922790e2..675596f5aa9 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1187,7 +1187,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
 }
 
 /*! ZSTD_getDictID_fromFrame() :
- *  Provides the dictID required to decompresse frame stored within `src`.
+ *  Provides the dictID required to decompress frame stored within `src`.
  *  If @return == 0, the dictID could not be decoded.
  *  This could for one of the following reasons :
  *  - The frame does not require a dictionary (most common case).
diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c
index 4418c51dbd7..a2a7eedcf21 100644
--- a/lib/decompress/zstd_decompress_block.c
+++ b/lib/decompress/zstd_decompress_block.c
@@ -801,7 +801,7 @@ ZSTD_updateFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD)
 /* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum
  * offset bits. But we can only read at most (STREAM_ACCUMULATOR_MIN_32 - 1)
  * bits before reloading. This value is the maximum number of bytes we read
- * after reloading when we are decoding long offets.
+ * after reloading when we are decoding long offsets.
  */
 #define LONG_OFFSETS_MAX_EXTRA_BITS_32                       \
     (ZSTD_WINDOWLOG_MAX_32 > STREAM_ACCUMULATOR_MIN_32       \
@@ -1180,7 +1180,7 @@ ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize,
 /* ZSTD_decompressSequencesLong() :
  * decompression function triggered when a minimum share of offsets is considered "long",
  * aka out of cache.
- * note : "long" definition seems overloaded here, sometimes meaning "wider than bitstream register", and sometimes mearning "farther than memory cache distance".
+ * note : "long" definition seems overloaded here, sometimes meaning "wider than bitstream register", and sometimes meaning "farther than memory cache distance".
  * This function will try to mitigate main memory latency through the use of prefetching */
 static size_t
 ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx,
diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c
index ed5a02f9849..21464ad0310 100644
--- a/lib/dictBuilder/cover.c
+++ b/lib/dictBuilder/cover.c
@@ -391,7 +391,7 @@ static void COVER_group(COVER_ctx_t *ctx, const void *group,
  *
  *     Score(S) = F(S_1) + F(S_2) + ... + F(S_{k-d+1})
  *
- * Once the dmer d is in the dictionay we set F(d) = 0.
+ * Once the dmer d is in the dictionary we set F(d) = 0.
  */
 static COVER_segment_t COVER_selectSegment(const COVER_ctx_t *ctx, U32 *freqs,
                                            COVER_map_t *activeDmers, U32 begin,
@@ -435,7 +435,7 @@ static COVER_segment_t COVER_selectSegment(const COVER_ctx_t *ctx, U32 *freqs,
       U32 *delDmerOcc = COVER_map_at(activeDmers, delDmer);
       activeSegment.begin += 1;
       *delDmerOcc -= 1;
-      /* If this is the last occurence of the dmer, subtract its score */
+      /* If this is the last occurrence of the dmer, subtract its score */
       if (*delDmerOcc == 0) {
         COVER_map_remove(activeDmers, delDmer);
         activeSegment.score -= freqs[delDmer];
diff --git a/lib/dictBuilder/cover.h b/lib/dictBuilder/cover.h
index 27e6fb7a3cc..efb46807c78 100644
--- a/lib/dictBuilder/cover.h
+++ b/lib/dictBuilder/cover.h
@@ -51,9 +51,9 @@ typedef struct {
  * We will make sure that each epoch gets at least 10 * k bytes.
  *
  * The COVER algorithms divide the data up into epochs of equal size and
- * select one segemnt from each epoch.
+ * select one segment from each epoch.
  *
- * @param maxDictSize The maximum allowed dictioary size.
+ * @param maxDictSize The maximum allowed dictionary size.
  * @param nbDmers     The number of dmers we are training on.
  * @param k           The parameter k (segment size).
  * @param passes      The target number of passes over the dmer corpus.
diff --git a/lib/dictBuilder/fastcover.c b/lib/dictBuilder/fastcover.c
index 6cf37026f44..5b6b941a908 100644
--- a/lib/dictBuilder/fastcover.c
+++ b/lib/dictBuilder/fastcover.c
@@ -132,7 +132,7 @@ typedef struct {
  *
  *     Score(S) = F(S_1) + F(S_2) + ... + F(S_{k-d+1})
  *
- * Once the dmer with hash value d is in the dictionay we set F(d) = 0.
+ * Once the dmer with hash value d is in the dictionary we set F(d) = 0.
  */
 static COVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx,
                                               U32 *freqs, U32 begin, U32 end,
@@ -161,7 +161,7 @@ static COVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx,
     /* Get hash value of current dmer */
     const size_t idx = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.end, f, d);
 
-    /* Add frequency of this index to score if this is the first occurence of index in active segment */
+    /* Add frequency of this index to score if this is the first occurrence of index in active segment */
     if (segmentFreqs[idx] == 0) {
       activeSegment.score += freqs[idx];
     }
diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c
index bb0f4b59321..cad2b99b4a7 100644
--- a/lib/legacy/zstd_v01.c
+++ b/lib/legacy/zstd_v01.c
@@ -1759,7 +1759,7 @@ static size_t ZSTD_execSequence(BYTE* op,
                                 BYTE* const base, BYTE* const oend)
 {
     static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4};   /* added */
-    static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11};   /* substracted */
+    static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11};   /* subtracted */
     const BYTE* const ostart = op;
     const size_t litLength = sequence.litLength;
     BYTE* const endMatch = op + litLength + sequence.matchLength;    /* risk : address space overflow (32-bits) */
diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c
index 594835943f4..561bc412e9a 100644
--- a/lib/legacy/zstd_v02.c
+++ b/lib/legacy/zstd_v02.c
@@ -3098,7 +3098,7 @@ static size_t ZSTD_execSequence(BYTE* op,
                                 BYTE* const base, BYTE* const oend)
 {
     static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4};   /* added */
-    static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11};   /* substracted */
+    static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11};   /* subtracted */
     const BYTE* const ostart = op;
     BYTE* const oLitEnd = op + sequence.litLength;
     BYTE* const oMatchEnd = op + sequence.litLength + sequence.matchLength;   /* risk : address space overflow (32-bits) */
diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c
index b6c60d296c8..a1bf0fa9b5c 100644
--- a/lib/legacy/zstd_v03.c
+++ b/lib/legacy/zstd_v03.c
@@ -2739,7 +2739,7 @@ static size_t ZSTD_execSequence(BYTE* op,
                                 BYTE* const base, BYTE* const oend)
 {
     static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4};   /* added */
-    static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11};   /* substracted */
+    static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11};   /* subtracted */
     const BYTE* const ostart = op;
     BYTE* const oLitEnd = op + sequence.litLength;
     BYTE* const oMatchEnd = op + sequence.litLength + sequence.matchLength;   /* risk : address space overflow (32-bits) */
diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c
index 65dc64dbbab..4342330e255 100644
--- a/lib/legacy/zstd_v04.c
+++ b/lib/legacy/zstd_v04.c
@@ -2862,7 +2862,7 @@ static size_t ZSTD_execSequence(BYTE* op,
                                 const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
 {
     static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 };   /* added */
-    static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 };   /* substracted */
+    static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 };   /* subtracted */
     BYTE* const oLitEnd = op + sequence.litLength;
     const size_t sequenceLength = sequence.litLength + sequence.matchLength;
     BYTE* const oMatchEnd = op + sequenceLength;   /* risk : address space overflow (32-bits) */
diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c
index 1c39f2f2682..caaf15f9bdb 100644
--- a/lib/legacy/zstd_v05.c
+++ b/lib/legacy/zstd_v05.c
@@ -3219,7 +3219,7 @@ static size_t ZSTDv05_execSequence(BYTE* op,
                                 const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
 {
     static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 };   /* added */
-    static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 };   /* substracted */
+    static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 };   /* subtracted */
     BYTE* const oLitEnd = op + sequence.litLength;
     const size_t sequenceLength = sequence.litLength + sequence.matchLength;
     BYTE* const oMatchEnd = op + sequenceLength;   /* risk : address space overflow (32-bits) */
diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c
index 65975ac2933..a695cbb8a6d 100644
--- a/lib/legacy/zstd_v06.c
+++ b/lib/legacy/zstd_v06.c
@@ -3408,7 +3408,7 @@ static size_t ZSTDv06_execSequence(BYTE* op,
     if (sequence.offset < 8) {
         /* close range match, overlap */
         static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 };   /* added */
-        static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 };   /* substracted */
+        static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 };   /* subtracted */
         int const sub2 = dec64table[sequence.offset];
         op[0] = match[0];
         op[1] = match[1];
diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c
index 443524b3ac2..6b9488931f3 100644
--- a/lib/legacy/zstd_v07.c
+++ b/lib/legacy/zstd_v07.c
@@ -3633,7 +3633,7 @@ size_t ZSTDv07_execSequence(BYTE* op,
     if (sequence.offset < 8) {
         /* close range match, overlap */
         static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 };   /* added */
-        static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 };   /* substracted */
+        static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 };   /* subtracted */
         int const sub2 = dec64table[sequence.offset];
         op[0] = match[0];
         op[1] = match[1];
diff --git a/lib/zstd.h b/lib/zstd.h
index cc87628b164..53470c18f36 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -432,7 +432,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param
  *  Note 3 : Whenever all input data is provided and consumed in a single round,
  *           for example with ZSTD_compress2(),
  *           or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end),
- *           this value is automatically overriden by srcSize instead.
+ *           this value is automatically overridden by srcSize instead.
  */
 ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
 
@@ -635,8 +635,8 @@ typedef enum {
  *  Behaves about the same as ZSTD_compressStream, with additional control on end directive.
  *  - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
  *  - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
- *  - outpot->pos must be <= dstCapacity, input->pos must be <= srcSize
- *  - outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
+ *  - output->pos must be <= dstCapacity, input->pos must be <= srcSize
+ *  - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
  *  - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
  *  - When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads, flush whatever is available,
  *                                                  and then immediately returns, just indicating that there is some data remaining to be flushed.
@@ -662,7 +662,7 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output
 
 /*******************************************************************************
  * This is a legacy streaming API, and can be replaced by ZSTD_CCtx_reset() and
- * ZSTD_compressStream2(). It is redundent, but is still fully supported.
+ * ZSTD_compressStream2(). It is redundant, but is still fully supported.
  * Advanced parameters and dictionary compression can only be used through the
  * new API.
  ******************************************************************************/
@@ -1023,7 +1023,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 #define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27   /* by default, the streaming decoder will refuse any frame
                                            * requiring larger than (1<mode == BMK_decodeOnly) {
         memcpy(compressedBuffer, srcBuffer, loadedCompressedSize);
     } else {
diff --git a/programs/benchzstd.h b/programs/benchzstd.h
index 376a80a90a0..2c7627713ee 100644
--- a/programs/benchzstd.h
+++ b/programs/benchzstd.h
@@ -170,7 +170,7 @@ BMK_benchOutcome_t BMK_syntheticTest(
  *  comprParams - basic compression parameters
  *  dictBuffer - a dictionary if used, null otherwise
  *  dictBufferSize - size of dictBuffer, 0 otherwise
- *  diplayLevel - see BMK_benchFiles
+ *  displayLevel - see BMK_benchFiles
  *  displayName - name used by display
  * @return:
  *      a variant, which expresses either an error, or a valid result.
diff --git a/programs/fileio.c b/programs/fileio.c
index 30514d41bf1..7ada5921b12 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1060,7 +1060,7 @@ FIO_compressZstdFrame(FIO_prefs_t* const prefs,
                         /* test if compression is blocked
                          * either because output is slow and all buffers are full
                          * or because input is slow and no job can start while waiting for at least one buffer to be filled.
-                         * note : excluse starting part, since currentJobID > 1 */
+                         * note : exclude starting part, since currentJobID > 1 */
                         if ( (zfp.consumed == previous_zfp_update.consumed)   /* no data compressed : no data available, or no more buffer to compress to, OR compression is really slow (compression of a single block is slower than update rate)*/
                           && (zfp.nbActiveWorkers == 0)                       /* confirmed : no compression ongoing */
                           ) {
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 904bcdf809b..fbb1c048a36 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -347,7 +347,7 @@ static unsigned parseFastCoverParameters(const char* stringPtr, ZDICT_fastCover_
 
 /**
  * parseLegacyParameters() :
- * reads legacy dictioanry builter parameters from *stringPtr (e.g. "--train-legacy=selectivity=8") into *selectivity
+ * reads legacy dictionary builder parameters from *stringPtr (e.g. "--train-legacy=selectivity=8") into *selectivity
  * @return 1 means that legacy dictionary builder parameters were correct
  * @return 0 in case of malformed parameters
  */
diff --git a/tests/README.md b/tests/README.md
index 7c6fb0db845..f3450119747 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -72,7 +72,7 @@ Command line tool to generate test .zst files.
 
 This tool will generate .zst files with checksums,
 as well as optionally output the corresponding correct uncompressed data for
-extra verfication.
+extra verification.
 
 Example:
 ```
@@ -123,7 +123,7 @@ Full list of arguments
                     Higher values will make optimizer run longer, more chances to find better solution.
     memLog    : Limits the log of the size of each memotable (1 per strategy). Will use hash tables when state space is larger than max size.
                     Setting memLog = 0 turns off memoization
- --display=   : specifiy which parameters are included in the output
+ --display=   : specify which parameters are included in the output
                     can use all --zstd parameter names and 'cParams' as a shorthand for all parameters used in ZSTD_compressionParameters
                     (Default: display all params available)
  -P#          : generated sample compressibility (when no file is provided)
diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c
index d8b332479ed..9910d3c551a 100644
--- a/tests/decodecorpus.c
+++ b/tests/decodecorpus.c
@@ -514,7 +514,7 @@ static size_t writeLiteralsBlockCompressed(U32* seed, frame_t* frame, size_t con
         if ((RAND(seed) & 3) || !frame->stats.hufInit) {
             do {
                 if (RAND(seed) & 3) {
-                    /* add 10 to ensure some compressability */
+                    /* add 10 to ensure some compressibility */
                     double const weight = ((RAND(seed) % 90) + 10) / 100.0;
 
                     DISPLAYLEVEL(5, "    distribution weight: %d%%\n",
diff --git a/tests/fuzz/Makefile b/tests/fuzz/Makefile
index 12ec9524bfc..31b151b857b 100644
--- a/tests/fuzz/Makefile
+++ b/tests/fuzz/Makefile
@@ -104,7 +104,7 @@ libregression.a: $(FUZZ_HEADERS) $(PRGDIR)/util.h $(PRGDIR)/util.c regression_dr
 	$(AR) $(FUZZ_ARFLAGS) $@ regression_driver.o
 
 # Install libfuzzer (not usable for MSAN testing)
-# Provided for convienence. To use this library run make libFuzzer and
+# Provided for convenience. To use this library run make libFuzzer and
 # set LDFLAGS=-L.
 .PHONY: libFuzzer
 libFuzzer:
diff --git a/tests/fuzz/README.md b/tests/fuzz/README.md
index f184be646ef..9e0bb259a9e 100644
--- a/tests/fuzz/README.md
+++ b/tests/fuzz/README.md
@@ -37,8 +37,8 @@ The specific fuzzing engine is selected with `LIB_FUZZING_ENGINE` or
 `--lib-fuzzing-engine`, the default is `libregression.a`.
 It has flags that can easily set up sanitizers `--enable-{a,ub,m}san`, and
 coverage instrumentation `--enable-coverage`.
-It sets sane defaults which can be overriden with flags `--debug`,
-`--enable-ubsan-pointer-overlow`, etc.
+It sets sane defaults which can be overridden with flags `--debug`,
+`--enable-ubsan-pointer-overflow`, etc.
 Run `./fuzz.py build -h` for help.
 
 ### Running Fuzzers
diff --git a/tests/fuzz/fuzz.py b/tests/fuzz/fuzz.py
index ee27015a535..cd2a5b4d442 100755
--- a/tests/fuzz/fuzz.py
+++ b/tests/fuzz/fuzz.py
@@ -339,13 +339,13 @@ def build_parser(args):
     args = parse_env_flags(args, ' '.join(
         [args.cppflags, args.cflags, args.cxxflags, args.ldflags]))
 
-    # Check option sanitiy
+    # Check option sanity
     if args.msan and (args.asan or args.ubsan):
         raise RuntimeError('MSAN may not be used with any other sanitizers')
     if args.msan_track_origins and not args.msan:
         raise RuntimeError('--enable-msan-track-origins requires MSAN')
     if args.ubsan_pointer_overflow and not args.ubsan:
-        raise RuntimeError('--enable-ubsan-pointer-overlow requires UBSAN')
+        raise RuntimeError('--enable-ubsan-pointer-overflow requires UBSAN')
     if args.sanitize_recover and not args.sanitize:
         raise RuntimeError('--enable-sanitize-recover but no sanitizers used')
 
@@ -623,7 +623,7 @@ def regression(args):
 
 def gen_parser(args):
     description = """
-    Generate a seed corpus appropiate for TARGET with data generated with
+    Generate a seed corpus appropriate for TARGET with data generated with
     decodecorpus.
     The fuzz inputs are prepended with a seed before the zstd data, so the
     output of decodecorpus shouldn't be used directly.
diff --git a/tests/fuzz/fuzz_helpers.h b/tests/fuzz/fuzz_helpers.h
index 468c39fb42d..0cf79d0d7ce 100644
--- a/tests/fuzz/fuzz_helpers.h
+++ b/tests/fuzz/fuzz_helpers.h
@@ -55,7 +55,7 @@ extern "C" {
 #endif
 
 /**
- * Determininistically constructs a seed based on the fuzz input.
+ * Deterministically constructs a seed based on the fuzz input.
  * Consumes up to the first FUZZ_RNG_SEED_SIZE bytes of the input.
  */
 FUZZ_STATIC uint32_t FUZZ_seed(uint8_t const **src, size_t* size) {
diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c
index 0e64400e6bc..9dff2895a9c 100644
--- a/tests/fuzz/zstd_helpers.c
+++ b/tests/fuzz/zstd_helpers.c
@@ -75,7 +75,7 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state)
     setRand(cctx, ZSTD_c_contentSizeFlag, 0, 1, state);
     setRand(cctx, ZSTD_c_checksumFlag, 0, 1, state);
     setRand(cctx, ZSTD_c_dictIDFlag, 0, 1, state);
-    /* Select long distance matchig parameters */
+    /* Select long distance matching parameters */
     setRand(cctx, ZSTD_c_enableLongDistanceMatching, 0, 1, state);
     setRand(cctx, ZSTD_c_ldmHashLog, ZSTD_HASHLOG_MIN, 16, state);
     setRand(cctx, ZSTD_c_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN,
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index d5f872d2900..1a31c78e292 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -909,7 +909,7 @@ static int basicUnitTests(U32 seed, double compressibility)
         CHECK_EQ(value, 5);
         CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
         CHECK_EQ(value, 2 MB);
-        /* Set the number of worksers and check the overlap log and job size. */
+        /* Set the number of workers and check the overlap log and job size. */
         CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) );
         CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
         CHECK_EQ(value, 5);
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index fb3c776bb21..75c179a0ba0 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -1235,7 +1235,7 @@ static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, const size
     return 0;
 }
 
-/* allocates buffer's arguments. returns success / failuere */
+/* allocates buffer's arguments. returns success / failure */
 static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable,
                           size_t nbFiles) {
     size_t pos = 0;
@@ -1508,7 +1508,7 @@ createMemoTableArray(const paramValues_t p,
 }
 
 /* Sets pc to random unmeasured set of parameters */
-/* specifiy strategy */
+/* specify strategy */
 static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTableArray, const ZSTD_strategy st)
 {
     size_t j;
@@ -1573,7 +1573,7 @@ BMK_benchMemInvertible( buffers_t buf, contexts_t ctx,
     display_params_tested(*comprParams);
     memset(&bResult, 0, sizeof(bResult));
 
-    /* warmimg up memory */
+    /* warming up memory */
     for (i = 0; i < buf.nbBlocks; i++) {
         if (mode != BMK_decodeOnly) {
             RDG_genBuffer(dstPtrs[i], dstCapacities[i], 0.10, 0.50, 1);
@@ -1992,7 +1992,7 @@ BMK_selectRandomStart( FILE* f,
 
 /* BMK_generate_cLevelTable() :
  * test a large number of configurations
- * and distribute them accross compression levels according to speed conditions.
+ * and distribute them across compression levels according to speed conditions.
  * display and save all intermediate results into rfName = "grillResults.txt".
  * the function automatically stops after g_timeLimit_s.
  * this function cannot error, it directly exit() in case of problem.
@@ -2270,7 +2270,7 @@ static winnerInfo_t climbOnce(const constraint_t target,
 
 /* Optimizes for a fixed strategy */
 
-/* flexible parameters: iterations of failed climbing (or if we do non-random, maybe this is when everything is close to visitied)
+/* flexible parameters: iterations of failed climbing (or if we do non-random, maybe this is when everything is close to visited)
    weight more on visit for bad results, less on good results/more on later results / ones with more failures.
    allocate memoTable here.
  */
diff --git a/tests/regression/test.c b/tests/regression/test.c
index 9e7b83c19da..812893b2730 100644
--- a/tests/regression/test.c
+++ b/tests/regression/test.c
@@ -153,7 +153,7 @@ static void print_help(void) {
     }
 }
 
-/** Parse the arguments. Teturn 0 on success. Print help on failure. */
+/** Parse the arguments. Return 0 on success. Print help on failure. */
 static int parse_args(int argc, char** argv) {
     int option_index = 0;
     int c;
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index f07477ae5b2..55c14ad5995 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -771,7 +771,7 @@ static int basicUnitTests(U32 seed, double compressibility)
     }
     DISPLAYLEVEL(3, "OK \n");
 
-    DISPLAYLEVEL(3, "test%3i : ZSTD_resetDStream() wtih dictionary : ", testNb++);
+    DISPLAYLEVEL(3, "test%3i : ZSTD_resetDStream() with dictionary : ", testNb++);
     {
         ZSTD_DCtx* dctx = ZSTD_createDCtx();
         /* We should succeed to decompress with the dictionary. */
@@ -1051,7 +1051,7 @@ static int basicUnitTests(U32 seed, double compressibility)
         inBuff.size = srcSize; assert(srcSize < COMPRESSIBLE_NOISE_LENGTH);
         inBuff.pos = 0;
     }
-    {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 4 KB, dictionary.filled);   /* intentionnally lies on estimatedSrcSize, to push cdict into targeting a small window size */
+    {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 4 KB, dictionary.filled);   /* intentionally lies on estimatedSrcSize, to push cdict into targeting a small window size */
         ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
         DISPLAYLEVEL(5, "cParams.windowLog = %u : ", cParams.windowLog);
         CHECK_Z( ZSTD_CCtx_refCDict(zc, cdict) );
@@ -2069,7 +2069,7 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest,
                     CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
                 }
 
-                /* multi-threading parameters. Only adjust ocassionally for small tests. */
+                /* multi-threading parameters. Only adjust occasionally for small tests. */
                 if (bigTests || (FUZ_rand(&lseed) & 0xF) == 0xF) {
                     U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1;
                     U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1;
diff --git a/zlibWrapper/README.md b/zlibWrapper/README.md
index 164b69ace81..e61767c468f 100644
--- a/zlibWrapper/README.md
+++ b/zlibWrapper/README.md
@@ -71,7 +71,7 @@ inflate with dictionary: hello, hello!
 The script used for compilation can be found at [zlibWrapper/Makefile](Makefile).
 
 
-#### The measurement of performace of Zstandard wrapper for zlib
+#### The measurement of performance of Zstandard wrapper for zlib
 
 The zstd distribution contains a tool called `zwrapbench` which can measure speed and ratio of zlib, zstd, and the wrapper.
 The benchmark is conducted using given filenames or synthetic data if filenames are not provided.
@@ -96,8 +96,8 @@ as this case is automatically detected.
 #### Reusing contexts
 
 The ordinary zlib compression of two files/streams allocates two contexts:
-- for the 1st file calls `deflateInit`, `deflate`, `...`, `deflate`, `defalateEnd`
-- for the 2nd file calls `deflateInit`, `deflate`, `...`, `deflate`, `defalateEnd`
+- for the 1st file calls `deflateInit`, `deflate`, `...`, `deflate`, `deflateEnd`
+- for the 2nd file calls `deflateInit`, `deflate`, `...`, `deflate`, `deflateEnd`
 
 The speed of compression can be improved with reusing a single context with following steps:
 - initialize the context with `deflateInit`
diff --git a/zlibWrapper/examples/fitblk.c b/zlibWrapper/examples/fitblk.c
index ee413c3ae91..6418ca38763 100644
--- a/zlibWrapper/examples/fitblk.c
+++ b/zlibWrapper/examples/fitblk.c
@@ -21,7 +21,7 @@
    data in order to determine how much of that input will compress to
    nearly the requested output block size.  The first pass generates
    enough deflate blocks to produce output to fill the requested
-   output size plus a specfied excess amount (see the EXCESS define
+   output size plus a specified excess amount (see the EXCESS define
    below).  The last deflate block may go quite a bit past that, but
    is discarded.  The second pass decompresses and recompresses just
    the compressed data that fit in the requested plus excess sized
@@ -217,7 +217,7 @@ int main(int argc, char **argv)
     if (ret == Z_MEM_ERROR)
         quit("out of memory");
 
-    /* set up for next reocmpression */
+    /* set up for next recompression */
     ret = inflateReset(&inf);
     assert(ret != Z_STREAM_ERROR);
     ret = deflateReset(&def);
diff --git a/zlibWrapper/examples/fitblk_original.c b/zlibWrapper/examples/fitblk_original.c
index c61de5c9967..20f351bfaf2 100644
--- a/zlibWrapper/examples/fitblk_original.c
+++ b/zlibWrapper/examples/fitblk_original.c
@@ -17,7 +17,7 @@
    data in order to determine how much of that input will compress to
    nearly the requested output block size.  The first pass generates
    enough deflate blocks to produce output to fill the requested
-   output size plus a specfied excess amount (see the EXCESS define
+   output size plus a specified excess amount (see the EXCESS define
    below).  The last deflate block may go quite a bit past that, but
    is discarded.  The second pass decompresses and recompresses just
    the compressed data that fit in the requested plus excess sized
@@ -198,7 +198,7 @@ int main(int argc, char **argv)
     if (ret == Z_MEM_ERROR)
         quit("out of memory");
 
-    /* set up for next reocmpression */
+    /* set up for next recompression */
     ret = inflateReset(&inf);
     assert(ret != Z_STREAM_ERROR);
     ret = deflateReset(&def);
diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c
index 99f9e11b2be..61031b9de79 100644
--- a/zlibWrapper/examples/zwrapbench.c
+++ b/zlibWrapper/examples/zwrapbench.c
@@ -193,7 +193,7 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
                 remaining -= thisBlockSize;
     }   }   }
 
-    /* warmimg up memory */
+    /* warming up memory */
     RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1);
 
     /* Bench */

From 203aac5bc62d9fab44283d7ab0325ddd5ce79989 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Mon, 15 Apr 2019 13:57:14 -0700
Subject: [PATCH 172/178] Update benchmarks in README

---
 README.md | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/README.md b/README.md
index 65dcd18213e..290341cc78e 100644
--- a/README.md
+++ b/README.md
@@ -28,10 +28,10 @@ a list of known ports and bindings is provided on [Zstandard homepage](http://ww
 ## Benchmarks
 
 For reference, several fast compression algorithms were tested and compared
-on a server running Linux Debian (`Linux version 4.14.0-3-amd64`),
-with a Core i7-6700K CPU @ 4.0GHz,
+on a server running Arch Linux (`Linux version 5.0.5-arch1-1`),
+with a Core i9-9900K CPU @ 5.0GHz,
 using [lzbench], an open-source in-memory benchmark by @inikep
-compiled with [gcc] 7.3.0,
+compiled with [gcc] 8.2.1,
 on the [Silesia compression corpus].
 
 [lzbench]: https://github.com/inikep/lzbench
@@ -40,14 +40,14 @@ on the [Silesia compression corpus].
 
 | Compressor name         | Ratio | Compression| Decompress.|
 | ---------------         | ------| -----------| ---------- |
-| **zstd 1.3.4 -1**       | 2.877 |   470 MB/s |  1380 MB/s |
-| zlib 1.2.11 -1          | 2.743 |   110 MB/s |   400 MB/s |
-| brotli 1.0.2 -0         | 2.701 |   410 MB/s |   430 MB/s |
-| quicklz 1.5.0 -1        | 2.238 |   550 MB/s |   710 MB/s |
-| lzo1x 2.09 -1           | 2.108 |   650 MB/s |   830 MB/s |
-| lz4 1.8.1               | 2.101 |   750 MB/s |  3700 MB/s |
-| snappy 1.1.4            | 2.091 |   530 MB/s |  1800 MB/s |
-| lzf 3.6 -1              | 2.077 |   400 MB/s |   860 MB/s |
+| **zstd 1.4.0 -1**       | 2.884 |   530 MB/s |  1360 MB/s |
+| zlib 1.2.11 -1          | 2.743 |   110 MB/s |   440 MB/s |
+| brotli 1.0.7 -0         | 2.701 |   430 MB/s |   470 MB/s |
+| quicklz 1.5.0 -1        | 2.238 |   600 MB/s |   800 MB/s |
+| lzo1x 2.09 -1           | 2.106 |   680 MB/s |   950 MB/s |
+| lz4 1.8.3               | 2.101 |   800 MB/s |  4220 MB/s |
+| snappy 1.1.4            | 2.073 |   580 MB/s |  2020 MB/s |
+| lzf 3.6 -1              | 2.077 |   440 MB/s |   930 MB/s |
 
 [zlib]: http://www.zlib.net/
 [LZ4]: http://www.lz4.org/

From bef61221a3958345b2f6931b06a24fe217aac800 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Mon, 15 Apr 2019 13:57:42 -0700
Subject: [PATCH 173/178] Update the changelog for zstd-1.4.0

---
 CHANGELOG | 38 +++++++++++++++++++++++++++++++++++---
 1 file changed, 35 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index cf82ccd9d11..61d0b3f56a3 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,38 @@
-dev
-api : Rename ZSTD_CCtxParam_getParameter to ZSTD_CCtxParams_getParameter
-api : Rename ZSTD_CCtxParam_setParameter to ZSTD_CCtxParams_setParameter
+v1.4.0
+perf: Improve level 1 compression speed in most scenarios by 6% by @gbtucker and @terrelln
+api: Move the advanced API, including all functions in the staging section, to the stable section
+api: Make ZSTD_e_flush and ZSTD_e_end block for maximum forward progress
+api: Rename ZSTD_CCtxParam_getParameter to ZSTD_CCtxParams_getParameter
+api: Rename ZSTD_CCtxParam_setParameter to ZSTD_CCtxParams_setParameter
+api: Don't export ZSTDMT functions from the shared library by default
+api: Add ZSTD_decompressBound() to provide an upper bound on decompressed size by @shakeelrao
+api: Fix ZSTD_decompressDCtx() corner cases with a dictionary
+api: Move ZSTD_getDictID_*() functions to the stable section
+api: Add ZSTD_c_literalCompressionMode flag to enable or disable literal compression by @terrelln
+api: Allow compression parameters to be set when a dictionary is used
+api: Allow setting parameters before or after ZSTD_CCtx_loadDictionary() is called
+api: Fix ZSTD_estimateCStreamSize_usingCCtxParams()
+api: Setting ZSTD_d_maxWindowLog to 0 means use the default
+cli: Ensure that a dictionary is not used to compress itself by @shakeelrao
+cli: Add --[no-]compress-literals flag to enable or disable literal compression
+doc: Update the examples to use the advanced API
+doc: Explain how to transition from old streaming functions to the advanced API in the header
+build: Improve the Windows release packages
+build: Improve CMake build by @hjmjohnson
+build: Build fixes for FreeBSD by @lwhsu
+build: Remove redundant warnings by @thatsafunnyname
+build: Fix tests on OpenBSD by @bket
+build: Extend fuzzer build system to work with the new clang engine
+build: CMake now creates the libzstd.so.1 symlink
+build: Improve Menson build by @lzutao
+misc: Fix symbolic link detection on FreeBSD
+misc: Use physical core count for -T0 on FreeBSD by @cemeyer
+misc: Fix zstd --list on truncated files by @kostmo
+misc: Improve logging in debug mode by @felixhandte
+misc: Add CirrusCI tests by @lwhsu
+misc: Optimize dictionary memory usage in corner cases
+misc: Improve the dictionary builder on small or homogeneous data
+misc: Fix spelling across the repo by @jsoref
 
 v1.3.8
 perf: better decompression speed on large files (+7%) and cold dictionaries (+15%)

From 6996804d05d1f20359c4addb8256de1bac0e15c2 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Mon, 15 Apr 2019 16:21:46 -0700
Subject: [PATCH 174/178] [playTets] Don't pass status=none to dd

The OS X test machine on TravisCI doesn't recognize the parameter
https://travis-ci.org/facebook/zstd/jobs/520517933.
---
 tests/playTests.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/playTests.sh b/tests/playTests.sh
index d22f617e456..bce2710159e 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -49,7 +49,7 @@ fileRoundTripTest() {
 }
 
 truncateLastByte() {
-	dd bs=1 count=$(($(wc -c < "$1") - 1)) if="$1" status=none
+	dd bs=1 count=$(($(wc -c < "$1") - 1)) if="$1"
 }
 
 UNAME=$(uname)
@@ -836,7 +836,7 @@ FULL_COMPRESSED_FILE=${TEST_DATA_FILE}.zst
 TRUNCATED_COMPRESSED_FILE=truncated-input.txt.zst
 ./datagen -g50000 > $TEST_DATA_FILE
 $ZSTD -f $TEST_DATA_FILE -o $FULL_COMPRESSED_FILE
-dd bs=1 count=100 if=$FULL_COMPRESSED_FILE of=$TRUNCATED_COMPRESSED_FILE status=none
+dd bs=1 count=100 if=$FULL_COMPRESSED_FILE of=$TRUNCATED_COMPRESSED_FILE
 $ZSTD --list $TRUNCATED_COMPRESSED_FILE && die "-l must fail on truncated file"
 
 rm $TEST_DATA_FILE

From a48e32ca508a4ee07fc0525b8e2333341105cea7 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Mon, 15 Apr 2019 21:01:27 -0700
Subject: [PATCH 175/178] [fullbench] Fix measurements

---
 tests/fullbench.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/fullbench.c b/tests/fullbench.c
index 6e42d2103fb..b06e2edc53a 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -497,7 +497,7 @@ static int benchMem(unsigned benchNb,
         BMK_benchParams_t bp;
         BMK_runTime_t bestResult;
         bestResult.sumOfReturn = 0;
-        bestResult.nanoSecPerRun = (double)(-1);
+        bestResult.nanoSecPerRun = (double)TIMELOOP_NANOSEC * 2000000000;  /* hopefully large enough : must be larger than any potential measurement */
         assert(tfs != NULL);
 
         bp.benchFn = benchFunction;

From de0499f7fa315f36725c93b4d97bd2ceaf6a0dbb Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Mon, 15 Apr 2019 22:49:44 -0700
Subject: [PATCH 176/178] [libzstd] Require ZSTD_MULTITHREAD to create a
 ZSTDMT_CCtx

ZSTDMT was broken when compiled without ZSTD_MULTITHREAD defined,
because `ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbWorkerss)`
failed. It was detected by the MSVC test which runs the fuzzer with
multithreading disabled.

This is a very niche use case of a deprecated API, because the API is
inefficient and synchronous, since `threading.h` will be synchronous.
Users almost certainly don't want this, and anyone who tested their code
should realize that it is broken. Therefore, I think it is safe to
require `ZSTD_MULTITHREAD` to be defined to use ZSTDMT.
---
 CHANGELOG                      | 1 +
 lib/compress/zstdmt_compress.c | 4 ++++
 lib/compress/zstdmt_compress.h | 8 +++++++-
 3 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG b/CHANGELOG
index 61d0b3f56a3..bf04a8a131c 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -5,6 +5,7 @@ api: Make ZSTD_e_flush and ZSTD_e_end block for maximum forward progress
 api: Rename ZSTD_CCtxParam_getParameter to ZSTD_CCtxParams_getParameter
 api: Rename ZSTD_CCtxParam_setParameter to ZSTD_CCtxParams_setParameter
 api: Don't export ZSTDMT functions from the shared library by default
+api: Require ZSTD_MULTITHREAD to be defined to use ZSTDMT
 api: Add ZSTD_decompressBound() to provide an upper bound on decompressed size by @shakeelrao
 api: Fix ZSTD_decompressDCtx() corner cases with a dictionary
 api: Move ZSTD_getDictID_*() functions to the stable section
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 4b29fb3e4ee..d0b4ee84a1d 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -874,6 +874,10 @@ ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, ZSTD_customMem cMem)
     int initError;
     DEBUGLOG(3, "ZSTDMT_createCCtx_advanced (nbWorkers = %u)", nbWorkers);
 
+#ifndef ZSTD_MULTITHREAD
+    return NULL;
+#endif
+
     if (nbWorkers < 1) return NULL;
     nbWorkers = MIN(nbWorkers , ZSTDMT_NBWORKERS_MAX);
     if ((cMem.customAlloc!=NULL) ^ (cMem.customFree!=NULL))
diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h
index 8d274ce8d3c..12e6bcb3a33 100644
--- a/lib/compress/zstdmt_compress.h
+++ b/lib/compress/zstdmt_compress.h
@@ -19,12 +19,16 @@
 /* Note : This is an internal API.
  *        These APIs used to be exposed with ZSTDLIB_API,
  *        because it used to be the only way to invoke MT compression.
- *        Now, it's recommended to use ZSTD_compress_generic() instead.
+ *        Now, it's recommended to use ZSTD_compress2 and ZSTD_compressStream2()
+ *        instead.
  *
  *        If you depend on these APIs and can't switch, then define
  *        ZSTD_LEGACY_MULTITHREADED_API when making the dynamic library.
  *        However, we may completely remove these functions in a future
  *        release, so please switch soon.
+ *
+ *        This API requires ZSTD_MULTITHREAD to be defined during compilation,
+ *        otherwise ZSTDMT_createCCtx*() will fail.
  */
 
 #ifdef ZSTD_LEGACY_MULTITHREADED_API
@@ -51,7 +55,9 @@
 
 /* ===   Memory management   === */
 typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx;
+/* Requires ZSTD_MULTITHREAD to be defined during compilation, otherwise it will return NULL. */
 ZSTDMT_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbWorkers);
+/* Requires ZSTD_MULTITHREAD to be defined during compilation, otherwise it will return NULL. */
 ZSTDMT_API ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers,
                                                     ZSTD_customMem cMem);
 ZSTDMT_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx);

From 82d5a9870e1809e64faee64bb14acacd523e7166 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Mon, 15 Apr 2019 22:54:07 -0700
Subject: [PATCH 177/178] [visual] Compile with ZSTD_MULTITHREAD=1

This branch `appveyorTests` runs the tests that `master` runs, so
passing Appveyor means that the tests are fixed.
---
 build/VS2008/fuzzer/fuzzer.vcproj  | 8 ++++----
 build/VS2010/fuzzer/fuzzer.vcxproj | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/build/VS2008/fuzzer/fuzzer.vcproj b/build/VS2008/fuzzer/fuzzer.vcproj
index 46cc4d4f244..c81791f4cde 100644
--- a/build/VS2008/fuzzer/fuzzer.vcproj
+++ b/build/VS2008/fuzzer/fuzzer.vcproj
@@ -45,7 +45,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
-				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -121,7 +121,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
-				PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@@ -195,7 +195,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
-				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -272,7 +272,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\programs"
-				PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
diff --git a/build/VS2010/fuzzer/fuzzer.vcxproj b/build/VS2010/fuzzer/fuzzer.vcxproj
index 53881c194b8..de649de1c44 100644
--- a/build/VS2010/fuzzer/fuzzer.vcxproj
+++ b/build/VS2010/fuzzer/fuzzer.vcxproj
@@ -90,7 +90,7 @@
       
       Level4
       Disabled
-      WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
+      ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
       true
       false
     
@@ -105,7 +105,7 @@
       
       Level4
       Disabled
-      WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
+      ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
       true
       false
     
@@ -122,7 +122,7 @@
       MaxSpeed
       true
       true
-      WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
+      ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
       false
       false
       MultiThreaded
@@ -142,7 +142,7 @@
       MaxSpeed
       true
       true
-      WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
+      ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
       false
       false
       MultiThreaded

From a17fe4c9e51cefe28c1097e157cab26d7989ae0e Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Tue, 16 Apr 2019 10:22:47 -0700
Subject: [PATCH 178/178] [visual] Fix unreachable code warning

---
 lib/compress/zstdmt_compress.c | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index d0b4ee84a1d..38fbb907685 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -22,6 +22,7 @@
 /* ======   Dependencies   ====== */
 #include       /* memcpy, memset */
 #include       /* INT_MAX, UINT_MAX */
+#include "mem.h"         /* MEM_STATIC */
 #include "pool.h"        /* threadpool */
 #include "threading.h"   /* mutex */
 #include "zstd_compress_internal.h"  /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */
@@ -867,17 +868,13 @@ size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorker
     return ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, (int)nbWorkers);
 }
 
-ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, ZSTD_customMem cMem)
+MEM_STATIC ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced_internal(unsigned nbWorkers, ZSTD_customMem cMem)
 {
     ZSTDMT_CCtx* mtctx;
     U32 nbJobs = nbWorkers + 2;
     int initError;
     DEBUGLOG(3, "ZSTDMT_createCCtx_advanced (nbWorkers = %u)", nbWorkers);
 
-#ifndef ZSTD_MULTITHREAD
-    return NULL;
-#endif
-
     if (nbWorkers < 1) return NULL;
     nbWorkers = MIN(nbWorkers , ZSTDMT_NBWORKERS_MAX);
     if ((cMem.customAlloc!=NULL) ^ (cMem.customFree!=NULL))
@@ -906,6 +903,17 @@ ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, ZSTD_customMem cMem)
     return mtctx;
 }
 
+ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, ZSTD_customMem cMem)
+{
+#ifdef ZSTD_MULTITHREAD
+    return ZSTDMT_createCCtx_advanced_internal(nbWorkers, cMem);
+#else
+    (void)nbWorkers;
+    (void)cMem;
+    return NULL;
+#endif
+}
+
 ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbWorkers)
 {
     return ZSTDMT_createCCtx_advanced(nbWorkers, ZSTD_defaultCMem);

@@ -71,7 +80,9 @@ 
zstd 1.3.8 Manual