zstd/decompress/zstd_ddict.c

0001 /*
0002  * Copyright (c) Yann Collet, Facebook, Inc.
0003  * All rights reserved.
0004  *
0005  * This source code is licensed under both the BSD-style license (found in the
0006  * LICENSE file in the root directory of this source tree) and the GPLv2 (found
0007  * in the COPYING file in the root directory of this source tree).
0008  * You may select, at your option, one of the above-listed licenses.
0009  */
0010
0011 /* zstd_ddict.c :
0012  * concentrates all logic that needs to know the internals of ZSTD_DDict object */
0013
0014 /*-*******************************************************
0015 *  Dependencies
0016 *********************************************************/
0017 #include "../common/zstd_deps.h"   /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
0018 #include "../common/cpu.h"         /* bmi2 */
0019 #include "../common/mem.h"         /* low level memory routines */
0020 #define FSE_STATIC_LINKING_ONLY
0021 #include "../common/fse.h"
0022 #define HUF_STATIC_LINKING_ONLY
0023 #include "../common/huf.h"
0024 #include "zstd_decompress_internal.h"
0025 #include "zstd_ddict.h"
0026
0027
0028
0029
0030 /*-*******************************************************
0031 *  Types
0032 *********************************************************/
0033 struct ZSTD_DDict_s {
0034     void* dictBuffer;
0035     const void* dictContent;
0036     size_t dictSize;
0037     ZSTD_entropyDTables_t entropy;
0038     U32 dictID;
0039     U32 entropyPresent;
0040     ZSTD_customMem cMem;
0041 };  /* typedef'd to ZSTD_DDict within "zstd.h" */
0042
0043 const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict)
0044 {
0045     assert(ddict != NULL);
0046     return ddict->dictContent;
0047 }
0048
0049 size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict)
0050 {
0051     assert(ddict != NULL);
0052     return ddict->dictSize;
0053 }
0054
0055 void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
0056 {
0057     DEBUGLOG(4, "ZSTD_copyDDictParameters");
0058     assert(dctx != NULL);
0059     assert(ddict != NULL);
0060     dctx->dictID = ddict->dictID;
0061     dctx->prefixStart = ddict->dictContent;
0062     dctx->virtualStart = ddict->dictContent;
0063     dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize;
0064     dctx->previousDstEnd = dctx->dictEnd;
0065 #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
0066     dctx->dictContentBeginForFuzzing = dctx->prefixStart;
0067     dctx->dictContentEndForFuzzing = dctx->previousDstEnd;
0068 #endif
0069     if (ddict->entropyPresent) {
0070         dctx->litEntropy = 1;
0071         dctx->fseEntropy = 1;
0072         dctx->LLTptr = ddict->entropy.LLTable;
0073         dctx->MLTptr = ddict->entropy.MLTable;
0074         dctx->OFTptr = ddict->entropy.OFTable;
0075         dctx->HUFptr = ddict->entropy.hufTable;
0076         dctx->entropy.rep[0] = ddict->entropy.rep[0];
0077         dctx->entropy.rep[1] = ddict->entropy.rep[1];
0078         dctx->entropy.rep[2] = ddict->entropy.rep[2];
0079     } else {
0080         dctx->litEntropy = 0;
0081         dctx->fseEntropy = 0;
0082     }
0083 }
0084
0085
0086 static size_t
0087 ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict,
0088                            ZSTD_dictContentType_e dictContentType)
0089 {
0090     ddict->dictID = 0;
0091     ddict->entropyPresent = 0;
0092     if (dictContentType == ZSTD_dct_rawContent) return 0;
0093
0094     if (ddict->dictSize < 8) {
0095         if (dictContentType == ZSTD_dct_fullDict)
0096             return ERROR(dictionary_corrupted);   /* only accept specified dictionaries */
0097         return 0;   /* pure content mode */
0098     }
0099     {   U32 const magic = MEM_readLE32(ddict->dictContent);
0100         if (magic != ZSTD_MAGIC_DICTIONARY) {
0101             if (dictContentType == ZSTD_dct_fullDict)
0102                 return ERROR(dictionary_corrupted);   /* only accept specified dictionaries */
0103             return 0;   /* pure content mode */
0104         }
0105     }
0106     ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE);
0107
0108     /* load entropy tables */
0109     RETURN_ERROR_IF(ZSTD_isError(ZSTD_loadDEntropy(
0110             &ddict->entropy, ddict->dictContent, ddict->dictSize)),
0111         dictionary_corrupted, "");
0112     ddict->entropyPresent = 1;
0113     return 0;
0114 }
0115
0116
0117 static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict,
0118                                       const void* dict, size_t dictSize,
0119                                       ZSTD_dictLoadMethod_e dictLoadMethod,
0120                                       ZSTD_dictContentType_e dictContentType)
0121 {
0122     if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) {
0123         ddict->dictBuffer = NULL;
0124         ddict->dictContent = dict;
0125         if (!dict) dictSize = 0;
0126     } else {
0127         void* const internalBuffer = ZSTD_customMalloc(dictSize, ddict->cMem);
0128         ddict->dictBuffer = internalBuffer;
0129         ddict->dictContent = internalBuffer;
0130         if (!internalBuffer) return ERROR(memory_allocation);
0131         ZSTD_memcpy(internalBuffer, dict, dictSize);
0132     }
0133     ddict->dictSize = dictSize;
0134     ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001);  /* cover both little and big endian */
0135
0136     /* parse dictionary content */
0137     FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , "");
0138
0139     return 0;
0140 }
0141
0142 ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
0143                                       ZSTD_dictLoadMethod_e dictLoadMethod,
0144                                       ZSTD_dictContentType_e dictContentType,
0145                                       ZSTD_customMem customMem)
0146 {
0147     if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL;
0148
0149     {   ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_customMalloc(sizeof(ZSTD_DDict), customMem);
0150         if (ddict == NULL) return NULL;
0151         ddict->cMem = customMem;
0152         {   size_t const initResult = ZSTD_initDDict_internal(ddict,
0153                                             dict, dictSize,
0154                                             dictLoadMethod, dictContentType);
0155             if (ZSTD_isError(initResult)) {
0156                 ZSTD_freeDDict(ddict);
0157                 return NULL;
0158         }   }
0159         return ddict;
0160     }
0161 }
0162
0163 /*! ZSTD_createDDict() :
0164 *   Create a digested dictionary, to start decompression without startup delay.
0165 *   `dict` content is copied inside DDict.
0166 *   Consequently, `dict` can be released after `ZSTD_DDict` creation */
0167 ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
0168 {
0169     ZSTD_customMem const allocator = { NULL, NULL, NULL };
0170     return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator);
0171 }
0172
0173 /*! ZSTD_createDDict_byReference() :
0174  *  Create a digested dictionary, to start decompression without startup delay.
0175  *  Dictionary content is simply referenced, it will be accessed during decompression.
0176  *  Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */
0177 ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize)
0178 {
0179     ZSTD_customMem const allocator = { NULL, NULL, NULL };
0180     return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator);
0181 }
0182
0183
0184 const ZSTD_DDict* ZSTD_initStaticDDict(
0185                                 void* sBuffer, size_t sBufferSize,
0186                                 const void* dict, size_t dictSize,
0187                                 ZSTD_dictLoadMethod_e dictLoadMethod,
0188                                 ZSTD_dictContentType_e dictContentType)
0189 {
0190     size_t const neededSpace = sizeof(ZSTD_DDict)
0191                              + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
0192     ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer;
0193     assert(sBuffer != NULL);
0194     assert(dict != NULL);
0195     if ((size_t)sBuffer & 7) return NULL;   /* 8-aligned */
0196     if (sBufferSize < neededSpace) return NULL;
0197     if (dictLoadMethod == ZSTD_dlm_byCopy) {
0198         ZSTD_memcpy(ddict+1, dict, dictSize);  /* local copy */
0199         dict = ddict+1;
0200     }
0201     if (ZSTD_isError( ZSTD_initDDict_internal(ddict,
0202                                               dict, dictSize,
0203                                               ZSTD_dlm_byRef, dictContentType) ))
0204         return NULL;
0205     return ddict;
0206 }
0207
0208
0209 size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
0210 {
0211     if (ddict==NULL) return 0;   /* support free on NULL */
0212     {   ZSTD_customMem const cMem = ddict->cMem;
0213         ZSTD_customFree(ddict->dictBuffer, cMem);
0214         ZSTD_customFree(ddict, cMem);
0215         return 0;
0216     }
0217 }
0218
0219 /*! ZSTD_estimateDDictSize() :
0220  *  Estimate amount of memory that will be needed to create a dictionary for decompression.
0221  *  Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */
0222 size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod)
0223 {
0224     return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
0225 }
0226
0227 size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
0228 {
0229     if (ddict==NULL) return 0;   /* support sizeof on NULL */
0230     return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ;
0231 }
0232
0233 /*! ZSTD_getDictID_fromDDict() :
0234  *  Provides the dictID of the dictionary loaded into `ddict`.
0235  *  If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
0236  *  Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
0237 unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
0238 {
0239     if (ddict==NULL) return 0;
0240     return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize);
0241 }