mirrors/zstd
0
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2024-11-24 04:06:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

s/chunk/frame/

Browse Source
This commit is contained in:
Sean Purcell 2017-04-12 11:06:00 -07:00
parent e80f1d74b3
commit 5ee1135f30
7 changed files with 142 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
contrib/seekable_format/examples/seekable_compression.c
View File

@ -59,7 +59,7 @@ static size_t fclose_orDie(FILE* file)
exit(6);
}
static void compressFile_orDie(const char* fname, const char* outName, int cLevel, unsigned chunkSize)
static void compressFile_orDie(const char* fname, const char* outName, int cLevel, unsigned frameSize)
{
FILE* const fin = fopen_orDie(fname, "rb");
FILE* const fout = fopen_orDie(outName, "wb");
@ -70,7 +70,7 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve
ZSTD_seekable_CStream* const cstream = ZSTD_seekable_createCStream();
if (cstream==NULL) { fprintf(stderr, "ZSTD_seekable_createCStream() error \n"); exit(10); }
size_t const initResult = ZSTD_seekable_initCStream(cstream, cLevel, 1, chunkSize);
size_t const initResult = ZSTD_seekable_initCStream(cstream, cLevel, 1, frameSize);
if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_seekable_initCStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
size_t read, toRead = buffInSize;
@ -116,15 +116,15 @@ int main(int argc, const char** argv) {
if (argc!=3) {
printf("wrong arguments\n");
printf("usage:\n");
printf("%s FILE CHUNK_SIZE\n", exeName);
printf("%s FILE FRAME_SIZE\n", exeName);
return 1;
}
{ const char* const inFileName = argv[1];
unsigned const chunkSize = (unsigned)atoi(argv[2]);
unsigned const frameSize = (unsigned)atoi(argv[2]);
const char* const outFileName = createOutFilename_orDie(inFileName);
compressFile_orDie(inFileName, outFileName, 5, chunkSize);
compressFile_orDie(inFileName, outFileName, 5, frameSize);
}
return 0;

36
contrib/seekable_format/zstd_seekable.h
View File

@ -9,17 +9,17 @@ static const unsigned ZSTD_seekTableFooterSize = 9;
#define ZSTD_SEEKABLE_MAGICNUMBER 0x8F92EAB1
#define ZSTD_SEEKABLE_MAXCHUNKS 0x8000000U
#define ZSTD_SEEKABLE_MAXFRAMES 0x8000000U
/* 0xFE03F607 is the largest number x such that ZSTD_compressBound(x) fits in a 32-bit integer */
#define ZSTD_SEEKABLE_MAX_CHUNK_DECOMPRESSED_SIZE 0xFE03F607
#define ZSTD_SEEKABLE_MAX_FRAME_DECOMPRESSED_SIZE 0xFE03F607
/*-****************************************************************************
* Seekable Format
*
* The seekable format splits the compressed data into a series of "chunks",
* The seekable format splits the compressed data into a series of "frames",
* each compressed individually so that decompression of a section in the
* middle of an archive only requires zstd to decompress at most a chunk's
* middle of an archive only requires zstd to decompress at most a frame's
* worth of extra data, instead of the entire archive.
******************************************************************************/
@ -37,15 +37,15 @@ typedef struct ZSTD_seekable_DStream_s ZSTD_seekable_DStream;
* compressor.
*
* Data streamed to the seekable compressor will automatically be split into
* chunks of size `maxChunkSize` (provided in ZSTD_seekable_initCStream()),
* or if none is provided, will be cut off whenver ZSTD_endChunk() is called
* or when the default maximum chunk size is reached (approximately 4GB).
* frames of size `maxFrameSize` (provided in ZSTD_seekable_initCStream()),
* or if none is provided, will be cut off whenver ZSTD_endFrame() is called
* or when the default maximum frame size is reached (approximately 4GB).
*
* Use ZSTD_seekable_initCStream() to initialize a ZSTD_seekable_CStream object
* for a new compression operation.
* `maxChunkSize` indicates the size at which to automatically start a new
* seekable frame. `maxChunkSize == 0` implies the default maximum size.
* `checksumFlag` indicates whether or not the seek table should include chunk
* `maxFrameSize` indicates the size at which to automatically start a new
* seekable frame. `maxFrameSize == 0` implies the default maximum size.
* `checksumFlag` indicates whether or not the seek table should include frame
* checksums on the uncompressed data for verification.
* @return : a size hint for input to provide for compression, or an error code
* checkable with ZSTD_isError()
@ -61,14 +61,14 @@ typedef struct ZSTD_seekable_DStream_s ZSTD_seekable_DStream;
* value will work fine.
* Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize()
*
* At any time, call ZSTD_seekable_endChunk() to end the current chunk and
* At any time, call ZSTD_seekable_endFrame() to end the current frame and
* start a new one.
*
* ZSTD_endStream() will end the current chunk, and then write the seek table
* so that decompressors can efficiently find compressed chunks.
* ZSTD_endStream() may return a number > 0 if it was unable to flush all the
* necessary data to `output`. In this case, it should be called again until
* all remaining data is flushed out and 0 is returned.
* ZSTD_seekable_endStream() will end the current frame, and then write the seek
* table so that decompressors can efficiently find compressed frames.
* ZSTD_seekable_endStream() may return a number > 0 if it was unable to flush
* all the necessary data to `output`. In this case, it should be called again
* until all remaining data is flushed out and 0 is returned.
******************************************************************************/
/*===== Seekable compressor management =====*/
@ -76,9 +76,9 @@ ZSTDLIB_API ZSTD_seekable_CStream* ZSTD_seekable_createCStream(void);
ZSTDLIB_API size_t ZSTD_seekable_freeCStream(ZSTD_seekable_CStream* zcs);
/*===== Seekable compression functions =====*/
ZSTDLIB_API size_t ZSTD_seekable_initCStream(ZSTD_seekable_CStream* zcs, int compressionLevel, int checksumFlag, unsigned maxChunkSize);
ZSTDLIB_API size_t ZSTD_seekable_initCStream(ZSTD_seekable_CStream* zcs, int compressionLevel, int checksumFlag, unsigned maxFrameSize);
ZSTDLIB_API size_t ZSTD_seekable_compressStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
ZSTDLIB_API size_t ZSTD_seekable_endChunk(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output);
ZSTDLIB_API size_t ZSTD_seekable_endFrame(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output);
ZSTDLIB_API size_t ZSTD_seekable_endStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output);
/*-****************************************************************************

20
contrib/seekable_format/zstd_seekable_compression_format.md
View File

@ -18,7 +18,7 @@ Distribution of this document is unlimited.
## Introduction
This document defines a format for compressed data to be stored so that subranges of the data can be efficiently decompressed without requiring the entire document to be decompressed.
This is done by splitting up the input data into chunks,
This is done by splitting up the input data into frames,
each of which are compressed independently,
and so can be decompressed independently.
Decompression then takes advantage of a provided 'seek table', which allows the decompressor to immediately jump to the desired data. This is done in a way that is compatible with the original Zstandard format by placing the seek table in a Zstandard skippable frame.
@ -31,7 +31,7 @@ In this document:
## Format
The format consists of a number of chunks (Zstandard compressed frames and skippable frames), followed by a final skippable frame at the end containing the seek table.
The format consists of a number of frames (Zstandard compressed frames and skippable frames), followed by a final skippable frame at the end containing the seek table.
### Seek Table Format
The structure of the seek table frame is as follows:
@ -58,7 +58,7 @@ This is for compatibility with [Zstandard skippable frames].
#### `Seek_Table_Footer`
The seek table footer format is as follows:
|`Number_Of_Chunks`|`Seek_Table_Descriptor`|`Seekable_Magic_Number`|
|`Number_Of_Frames`|`Seek_Table_Descriptor`|`Seekable_Magic_Number`|
|------------------|-----------------------|-----------------------|
| 4 bytes | 1 byte | 4 bytes |
@ -68,9 +68,9 @@ Value : 0x8F92EAB1.
This value must be the last bytes present in the compressed file so that decoders
can efficiently find it and determine if there is an actual seek table present.
__`Number_Of_Chunks`__
__`Number_Of_Frames`__
The number of stored chunks in the data.
The number of stored frames in the data.
__`Seek_Table_Descriptor`__
@ -87,13 +87,13 @@ for example the addition of inline dictionaries.
__`Checksum_Flag`__
If the checksum flag is set, each of the seek table entries contains a 4 byte checksum of the uncompressed data contained in its chunk.
If the checksum flag is set, each of the seek table entries contains a 4 byte checksum of the uncompressed data contained in its frame.
`Reserved_Bits` are not currently used but may be used in the future for breaking changes, so a compliant decoder should ensure they are set to 0. `Unused_Bits` may be used in the future for non-breaking changes, so a compliant decoder should not interpret these bits.
#### __`Seek_Table_Entries`__
`Seek_Table_Entries` consists of `Number_Of_Chunks` (one for each chunk in the data, not including the seek table frame) entries of the following form, in sequence:
`Seek_Table_Entries` consists of `Number_Of_Frames` (one for each frame in the data, not including the seek table frame) entries of the following form, in sequence:
|`Compressed_Size`|`Decompressed_Size`|`[Checksum]`|
|-----------------|-------------------|------------|
@ -101,12 +101,12 @@ If the checksum flag is set, each of the seek table entries contains a 4 byte ch
__`Compressed_Size`__
The compressed size of the chunk.
The cumulative sum of the `Compressed_Size` fields of chunks `0` to `i` gives the offset in the compressed file of chunk `i+1`.
The compressed size of the frame.
The cumulative sum of the `Compressed_Size` fields of frames `0` to `i` gives the offset in the compressed file of frame `i+1`.
__`Decompressed_Size`__
The size of the decompressed data contained in the chunk. For skippable or otherwise empty frames, this value is 0.
The size of the decompressed data contained in the frame. For skippable or otherwise empty frames, this value is 0.
__`Checksum`__

160
contrib/seekable_format/zstdseek_compress.c
View File

@ -20,24 +20,24 @@ typedef struct {
U32 cSize;
U32 dSize;
U32 checksum;
} chunklogEntry_t;
} framelogEntry_t;
typedef struct {
chunklogEntry_t* entries;
framelogEntry_t* entries;
U32 size;
U32 capacity;
} chunklog_t;
} framelog_t;
struct ZSTD_seekable_CStream_s {
ZSTD_CStream* cstream;
chunklog_t chunklog;
framelog_t framelog;
U32 chunkCSize;
U32 chunkDSize;
U32 frameCSize;
U32 frameDSize;
XXH64_state_t xxhState;
U32 maxChunkSize;
U32 maxFrameSize;
int checksumFlag;
@ -56,11 +56,11 @@ ZSTD_seekable_CStream* ZSTD_seekable_createCStream()
if (zcs->cstream == NULL) goto failed1;
/* allocate some initial space */
{ size_t const CHUNKLOG_STARTING_CAPACITY = 16;
zcs->chunklog.entries =
malloc(sizeof(chunklogEntry_t) * CHUNKLOG_STARTING_CAPACITY);
if (zcs->chunklog.entries == NULL) goto failed2;
zcs->chunklog.capacity = CHUNKLOG_STARTING_CAPACITY;
{ size_t const FRAMELOG_STARTING_CAPACITY = 16;
zcs->framelog.entries =
malloc(sizeof(framelogEntry_t) * FRAMELOG_STARTING_CAPACITY);
if (zcs->framelog.entries == NULL) goto failed2;
zcs->framelog.capacity = FRAMELOG_STARTING_CAPACITY;
}
return zcs;
@ -76,7 +76,7 @@ size_t ZSTD_seekable_freeCStream(ZSTD_seekable_CStream* zcs)
{
if (zcs == NULL) return 0; /* support free on null */
ZSTD_freeCStream(zcs->cstream);
free(zcs->chunklog.entries);
free(zcs->framelog.entries);
free(zcs);
return 0;
@ -85,20 +85,20 @@ size_t ZSTD_seekable_freeCStream(ZSTD_seekable_CStream* zcs)
size_t ZSTD_seekable_initCStream(ZSTD_seekable_CStream* zcs,
int compressionLevel,
int checksumFlag,
U32 maxChunkSize)
U32 maxFrameSize)
{
zcs->chunklog.size = 0;
zcs->chunkCSize = 0;
zcs->chunkDSize = 0;
zcs->framelog.size = 0;
zcs->frameCSize = 0;
zcs->frameDSize = 0;
/* make sure maxChunkSize has a reasonable value */
if (maxChunkSize > ZSTD_SEEKABLE_MAX_CHUNK_DECOMPRESSED_SIZE) {
/* make sure maxFrameSize has a reasonable value */
if (maxFrameSize > ZSTD_SEEKABLE_MAX_FRAME_DECOMPRESSED_SIZE) {
return ERROR(compressionParameter_unsupported);
}
zcs->maxChunkSize = maxChunkSize
? maxChunkSize
: ZSTD_SEEKABLE_MAX_CHUNK_DECOMPRESSED_SIZE;
zcs->maxFrameSize = maxFrameSize
? maxFrameSize
: ZSTD_SEEKABLE_MAX_FRAME_DECOMPRESSED_SIZE;
zcs->checksumFlag = checksumFlag;
if (zcs->checksumFlag) {
@ -110,58 +110,58 @@ size_t ZSTD_seekable_initCStream(ZSTD_seekable_CStream* zcs,
return ZSTD_initCStream(zcs->cstream, compressionLevel);
}
static size_t ZSTD_seekable_logChunk(ZSTD_seekable_CStream* zcs)
static size_t ZSTD_seekable_logFrame(ZSTD_seekable_CStream* zcs)
{
if (zcs->chunklog.size == ZSTD_SEEKABLE_MAXCHUNKS)
return ERROR(chunkIndex_tooLarge);
if (zcs->framelog.size == ZSTD_SEEKABLE_MAXFRAMES)
return ERROR(frameIndex_tooLarge);
zcs->chunklog.entries[zcs->chunklog.size] = (chunklogEntry_t)
zcs->framelog.entries[zcs->framelog.size] = (framelogEntry_t)
{
.cSize = zcs->chunkCSize,
.dSize = zcs->chunkDSize,
.cSize = zcs->frameCSize,
.dSize = zcs->frameDSize,
};
if (zcs->checksumFlag)
zcs->chunklog.entries[zcs->chunklog.size].checksum =
zcs->framelog.entries[zcs->framelog.size].checksum =
/* take lower 32 bits of digest */
XXH64_digest(&zcs->xxhState) & 0xFFFFFFFFU;
zcs->chunklog.size++;
zcs->framelog.size++;
/* grow the buffer if required */
if (zcs->chunklog.size == zcs->chunklog.capacity) {
if (zcs->framelog.size == zcs->framelog.capacity) {
/* exponential size increase for constant amortized runtime */
size_t const newCapacity = zcs->chunklog.capacity * 2;
chunklogEntry_t* const newEntries = realloc(zcs->chunklog.entries,
sizeof(chunklogEntry_t) * newCapacity);
size_t const newCapacity = zcs->framelog.capacity * 2;
framelogEntry_t* const newEntries = realloc(zcs->framelog.entries,
sizeof(framelogEntry_t) * newCapacity);
if (newEntries == NULL) return ERROR(memory_allocation);
zcs->chunklog.entries = newEntries;
zcs->chunklog.capacity = newCapacity;
zcs->framelog.entries = newEntries;
zcs->framelog.capacity = newCapacity;
}
return 0;
}
size_t ZSTD_seekable_endChunk(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output)
size_t ZSTD_seekable_endFrame(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output)
{
size_t const prevOutPos = output->pos;
/* end the frame */
size_t ret = ZSTD_endStream(zcs->cstream, output);
zcs->chunkCSize += output->pos - prevOutPos;
zcs->frameCSize += output->pos - prevOutPos;
/* need to flush before doing the rest */
if (ret) return ret;
/* frame done */
/* store the chunk data for later */
ret = ZSTD_seekable_logChunk(zcs);
/* store the frame data for later */
ret = ZSTD_seekable_logFrame(zcs);
if (ret) return ret;
/* reset for the next chunk */
zcs->chunkCSize = 0;
zcs->chunkDSize = 0;
/* reset for the next frame */
zcs->frameCSize = 0;
zcs->frameDSize = 0;
ZSTD_resetCStream(zcs->cstream, 0);
if (zcs->checksumFlag)
@ -175,9 +175,9 @@ size_t ZSTD_seekable_compressStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer*
const BYTE* const inBase = (const BYTE*) input->src + input->pos;
size_t inLen = input->size - input->pos;
inLen = MIN(inLen, (size_t)(zcs->maxChunkSize - zcs->chunkDSize));
inLen = MIN(inLen, (size_t)(zcs->maxFrameSize - zcs->frameDSize));
/* if we haven't finished flushing the last chunk, don't start writing a new one */
/* if we haven't finished flushing the last frame, don't start writing a new one */
if (inLen > 0) {
ZSTD_inBuffer inTmp = { inBase, inLen, 0 };
size_t const prevOutPos = output->pos;
@ -188,31 +188,31 @@ size_t ZSTD_seekable_compressStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer*
XXH64_update(&zcs->xxhState, inBase, inTmp.pos);
}
zcs->chunkCSize += output->pos - prevOutPos;
zcs->chunkDSize += inTmp.pos;
zcs->frameCSize += output->pos - prevOutPos;
zcs->frameDSize += inTmp.pos;
input->pos += inTmp.pos;
if (ZSTD_isError(ret)) return ret;
}
if (zcs->maxChunkSize == zcs->chunkDSize) {
/* log the chunk and start over */
size_t const ret = ZSTD_seekable_endChunk(zcs, output);
if (zcs->maxFrameSize == zcs->frameDSize) {
/* log the frame and start over */
size_t const ret = ZSTD_seekable_endFrame(zcs, output);
if (ZSTD_isError(ret)) return ret;
/* get the client ready for the next chunk */
return (size_t)zcs->maxChunkSize;
/* get the client ready for the next frame */
return (size_t)zcs->maxFrameSize;
}
return (size_t)(zcs->maxChunkSize - zcs->chunkDSize);
return (size_t)(zcs->maxFrameSize - zcs->frameDSize);
}
static size_t ZSTD_seekable_seekTableSize(ZSTD_seekable_CStream* zcs)
{
size_t const sizePerChunk = 8 + (zcs->checksumFlag?4:0);
size_t const sizePerFrame = 8 + (zcs->checksumFlag?4:0);
size_t const seekTableLen = ZSTD_skippableHeaderSize +
sizePerChunk * zcs->chunklog.size +
sizePerFrame * zcs->framelog.size +
ZSTD_seekTableFooterSize;
return seekTableLen;
@ -224,60 +224,60 @@ static size_t ZSTD_seekable_writeSeekTable(ZSTD_seekable_CStream* zcs, ZSTD_outB
BYTE tmp[4]; /* so that we can work with buffers too small to write a whole word to */
/* repurpose
* zcs->chunkDSize: the current index in the table and
* zcs->chunkCSize: the amount of the table written so far
* zcs->frameDSize: the current index in the table and
* zcs->frameCSize: the amount of the table written so far
*
* This function is written this way so that if it has to return early
* because of a small buffer, it can keep going where it left off.
*/
size_t const sizePerChunk = 8 + (zcs->checksumFlag?4:0);
size_t const sizePerFrame = 8 + (zcs->checksumFlag?4:0);
size_t const seekTableLen = ZSTD_seekable_seekTableSize(zcs);
#define st_write32(x, o) \
do { \
if (zcs->chunkCSize < (o) + 4) { \
if (zcs->frameCSize < (o) + 4) { \
size_t const lenWrite = MIN(output->size - output->pos, \
(o) + 4 - zcs->chunkCSize); \
(o) + 4 - zcs->frameCSize); \
MEM_writeLE32(tmp, (x)); \
memcpy(op + output->pos, tmp + (zcs->chunkCSize - (o)), lenWrite); \
zcs->chunkCSize += lenWrite; \
memcpy(op + output->pos, tmp + (zcs->frameCSize - (o)), lenWrite); \
zcs->frameCSize += lenWrite; \
output->pos += lenWrite; \
if (lenWrite < 4) return seekTableLen - zcs->chunkCSize; \
if (lenWrite < 4) return seekTableLen - zcs->frameCSize; \
} \
} while (0)
st_write32(ZSTD_MAGIC_SKIPPABLE_START | 0xE, 0);
st_write32(seekTableLen - ZSTD_skippableHeaderSize, 4);
while (zcs->chunkDSize < zcs->chunklog.size) {
st_write32(zcs->chunklog.entries[zcs->chunkDSize].cSize,
ZSTD_skippableHeaderSize + sizePerChunk * zcs->chunkDSize);
st_write32(zcs->chunklog.entries[zcs->chunkDSize].dSize,
ZSTD_skippableHeaderSize + sizePerChunk * zcs->chunkDSize + 4);
while (zcs->frameDSize < zcs->framelog.size) {
st_write32(zcs->framelog.entries[zcs->frameDSize].cSize,
ZSTD_skippableHeaderSize + sizePerFrame * zcs->frameDSize);
st_write32(zcs->framelog.entries[zcs->frameDSize].dSize,
ZSTD_skippableHeaderSize + sizePerFrame * zcs->frameDSize + 4);
if (zcs->checksumFlag) {
st_write32(zcs->chunklog.entries[zcs->chunkDSize].checksum,
ZSTD_skippableHeaderSize + sizePerChunk * zcs->chunkDSize + 8);
st_write32(zcs->framelog.entries[zcs->frameDSize].checksum,
ZSTD_skippableHeaderSize + sizePerFrame * zcs->frameDSize + 8);
}
zcs->chunkDSize++;
zcs->frameDSize++;
}
st_write32(zcs->chunklog.size, seekTableLen - ZSTD_seekTableFooterSize);
st_write32(zcs->framelog.size, seekTableLen - ZSTD_seekTableFooterSize);
if (output->size - output->pos < 1) return seekTableLen - zcs->chunkCSize;
if (zcs->chunkCSize < seekTableLen - 4) {
if (output->size - output->pos < 1) return seekTableLen - zcs->frameCSize;
if (zcs->frameCSize < seekTableLen - 4) {
BYTE sfd = 0;
sfd |= (zcs->checksumFlag) << 7;
op[output->pos] = sfd;
output->pos++;
zcs->chunkCSize++;
zcs->frameCSize++;
}
st_write32(ZSTD_SEEKABLE_MAGICNUMBER, seekTableLen - 4);
if (zcs->chunkCSize != seekTableLen) return ERROR(GENERIC);
if (zcs->frameCSize != seekTableLen) return ERROR(GENERIC);
return 0;
#undef st_write32
@ -285,11 +285,11 @@ static size_t ZSTD_seekable_writeSeekTable(ZSTD_seekable_CStream* zcs, ZSTD_outB
size_t ZSTD_seekable_endStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output)
{
if (!zcs->writingSeekTable && zcs->chunkDSize) {
const size_t endChunk = ZSTD_seekable_endChunk(zcs, output);
if (ZSTD_isError(endChunk)) return endChunk;
if (!zcs->writingSeekTable && zcs->frameDSize) {
const size_t endFrame = ZSTD_seekable_endFrame(zcs, output);
if (ZSTD_isError(endFrame)) return endFrame;
/* return an accurate size hint */
if (endChunk) return endChunk + ZSTD_seekable_seekTableSize(zcs);
if (endFrame) return endFrame + ZSTD_seekable_seekTableSize(zcs);
}
zcs->writingSeekTable = 1;

54
contrib/seekable_format/zstdseek_decompress.c
View File

@ -29,11 +29,11 @@ typedef struct {
int checksumFlag;
} seekTable_t;
/** ZSTD_seekable_offsetToChunk() :
* Performs a binary search to find the last chunk with a decompressed offset
/** ZSTD_seekable_offsetToFrame() :
* Performs a binary search to find the last frame with a decompressed offset
* <= pos
* @return : the chunk's index */
static U32 ZSTD_seekable_offsetToChunk(const seekTable_t* table, U64 pos)
* @return : the frame's index */
static U32 ZSTD_seekable_offsetToFrame(const seekTable_t* table, U64 pos)
{
U32 lo = 0;
U32 hi = table->tableLen;
@ -61,7 +61,7 @@ struct ZSTD_seekable_DStream_s {
ZSTD_DStream* dstream;
seekTable_t seekTable;
U32 curChunk;
U32 curFrame;
U64 compressedOffset;
U64 decompressedOffset;
@ -107,7 +107,7 @@ size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable_DStream* zds, const void* src,
{
const BYTE* ip = (const BYTE*)src + srcSize;
U32 numChunks;
U32 numFrames;
int checksumFlag;
U32 sizePerEntry;
@ -129,10 +129,10 @@ size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable_DStream* zds, const void* src,
}
}
numChunks = MEM_readLE32(ip-9);
numFrames = MEM_readLE32(ip-9);
sizePerEntry = 8 + (checksumFlag?4:0);
{ U32 const tableSize = sizePerEntry * numChunks;
{ U32 const tableSize = sizePerEntry * numFrames;
U32 const frameSize = tableSize + ZSTD_seekTableFooterSize + ZSTD_skippableHeaderSize;
const BYTE* base = ip - frameSize;
@ -148,7 +148,7 @@ size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable_DStream* zds, const void* src,
{ /* Allocate an extra entry at the end so that we can do size
* computations on the last element without special case */
seekEntry_t* entries = malloc(sizeof(seekEntry_t) * (numChunks + 1));
seekEntry_t* entries = malloc(sizeof(seekEntry_t) * (numFrames + 1));
const BYTE* tableBase = base + ZSTD_skippableHeaderSize;
U32 idx;
@ -163,7 +163,7 @@ size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable_DStream* zds, const void* src,
}
/* compute cumulative positions */
for (idx = 0, pos = 0; idx < numChunks; idx++) {
for (idx = 0, pos = 0; idx < numFrames; idx++) {
entries[idx].cOffset = cOffset;
entries[idx].dOffset = dOffset;
@ -174,11 +174,11 @@ size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable_DStream* zds, const void* src,
pos += 4;
}
}
entries[numChunks].cOffset = cOffset;
entries[numChunks].dOffset = dOffset;
entries[numFrames].cOffset = cOffset;
entries[numFrames].dOffset = dOffset;
zds->seekTable.entries = entries;
zds->seekTable.tableLen = numChunks;
zds->seekTable.tableLen = numFrames;
zds->seekTable.checksumFlag = checksumFlag;
return 0;
}
@ -197,7 +197,7 @@ size_t ZSTD_seekable_initDStream(ZSTD_seekable_DStream* zds, U64 rangeStart, U64
zds->stage = zsds_seek;
/* force a seek first */
zds->curChunk = (U32) -1;
zds->curFrame = (U32) -1;
zds->compressedOffset = (U64) -1;
zds->decompressedOffset = (U64) -1;
@ -271,7 +271,7 @@ size_t ZSTD_seekable_decompressStream(ZSTD_seekable_DStream* zds, ZSTD_outBuffer
/* need more input */
return MIN(
ZSTD_DStreamInSize(),
(size_t)(jt->entries[zds->curChunk + 1]
(size_t)(jt->entries[zds->curFrame + 1]
.cOffset -
zds->compressedOffset));
}
@ -281,7 +281,7 @@ size_t ZSTD_seekable_decompressStream(ZSTD_seekable_DStream* zds, ZSTD_outBuffer
{
U64 const toDecompress =
MIN(zds->targetEnd,
jt->entries[zds->curChunk + 1].dOffset) -
jt->entries[zds->curFrame + 1].dOffset) -
zds->decompressedOffset;
size_t const prevInputPos = input->pos;
@ -305,7 +305,7 @@ size_t ZSTD_seekable_decompressStream(ZSTD_seekable_DStream* zds, ZSTD_outBuffer
if (ret == 0) {
/* verify the checksum */
U32 const digest = XXH64_digest(&zds->xxhState) & 0xFFFFFFFFU;
if (digest != jt->entries[zds->curChunk].checksum) {
if (digest != jt->entries[zds->curFrame].checksum) {
return ERROR(checksum_wrong);
}
@ -323,9 +323,9 @@ size_t ZSTD_seekable_decompressStream(ZSTD_seekable_DStream* zds, ZSTD_outBuffer
/* frame is done */
/* make sure this lines up with the expected frame border */
if (zds->decompressedOffset !=
jt->entries[zds->curChunk + 1].dOffset ||
jt->entries[zds->curFrame + 1].dOffset ||
zds->compressedOffset !=
jt->entries[zds->curChunk + 1].cOffset)
jt->entries[zds->curFrame + 1].cOffset)
return ERROR(corruption_detected);
ZSTD_resetDStream(zds->dstream);
zds->stage = zsds_seek;
@ -334,29 +334,29 @@ size_t ZSTD_seekable_decompressStream(ZSTD_seekable_DStream* zds, ZSTD_outBuffer
/* need more input */
return MIN(ZSTD_DStreamInSize(), (size_t)(
jt->entries[zds->curChunk + 1].cOffset -
jt->entries[zds->curFrame + 1].cOffset -
zds->compressedOffset));
}
}
case zsds_seek: {
U32 targetChunk;
U32 targetFrame;
if (zds->decompressedOffset < zds->targetStart ||
zds->decompressedOffset >= zds->targetEnd) {
/* haven't started yet */
targetChunk = ZSTD_seekable_offsetToChunk(jt, zds->targetStart);
targetFrame = ZSTD_seekable_offsetToFrame(jt, zds->targetStart);
} else {
targetChunk = ZSTD_seekable_offsetToChunk(jt, zds->decompressedOffset);
targetFrame = ZSTD_seekable_offsetToFrame(jt, zds->decompressedOffset);
}
zds->curChunk = targetChunk;
zds->curFrame = targetFrame;
if (zds->compressedOffset == jt->entries[targetChunk].cOffset) {
if (zds->compressedOffset == jt->entries[targetFrame].cOffset) {
zds->stage = zsds_decompress;
break;
}
zds->nextSeek = jt->entries[targetChunk].cOffset;
zds->decompressedOffset = jt->entries[targetChunk].dOffset;
zds->nextSeek = jt->entries[targetFrame].cOffset;
zds->decompressedOffset = jt->entries[targetFrame].dOffset;
/* signal to user that a seek is required */
return ERROR(needSeek);
}

2
lib/common/error_private.c
View File

@ -38,7 +38,7 @@ const char* ERR_getErrorString(ERR_enum code)
case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
case PREFIX(dictionary_wrong): return "Dictionary mismatch";
case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples";
case PREFIX(chunkIndex_tooLarge): return "Chunk index is too large";
case PREFIX(frameIndex_tooLarge): return "Frame index is too large";
case PREFIX(needSeek): return "Wrong file position, a seek is required to continue";
case PREFIX(maxCode):
default: return notErrorCode;

2
lib/common/zstd_errors.h
View File

@ -58,7 +58,7 @@ typedef enum {
ZSTD_error_dictionary_corrupted,
ZSTD_error_dictionary_wrong,
ZSTD_error_dictionaryCreation_failed,
ZSTD_error_chunkIndex_tooLarge,
ZSTD_error_frameIndex_tooLarge,
ZSTD_error_needSeek,
ZSTD_error_maxCode
} ZSTD_ErrorCode;