Blender  V3.3
filereader_zstd.c
Go to the documentation of this file.
1 /* SPDX-License-Identifier: GPL-2.0-or-later
2  * Copyright 2021 Blender Foundation. All rights reserved. */
3 
8 #include <string.h>
9 #include <zstd.h>
10 
11 #include "BLI_blenlib.h"
12 #include "BLI_endian_switch.h"
13 #include "BLI_filereader.h"
14 #include "BLI_math_base.h"
15 
16 #include "MEM_guardedalloc.h"
17 
18 typedef struct {
20 
22 
23  ZSTD_DCtx *ctx;
24  ZSTD_inBuffer in_buf;
26 
27  struct {
29  size_t *compressed_ofs;
31 
34  } seek;
35 } ZstdReader;
36 
37 static bool zstd_read_u32(FileReader *base, uint32_t *val)
38 {
39  if (base->read(base, val, sizeof(uint32_t)) != sizeof(uint32_t)) {
40  return false;
41  }
42 #ifdef __BIG_ENDIAN__
44 #endif
45  return true;
46 }
47 
48 static bool zstd_read_seek_table(ZstdReader *zstd)
49 {
50  FileReader *base = zstd->base;
51 
52  /* The seek table frame is at the end of the file, so seek there
53  * and verify that there is enough data. */
54  if (base->seek(base, -4, SEEK_END) < 13) {
55  return false;
56  }
58  if (!zstd_read_u32(base, &magic) || magic != 0x8F92EAB1) {
59  return false;
60  }
61 
62  uint8_t flags;
63  if (base->seek(base, -5, SEEK_END) < 0 || base->read(base, &flags, 1) != 1) {
64  return false;
65  }
66  /* Bit 7 indicates check-sums. Bits 5 and 6 must be zero. */
67  bool has_checksums = (flags & 0x80);
68  if (flags & 0x60) {
69  return false;
70  }
71 
72  uint32_t frames_num;
73  if (base->seek(base, -9, SEEK_END) < 0 || !zstd_read_u32(base, &frames_num)) {
74  return false;
75  }
76 
77  /* Each frame has either 2 or 3 uint32_t, and after that we have
78  * frames_num, flags and magic for another 9 bytes. */
79  uint32_t expected_frame_length = frames_num * (has_checksums ? 12 : 8) + 9;
80  /* The frame starts with another magic number and its length, but these
81  * two fields are not included when counting length. */
82  off64_t frame_start_ofs = 8 + expected_frame_length;
83  /* Sanity check: Before the start of the seek table frame,
84  * there must be frames_num frames, each of which at least 8 bytes long. */
85  off64_t seek_frame_start = base->seek(base, -frame_start_ofs, SEEK_END);
86  if (seek_frame_start < frames_num * 8) {
87  return false;
88  }
89 
90  if (!zstd_read_u32(base, &magic) || magic != 0x184D2A5E) {
91  return false;
92  }
93 
94  uint32_t frame_length;
95  if (!zstd_read_u32(base, &frame_length) || frame_length != expected_frame_length) {
96  return false;
97  }
98 
99  zstd->seek.frames_num = frames_num;
100  zstd->seek.compressed_ofs = MEM_malloc_arrayN(frames_num + 1, sizeof(size_t), __func__);
101  zstd->seek.uncompressed_ofs = MEM_malloc_arrayN(frames_num + 1, sizeof(size_t), __func__);
102 
103  size_t compressed_ofs = 0;
104  size_t uncompressed_ofs = 0;
105  for (int i = 0; i < frames_num; i++) {
106  uint32_t compressed_size, uncompressed_size;
107  if (!zstd_read_u32(base, &compressed_size) || !zstd_read_u32(base, &uncompressed_size)) {
108  break;
109  }
110  if (has_checksums && base->seek(base, 4, SEEK_CUR) < 0) {
111  break;
112  }
113  zstd->seek.compressed_ofs[i] = compressed_ofs;
114  zstd->seek.uncompressed_ofs[i] = uncompressed_ofs;
115  compressed_ofs += compressed_size;
116  uncompressed_ofs += uncompressed_size;
117  }
118  zstd->seek.compressed_ofs[frames_num] = compressed_ofs;
119  zstd->seek.uncompressed_ofs[frames_num] = uncompressed_ofs;
120 
121  /* Seek to the end of the previous frame for the following #BHead frame detection. */
122  if (seek_frame_start != compressed_ofs || base->seek(base, seek_frame_start, SEEK_SET) < 0) {
125  memset(&zstd->seek, 0, sizeof(zstd->seek));
126  return false;
127  }
128 
129  zstd->seek.cached_frame = -1;
130 
131  return true;
132 }
133 
134 /* Find out which frame contains the given position in the uncompressed stream.
135  * Basically just bisection. */
136 static int zstd_frame_from_pos(ZstdReader *zstd, size_t pos)
137 {
138  int low = 0, high = zstd->seek.frames_num;
139 
140  if (pos >= zstd->seek.uncompressed_ofs[zstd->seek.frames_num]) {
141  return -1;
142  }
143 
144  while (low + 1 < high) {
145  int mid = low + ((high - low) >> 1);
146  if (zstd->seek.uncompressed_ofs[mid] <= pos) {
147  low = mid;
148  }
149  else {
150  high = mid;
151  }
152  }
153 
154  return low;
155 }
156 
157 /* Ensure that the currently loaded frame is the correct one. */
158 static const char *zstd_ensure_cache(ZstdReader *zstd, int frame)
159 {
160  if (zstd->seek.cached_frame == frame) {
161  /* Cached frame matches, so just return it. */
162  return zstd->seek.cached_content;
163  }
164 
165  /* Cached frame doesn't match, so discard it and cache the wanted one instead. */
167 
168  size_t compressed_size = zstd->seek.compressed_ofs[frame + 1] - zstd->seek.compressed_ofs[frame];
169  size_t uncompressed_size = zstd->seek.uncompressed_ofs[frame + 1] -
170  zstd->seek.uncompressed_ofs[frame];
171 
172  char *uncompressed_data = MEM_mallocN(uncompressed_size, __func__);
173  char *compressed_data = MEM_mallocN(compressed_size, __func__);
174  if (zstd->base->seek(zstd->base, zstd->seek.compressed_ofs[frame], SEEK_SET) < 0 ||
175  zstd->base->read(zstd->base, compressed_data, compressed_size) < compressed_size) {
176  MEM_freeN(compressed_data);
177  MEM_freeN(uncompressed_data);
178  return NULL;
179  }
180 
181  size_t res = ZSTD_decompressDCtx(
182  zstd->ctx, uncompressed_data, uncompressed_size, compressed_data, compressed_size);
183  MEM_freeN(compressed_data);
184  if (ZSTD_isError(res) || res < uncompressed_size) {
185  MEM_freeN(uncompressed_data);
186  return NULL;
187  }
188 
189  zstd->seek.cached_frame = frame;
190  zstd->seek.cached_content = uncompressed_data;
191  return uncompressed_data;
192 }
193 
194 static ssize_t zstd_read_seekable(FileReader *reader, void *buffer, size_t size)
195 {
196  ZstdReader *zstd = (ZstdReader *)reader;
197 
198  size_t end_offset = zstd->reader.offset + size, read_len = 0;
199  while (zstd->reader.offset < end_offset) {
200  int frame = zstd_frame_from_pos(zstd, zstd->reader.offset);
201  if (frame < 0) {
202  /* EOF is reached, so return as much as we can. */
203  break;
204  }
205 
206  const char *framedata = zstd_ensure_cache(zstd, frame);
207  if (framedata == NULL) {
208  /* Error while reading the frame, so return as much as we can. */
209  break;
210  }
211 
212  size_t frame_end_offset = min_zz(zstd->seek.uncompressed_ofs[frame + 1], end_offset);
213  size_t frame_read_len = frame_end_offset - zstd->reader.offset;
214 
215  size_t offset_in_frame = zstd->reader.offset - zstd->seek.uncompressed_ofs[frame];
216  memcpy((char *)buffer + read_len, framedata + offset_in_frame, frame_read_len);
217  read_len += frame_read_len;
218  zstd->reader.offset = frame_end_offset;
219  }
220 
221  return read_len;
222 }
223 
224 static off64_t zstd_seek(FileReader *reader, off64_t offset, int whence)
225 {
226  ZstdReader *zstd = (ZstdReader *)reader;
227  off64_t new_pos;
228  if (whence == SEEK_SET) {
229  new_pos = offset;
230  }
231  else if (whence == SEEK_END) {
232  new_pos = zstd->seek.uncompressed_ofs[zstd->seek.frames_num] + offset;
233  }
234  else {
235  new_pos = zstd->reader.offset + offset;
236  }
237 
238  if (new_pos < 0 || new_pos > zstd->seek.uncompressed_ofs[zstd->seek.frames_num]) {
239  return -1;
240  }
241  zstd->reader.offset = new_pos;
242  return zstd->reader.offset;
243 }
244 
245 static ssize_t zstd_read(FileReader *reader, void *buffer, size_t size)
246 {
247  ZstdReader *zstd = (ZstdReader *)reader;
248  ZSTD_outBuffer output = {buffer, size, 0};
249 
250  while (output.pos < output.size) {
251  if (zstd->in_buf.pos == zstd->in_buf.size) {
252  /* Ran out of buffered input data, read some more. */
253  zstd->in_buf.pos = 0;
254  ssize_t readsize = zstd->base->read(
255  zstd->base, (char *)zstd->in_buf.src, zstd->in_buf_max_size);
256 
257  if (readsize > 0) {
258  /* We got some data, so mark the buffer as refilled. */
259  zstd->in_buf.size = readsize;
260  }
261  else {
262  /* The underlying file is EOF, so return as much as we can. */
263  break;
264  }
265  }
266 
267  if (ZSTD_isError(ZSTD_decompressStream(zstd->ctx, &output, &zstd->in_buf))) {
268  break;
269  }
270  }
271 
272  zstd->reader.offset += output.pos;
273  return output.pos;
274 }
275 
276 static void zstd_close(FileReader *reader)
277 {
278  ZstdReader *zstd = (ZstdReader *)reader;
279 
280  ZSTD_freeDCtx(zstd->ctx);
281  if (zstd->reader.seek) {
284  /* When an error has occurred this may be NULL, see: T99744. */
285  if (zstd->seek.cached_content) {
287  }
288  }
289  else {
290  MEM_freeN((void *)zstd->in_buf.src);
291  }
292 
293  zstd->base->close(zstd->base);
294  MEM_freeN(zstd);
295 }
296 
298 {
299  ZstdReader *zstd = MEM_callocN(sizeof(ZstdReader), __func__);
300 
301  zstd->ctx = ZSTD_createDCtx();
302  zstd->base = base;
303 
304  if (zstd_read_seek_table(zstd)) {
306  zstd->reader.seek = zstd_seek;
307  }
308  else {
309  zstd->reader.read = zstd_read;
310  zstd->reader.seek = NULL;
311 
312  zstd->in_buf_max_size = ZSTD_DStreamInSize();
313  zstd->in_buf.src = MEM_mallocN(zstd->in_buf_max_size, "zstd in buf");
314  zstd->in_buf.size = zstd->in_buf_max_size;
315  /* This signals that the buffer has run out,
316  * which will make the read function refill it on the first call. */
317  zstd->in_buf.pos = zstd->in_buf_max_size;
318  }
319  zstd->reader.close = zstd_close;
320 
321  /* Rewind after the seek table check so that zstd_read starts at the file's start. */
322  zstd->base->seek(zstd->base, 0, SEEK_SET);
323 
324  return (FileReader *)zstd;
325 }
BLI_INLINE void BLI_endian_switch_uint32(unsigned int *val) ATTR_NONNULL(1)
Wrapper for reading from various sources (e.g. raw files, compressed files, memory....
MINLINE size_t min_zz(size_t a, size_t b)
SSIZE_T ssize_t
Definition: BLI_winstuff.h:71
Read Guarded memory(de)allocation.
#define MEM_SAFE_FREE(v)
__forceinline ssef low(const avxf &a)
Definition: avxf.h:264
__forceinline ssef high(const avxf &a)
Definition: avxf.h:268
static DBVT_INLINE btScalar size(const btDbvtVolume &a)
Definition: btDbvt.cpp:52
static ssize_t zstd_read(FileReader *reader, void *buffer, size_t size)
static void zstd_close(FileReader *reader)
static off64_t zstd_seek(FileReader *reader, off64_t offset, int whence)
static const char * zstd_ensure_cache(ZstdReader *zstd, int frame)
FileReader * BLI_filereader_new_zstd(FileReader *base)
static int zstd_frame_from_pos(ZstdReader *zstd, size_t pos)
static bool zstd_read_seek_table(ZstdReader *zstd)
static bool zstd_read_u32(FileReader *base, uint32_t *val)
static ssize_t zstd_read_seekable(FileReader *reader, void *buffer, size_t size)
uint pos
ccl_global float * buffer
ccl_global KernelShaderEvalInput ccl_global float * output
ccl_gpu_kernel_postfix ccl_global float int int int int float bool int offset
void *(* MEM_malloc_arrayN)(size_t len, size_t size, const char *str)
Definition: mallocn.c:34
void(* MEM_freeN)(void *vmemh)
Definition: mallocn.c:27
void *(* MEM_callocN)(size_t len, const char *str)
Definition: mallocn.c:31
void *(* MEM_mallocN)(size_t len, const char *str)
Definition: mallocn.c:33
unsigned int uint32_t
Definition: stdint.h:80
unsigned char uint8_t
Definition: stdint.h:78
FileReaderSeekFn seek
off64_t offset
FileReaderCloseFn close
FileReaderReadFn read
ZSTD_DCtx * ctx
FileReader * base
size_t in_buf_max_size
size_t * compressed_ofs
ZSTD_inBuffer in_buf
char * cached_content
struct ZstdReader::@127 seek
FileReader reader
size_t * uncompressed_ofs
static int magic(const Tex *tex, const float texvec[3], TexResult *texres)