GRASS GIS 7 Programmer's Manual  7.7.svn(2018)-r73390
 All Data Structures Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
compress.c
Go to the documentation of this file.
1 /*
2  ****************************************************************************
3  * -- GRASS Development Team --
4  *
5  * MODULE: GRASS gis library
6  * FILENAME: compress.c
7  * AUTHOR(S): Markus Metz
8  * PURPOSE: To provide an interface for compressing and
9  * decompressing data using various methods. Its primary
10  * use is in the storage and reading of GRASS rasters.
11  *
12  * DATE CREATED: Dec 17 2015
13  * COPYRIGHT: (C) 2015 by the GRASS Development Team
14  *
15  * This program is free software under the GNU General Public
16  * License (version 2 or greater). Read the file COPYING that
17  * comes with GRASS for details.
18  *
19  *****************************************************************************/
20 
21 /********************************************************************
22  * Compression methods: *
23  * 1 : RLE (generic Run-Length Encoding of single bytes) *
24  * 2 : ZLIB's DEFLATE (good speed and compression) *
25  * 3 : LZ4 (fastest, low compression) *
26  * 4 : BZIP2 (slowest, high compression) *
27  * 5 : ZSTD (faster than ZLIB, higher compression than ZLIB) *
28  * *
29  * int *
30  * G_read_compressed (fd, rbytes, dst, nbytes, compression_type) *
31  * int fd, rbytes, nbytes; *
32  * unsigned char *dst; *
33  * ---------------------------------------------------------------- *
34  * This is the basic function for reading a compressed chunk of a *
35  * data file. The file descriptor should be in the proper location *
36  * and the 'dst' array should have enough space for the data. *
37  * 'nbytes' is the size of 'dst'. The 'rbytes' parameter is the *
38  * number of bytes to read (knowable from the offsets index). For *
39  * best results, 'nbytes' should be the exact amount of space *
40  * needed for the expansion. Too large a value of nbytes may cause *
41  * more data to be expanded than is desired. *
42  * Returns: The number of bytes decompressed into dst, or an error. *
43  * *
44  * Errors include: *
45  * -1 -- Error Reading or Decompressing data. *
46  * -2 -- Not enough space in dst. You must make dst larger *
47  * and then call the function again (remembering to *
48  * reset the file descriptor to it's proper location. *
49  * *
50  * ================================================================ *
51  * int *
52  * G_write_compressed (fd, src, nbytes, compression_type) *
53  * int fd, nbytes; *
54  * unsigned char *src; *
55  * ---------------------------------------------------------------- *
56  * This is the basic function for writing and compressing a data *
57  * chunk to a file. The file descriptor should be in the correct *
58  * location prior to this call. The function will compress 'nbytes' *
59  * of 'src' and write it to the file 'fd'. Returns the number of *
60  * bytes written or an error code: *
61  * *
62  * Errors include: *
63  * -1 -- Compression Failed. *
64  * -2 -- Unable to write to file. *
65  * *
66  * ================================================================ *
67  * int *
68  * G_write_uncompressed (fd, src, nbytes) *
69  * int fd, nbytes; *
70  * unsigned char *src; *
71  * ---------------------------------------------------------------- *
72  * Works similar to G_write_compressed() except no attempt at *
73  * compression is made. This is quicker, but may result in larger *
74  * files. *
75  * Returns the number of bytes written, or -1 for an error. It will *
76  * return an error if it fails to write nbytes. Otherwise, the *
77  * return value will always be nbytes + 1 (for compression flag). *
78  * *
79  ********************************************************************
80  */
81 
82 #include <grass/config.h>
83 
84 #include <stdio.h>
85 #include <stdlib.h>
86 #include <string.h>
87 #include <errno.h>
88 #include <unistd.h>
89 #include <grass/gis.h>
90 #include <grass/glocale.h>
91 
92 #include "compress.h"
93 
94 #define G_COMPRESSED_NO (unsigned char)'0'
95 #define G_COMPRESSED_YES (unsigned char)'1'
96 
97 /* get compressor number
98  * return -1 on error
99  * return number >= 0 for known processor */
101 {
102  int i;
103 
104  if (!name)
105  return -1;
106 
107  for (i = 0; compressor[i].name ; i++) {
108  if (G_strcasecmp(name, compressor[i].name) == 0)
109  return i;
110  }
111 
112  return -1;
113 }
114 
115 /* get compressor name
116  * return NULL on error
117  * return string (name) of known processor */
118 char *G_compressor_name(int number)
119 {
120  if (number < 0 || number >= n_compressors)
121  return NULL;
122 
123  return compressor[number].name;
124 }
125 
127 {
128 #ifdef HAVE_ZSTD_H
129  /* ZSTD */
130  return 5;
131 #endif
132  /* ZLIB */
133  return 2;
134 }
135 
136 /* check compressor number
137  * return -1 on error
138  * return 0 known but not available
139  * return 1 known and available */
140 int G_check_compressor(int number)
141 {
142  if (number < 0 || number >= n_compressors) {
143  G_warning(_("Request for unsupported compressor"));
144  return -1;
145  }
146 
147  return compressor[number].available;
148 }
149 
150 int G_no_compress_bound(int src_sz)
151 {
152  return src_sz;
153 }
154 
155 int
156 G_no_compress(unsigned char *src, int src_sz, unsigned char *dst,
157  int dst_sz)
158 {
159  /* Catch errors early */
160  if (src == NULL || dst == NULL)
161  return -1;
162 
163  /* Don't do anything if src is empty */
164  if (src_sz <= 0)
165  return 0;
166 
167  /* dst too small */
168  if (dst_sz < src_sz)
169  return -2;
170 
171  /* Copy the data from src to dst */
172  memcpy(dst, src, src_sz);
173 
174  return src_sz;
175 }
176 
177 int
178 G_no_expand(unsigned char *src, int src_sz, unsigned char *dst,
179  int dst_sz)
180 {
181  /* Catch errors early */
182  if (src == NULL || dst == NULL)
183  return -1;
184 
185  /* Don't do anything if src is empty */
186  if (src_sz <= 0)
187  return 0;
188 
189  /* dst too small */
190  if (dst_sz < src_sz)
191  return -2;
192 
193  /* Copy the data from src to dst */
194  memcpy(dst, src, src_sz);
195 
196  return src_sz;
197 }
198 
199 /* G_*_compress_bound() returns an upper bound on the compressed size
200  * which can be larger than the input size
201  * some compressors are a bit faster if the size of the destination
202  * is at least the upper bound (no need to test for buffer overlflow)
203  * read comments on the specific compressor interfaces
204  */
205 int G_compress_bound(int src_sz, int number)
206 {
207  if (number < 0 || number >= n_compressors) {
208  G_fatal_error(_("Request for unsupported compressor"));
209  return -1;
210  }
211 
212  return compressor[number].bound(src_sz);
213 }
214 
215 /* G_*_compress() returns
216  * > 0: number of bytes in dst
217  * 0: nothing done
218  * -1: error
219  * -2: dst too small
220  */
221 int G_compress(unsigned char *src, int src_sz, unsigned char *dst,
222  int dst_sz, int number)
223 {
224  if (number < 0 || number >= n_compressors) {
225  G_fatal_error(_("Request for unsupported compressor"));
226  return -1;
227  }
228 
229  return compressor[number].compress(src, src_sz, dst, dst_sz);
230 }
231 
232 /* G_*_expand() returns
233  * > 0: number of bytes in dst
234  * -1: error
235  */
236 int G_expand(unsigned char *src, int src_sz, unsigned char *dst,
237  int dst_sz, int number)
238 {
239  if (number < 0 || number >= n_compressors) {
240  G_fatal_error(_("Request for unsupported compressor"));
241  return -1;
242  }
243 
244  return compressor[number].expand(src, src_sz, dst, dst_sz);
245 }
246 
247 int G_read_compressed(int fd, int rbytes, unsigned char *dst, int nbytes,
248  int number)
249 {
250  int bsize, nread, err;
251  unsigned char *b;
252 
253  if (dst == NULL || nbytes <= 0) {
254  if (dst == NULL)
255  G_warning(_("No destination buffer allocated"));
256  if (nbytes <= 0)
257  G_warning(_("Invalid destination buffer size %d"), nbytes);
258  return -2;
259  }
260 
261  if (rbytes <= 0) {
262  G_warning(_("Invalid read size %d"), nbytes);
263  return -2;
264  }
265 
266  bsize = rbytes;
267 
268  /* Our temporary input buffer for read */
269  if (NULL == (b = (unsigned char *)
270  G_calloc(bsize, sizeof(unsigned char))))
271  return -1;
272 
273  /* Read from the file until we get our bsize or an error */
274  nread = 0;
275  do {
276  err = read(fd, b + nread, bsize - nread);
277  if (err >= 0)
278  nread += err;
279  } while (err > 0 && nread < bsize);
280 
281  if (err <= 0) {
282  if (err == 0)
283  G_warning(_("Unable to read %d bytes: end of file"), rbytes);
284  else
285  G_warning(_("Unable to read %d bytes: %s"), rbytes, strerror(errno));
286  return -1;
287  }
288 
289  /* If the bsize if less than rbytes and we didn't get an error.. */
290  if (nread < rbytes) {
291  G_free(b);
292  G_warning("Unable to read %d bytes, got %d bytes", rbytes, nread);
293  return -1;
294  }
295 
296  /* Test if row is compressed */
297  if (b[0] == G_COMPRESSED_NO) {
298  /* Then just copy it to dst */
299  for (err = 0; err < nread - 1 && err < nbytes; err++)
300  dst[err] = b[err + 1];
301 
302  G_free(b);
303  return (nread - 1);
304  }
305  else if (b[0] != G_COMPRESSED_YES) {
306  /* We're not at the start of a row */
307  G_free(b);
308  G_warning("Read error: We're not at the start of a row");
309  return -1;
310  }
311  /* Okay it's a compressed row */
312 
313  /* Just call G_expand() with the buffer we read,
314  * Account for first byte being a flag
315  */
316  err = G_expand(b + 1, bsize - 1, dst, nbytes, number);
317 
318  /* We're done with b */
319  G_free(b);
320 
321  /* Return whatever G_expand() returned */
322  return err;
323 
324 } /* G_read_compressed() */
325 
326 int G_write_compressed(int fd, unsigned char *src, int nbytes,
327  int number)
328 {
329  int dst_sz, nwritten, err;
330  unsigned char *dst, compressed;
331 
332  /* Catch errors */
333  if (src == NULL || nbytes < 0) {
334  if (src == NULL)
335  G_warning(_("No source buffer"));
336  if (nbytes <= 0)
337  G_warning(_("Invalid source buffer size %d"), nbytes);
338  return -1;
339  }
340 
341  /* get upper bound of compressed size */
342  dst_sz = G_compress_bound(nbytes, number);
343  if (NULL == (dst = (unsigned char *)
344  G_calloc(dst_sz, sizeof(unsigned char))))
345  return -1;
346 
347  /* Now just call G_compress() */
348  err = G_compress(src, nbytes, dst, dst_sz, number);
349 
350  /* If compression succeeded write compressed row,
351  * otherwise write uncompressed row. Compression will fail
352  * if dst is too small (i.e. compressed data is larger)
353  */
354  if (err > 0 && err < nbytes) {
355  dst_sz = err;
356  /* Write the compression flag */
357  compressed = G_COMPRESSED_YES;
358  if (write(fd, &compressed, 1) != 1) {
359  G_free(dst);
360  G_warning(_("Unable to write compression flag"));
361  return -1;
362  }
363  nwritten = 0;
364  do {
365  err = write(fd, dst + nwritten, dst_sz - nwritten);
366  if (err >= 0)
367  nwritten += err;
368  } while (err > 0 && nwritten < dst_sz);
369  if (err <= 0) {
370  if (err == 0)
371  G_warning(_("Unable to write %d bytes: nothing written"), dst_sz);
372  else
373  G_warning(_("Unable to write %d bytes: %s"), dst_sz, strerror(errno));
374  }
375  /* Account for extra byte */
376  nwritten++;
377  }
378  else {
379  /* Write compression flag */
380  compressed = G_COMPRESSED_NO;
381  if (write(fd, &compressed, 1) != 1) {
382  G_free(dst);
383  G_warning(_("Unable to write compression flag"));
384  return -1;
385  }
386  nwritten = 0;
387  do {
388  err = write(fd, src + nwritten, nbytes - nwritten);
389  if (err >= 0)
390  nwritten += err;
391  } while (err > 0 && nwritten < nbytes);
392  if (err <= 0) {
393  if (err == 0)
394  G_warning(_("Unable to write %d bytes: nothing written"), nbytes);
395  else
396  G_warning(_("Unable to write %d bytes: %s"), nbytes, strerror(errno));
397  }
398  /* Account for extra byte */
399  nwritten++;
400  } /* if (err > 0) */
401 
402  /* Done with the dst buffer */
403  G_free(dst);
404 
405  /* If we didn't write all the data return an error */
406  if (err < 0)
407  return -2;
408 
409  return nwritten;
410 } /* G_write_compressed() */
411 
412 int G_write_uncompressed(int fd, const unsigned char *src, int nbytes)
413 {
414  int err, nwritten;
415  unsigned char compressed;
416 
417  /* Catch errors */
418  if (src == NULL || nbytes < 0)
419  return -1;
420 
421  /* Write the compression flag */
422  compressed = G_COMPRESSED_NO;
423  if (write(fd, &compressed, 1) != 1) {
424  G_warning(_("Unable to write compression flag"));
425  return -1;
426  }
427 
428  /* Now write the data */
429  nwritten = 0;
430  do {
431  err = write(fd, src + nwritten, nbytes - nwritten);
432  if (err > 0)
433  nwritten += err;
434  } while (err > 0 && nwritten < nbytes);
435  if (err <= 0) {
436  if (err == 0)
437  G_warning(_("Unable to write %d bytes: nothing written"), nbytes);
438  else
439  G_warning(_("Unable to write %d bytes: %s"), nbytes, strerror(errno));
440  }
441 
442  if (err < 0 || nwritten != nbytes)
443  return -1;
444 
445  /* Account for extra compressed flag */
446  nwritten++;
447 
448  /* That's all */
449  return nwritten;
450 
451 } /* G_write_uncompressed() */
452 
453 
454 /* vim: set softtabstop=4 shiftwidth=4 expandtab: */
int G_default_compressor(void)
Definition: compress.c:126
int G_read_compressed(int fd, int rbytes, unsigned char *dst, int nbytes, int number)
Definition: compress.c:247
int G_strcasecmp(const char *x, const char *y)
String compare ignoring case (upper or lower)
Definition: strings.c:46
void G_free(void *buf)
Free allocated memory.
Definition: gis/alloc.c:149
struct compressor_list compressor[]
Definition: compress.h:54
int G_no_compress_bound(int src_sz)
Definition: compress.c:150
int G_compress(unsigned char *src, int src_sz, unsigned char *dst, int dst_sz, int number)
Definition: compress.c:221
compress_fn * compress
Definition: compress.h:37
char * G_compressor_name(int number)
Definition: compress.c:118
expand_fn * expand
Definition: compress.h:38
int G_expand(unsigned char *src, int src_sz, unsigned char *dst, int dst_sz, int number)
Definition: compress.c:236
char * dst
Definition: lz4.h:599
#define NULL
Definition: ccmath.h:32
fd
Definition: d/range.c:69
void G_fatal_error(const char *msg,...)
Print a fatal error message to stderr.
Definition: gis/error.c:160
SYMBOL * err(FILE *fp, SYMBOL *s, char *msg)
Definition: symbol/read.c:220
int G_no_expand(unsigned char *src, int src_sz, unsigned char *dst, int dst_sz)
Definition: compress.c:178
int G_write_uncompressed(int fd, const unsigned char *src, int nbytes)
Definition: compress.c:412
char * name
Definition: compress.h:40
double b
Definition: r_raster.c:39
#define G_COMPRESSED_NO
Definition: compress.c:94
int G_no_compress(unsigned char *src, int src_sz, unsigned char *dst, int dst_sz)
Definition: compress.c:156
bound_fn * bound
Definition: compress.h:39
#define G_COMPRESSED_YES
Definition: compress.c:95
int G_compress_bound(int src_sz, int number)
Definition: compress.c:205
int G_check_compressor(int number)
Definition: compress.c:140
int G_compressor_number(char *name)
Definition: compress.c:100
int G_write_compressed(int fd, unsigned char *src, int nbytes, int number)
Definition: compress.c:326
#define _(str)
Definition: glocale.h:13
const char * name
Definition: named_colr.c:7
void G_warning(const char *msg,...)
Print a warning message to stderr.
Definition: gis/error.c:204