GRASS GIS 7 Programmer's Manual  7.5.svn(2017)-r71933
 All Data Structures Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
compress.c
Go to the documentation of this file.
1 /*
2  ****************************************************************************
3  * -- GRASS Development Team --
4  *
5  * MODULE: GRASS gis library
6  * FILENAME: compress.c
7  * AUTHOR(S): Markus Metz
8  * PURPOSE: To provide an interface for compressing and
9  * decompressing data using various methods. Its primary
10  * use is in the storage and reading of GRASS rasters.
11  *
12  * DATE CREATED: Dec 17 2015
13  * COPYRIGHT: (C) 2015 by the GRASS Development Team
14  *
15  * This program is free software under the GNU General Public
16  * License (version 2 or greater). Read the file COPYING that
17  * comes with GRASS for details.
18  *
19  *****************************************************************************/
20 
21 /********************************************************************
22  * Compression methods: *
23  * 1 : RLE (generic Run-Length Encoding of single bytes) *
24  * 2 : ZLIB's DEFLATE (good speed and compression) *
25  * 3 : LZ4 (fastest, low compression) *
26  * 4 : BZIP2 (slowest, high compression) *
27  * 5 : ZSTD (faster than ZLIB, higher compression than ZLIB) *
28  * *
29  * int *
30  * G_read_compressed (fd, rbytes, dst, nbytes, compression_type) *
31  * int fd, rbytes, nbytes; *
32  * unsigned char *dst; *
33  * ---------------------------------------------------------------- *
34  * This is the basic function for reading a compressed chunk of a *
35  * data file. The file descriptor should be in the proper location *
36  * and the 'dst' array should have enough space for the data. *
37  * 'nbytes' is the size of 'dst'. The 'rbytes' parameter is the *
38  * number of bytes to read (knowable from the offsets index). For *
39  * best results, 'nbytes' should be the exact amount of space *
40  * needed for the expansion. Too large a value of nbytes may cause *
41  * more data to be expanded than is desired. *
42  * Returns: The number of bytes decompressed into dst, or an error. *
43  * *
44  * Errors include: *
45  * -1 -- Error Reading or Decompressing data. *
46  * -2 -- Not enough space in dst. You must make dst larger *
47  * and then call the function again (remembering to *
48  * reset the file descriptor to it's proper location. *
49  * *
50  * ================================================================ *
51  * int *
52  * G_write_compressed (fd, src, nbytes, compression_type) *
53  * int fd, nbytes; *
54  * unsigned char *src; *
55  * ---------------------------------------------------------------- *
56  * This is the basic function for writing and compressing a data *
57  * chunk to a file. The file descriptor should be in the correct *
58  * location prior to this call. The function will compress 'nbytes' *
59  * of 'src' and write it to the file 'fd'. Returns the number of *
60  * bytes written or an error code: *
61  * *
62  * Errors include: *
63  * -1 -- Compression Failed. *
64  * -2 -- Unable to write to file. *
65  * *
66  * ================================================================ *
67  * int *
68  * G_write_uncompressed (fd, src, nbytes) *
69  * int fd, nbytes; *
70  * unsigned char *src; *
71  * ---------------------------------------------------------------- *
72  * Works similar to G_write_compressed() except no attempt at *
73  * compression is made. This is quicker, but may result in larger *
74  * files. *
75  * Returns the number of bytes written, or -1 for an error. It will *
76  * return an error if it fails to write nbytes. Otherwise, the *
77  * return value will always be nbytes + 1 (for compression flag). *
78  * *
79  ********************************************************************
80  */
81 
82 #include <grass/config.h>
83 
84 #include <stdio.h>
85 #include <stdlib.h>
86 #include <string.h>
87 #include <unistd.h>
88 #include <grass/gis.h>
89 #include <grass/glocale.h>
90 
91 #include "compress.h"
92 
93 #define G_COMPRESSED_NO (unsigned char)'0'
94 #define G_COMPRESSED_YES (unsigned char)'1'
95 
96 /* get compressor number
97  * return -1 on error
98  * return number >= 0 for known processor */
100 {
101  int i;
102 
103  if (!name)
104  return -1;
105 
106  for (i = 0; compressor[i].name ; i++) {
107  if (G_strcasecmp(name, compressor[i].name) == 0)
108  return i;
109  }
110 
111  return -1;
112 }
113 
114 /* get compressor name
115  * return NULL on error
116  * return string (name) of known processor */
117 char *G_compressor_name(int number)
118 {
119  if (number < 0 || number >= n_compressors)
120  return NULL;
121 
122  return compressor[number].name;
123 }
124 
126 {
127 #ifdef HAVE_ZSTD_H
128  /* ZSTD */
129  return 5;
130 #endif
131  /* ZLIB */
132  return 2;
133 }
134 
135 /* check compressor number
136  * return -1 on error
137  * return 0 known but not available
138  * return 1 known and available */
139 int G_check_compressor(int number)
140 {
141  if (number < 0 || number >= n_compressors) {
142  G_warning(_("Request for unsupported compressor"));
143  return -1;
144  }
145 
146  return compressor[number].available;
147 }
148 
149 int G_no_compress_bound(int src_sz)
150 {
151  return src_sz;
152 }
153 
154 int
155 G_no_compress(unsigned char *src, int src_sz, unsigned char *dst,
156  int dst_sz)
157 {
158  /* Catch errors early */
159  if (src == NULL || dst == NULL)
160  return -1;
161 
162  /* Don't do anything if src is empty */
163  if (src_sz <= 0)
164  return 0;
165 
166  /* dst too small */
167  if (dst_sz < src_sz)
168  return -2;
169 
170  /* Copy the data from src to dst */
171  memcpy(dst, src, src_sz);
172 
173  return src_sz;
174 }
175 
176 int
177 G_no_expand(unsigned char *src, int src_sz, unsigned char *dst,
178  int dst_sz)
179 {
180  /* Catch errors early */
181  if (src == NULL || dst == NULL)
182  return -1;
183 
184  /* Don't do anything if src is empty */
185  if (src_sz <= 0)
186  return 0;
187 
188  /* dst too small */
189  if (dst_sz < src_sz)
190  return -2;
191 
192  /* Copy the data from src to dst */
193  memcpy(dst, src, src_sz);
194 
195  return src_sz;
196 }
197 
198 /* G_*_compress_bound() returns an upper bound on the compressed size
199  * which can be larger than the input size
200  * some compressors are a bit faster if the size of the destination
201  * is at least the upper bound (no need to test for buffer overlflow)
202  * read comments on the specific compressor interfaces
203  */
204 int G_compress_bound(int src_sz, int number)
205 {
206  if (number < 0 || number >= n_compressors) {
207  G_fatal_error(_("Request for unsupported compressor"));
208  return -1;
209  }
210 
211  return compressor[number].bound(src_sz);
212 }
213 
214 /* G_*_compress() returns
215  * > 0: number of bytes in dst
216  * 0: nothing done
217  * -1: error
218  * -2: dst too small
219  */
220 int
221 G_compress(unsigned char *src, int src_sz, unsigned char *dst,
222  int dst_sz, int number)
223 {
224  if (number < 0 || number >= n_compressors) {
225  G_fatal_error(_("Request for unsupported compressor"));
226  return -1;
227  }
228 
229  return compressor[number].compress(src, src_sz, dst, dst_sz);
230 }
231 
232 /* G_*_expand() returns
233  * > 0: number of bytes in dst
234  * -1: error
235  */
236 int
237 G_expand(unsigned char *src, int src_sz, unsigned char *dst,
238  int dst_sz, int number)
239 {
240  if (number < 0 || number >= n_compressors) {
241  G_fatal_error(_("Request for unsupported compressor"));
242  return -1;
243  }
244 
245  return compressor[number].expand(src, src_sz, dst, dst_sz);
246 }
247 
248 int G_read_compressed(int fd, int rbytes, unsigned char *dst, int nbytes,
249  int number)
250 {
251  int bsize, nread, err;
252  unsigned char *b;
253 
254  if (dst == NULL || nbytes < 0)
255  return -2;
256 
257  bsize = rbytes;
258 
259  /* Our temporary input buffer for read */
260  if (NULL == (b = (unsigned char *)
261  G_calloc(bsize, sizeof(unsigned char))))
262  return -1;
263 
264  /* Read from the file until we get our bsize or an error */
265  nread = 0;
266  do {
267  err = read(fd, b + nread, bsize - nread);
268  if (err >= 0)
269  nread += err;
270  } while (err > 0 && nread < bsize);
271 
272  /* If the bsize if less than rbytes and we didn't get an error.. */
273  if (nread < rbytes && err > 0) {
274  G_free(b);
275  return -1;
276  }
277 
278  /* Test if row is compressed */
279  if (b[0] == G_COMPRESSED_NO) {
280  /* Then just copy it to dst */
281  for (err = 0; err < nread - 1 && err < nbytes; err++)
282  dst[err] = b[err + 1];
283 
284  G_free(b);
285  return (nread - 1);
286  }
287  else if (b[0] != G_COMPRESSED_YES) {
288  /* We're not at the start of a row */
289  G_free(b);
290  return -1;
291  }
292  /* Okay it's a compressed row */
293 
294  /* Just call G_expand() with the buffer we read,
295  * Account for first byte being a flag
296  */
297  err = G_expand(b + 1, bsize - 1, dst, nbytes, number);
298 
299  /* We're done with b */
300  G_free(b);
301 
302  /* Return whatever G_expand() returned */
303  return err;
304 
305 } /* G_read_compressed() */
306 
307 int G_write_compressed(int fd, unsigned char *src, int nbytes,
308  int number)
309 {
310  int dst_sz, nwritten, err;
311  unsigned char *dst, compressed;
312 
313  /* Catch errors */
314  if (src == NULL || nbytes < 0)
315  return -1;
316 
317  /* get upper bound of compressed size */
318  dst_sz = G_compress_bound(nbytes, number);
319  if (NULL == (dst = (unsigned char *)
320  G_calloc(dst_sz, sizeof(unsigned char))))
321  return -1;
322 
323  /* Now just call G_compress() */
324  err = G_compress(src, nbytes, dst, dst_sz, number);
325 
326  /* If compression succeeded write compressed row,
327  * otherwise write uncompressed row. Compression will fail
328  * if dst is too small (i.e. compressed data is larger)
329  */
330  if (err > 0 && err < nbytes) {
331  dst_sz = err;
332  /* Write the compression flag */
333  compressed = G_COMPRESSED_YES;
334  if (write(fd, &compressed, 1) != 1) {
335  G_free(dst);
336  return -1;
337  }
338  nwritten = 0;
339  do {
340  err = write(fd, dst + nwritten, dst_sz - nwritten);
341  if (err >= 0)
342  nwritten += err;
343  } while (err > 0 && nwritten < dst_sz);
344  /* Account for extra byte */
345  nwritten++;
346  }
347  else {
348  /* Write compression flag */
349  compressed = G_COMPRESSED_NO;
350  if (write(fd, &compressed, 1) != 1) {
351  G_free(dst);
352  return -1;
353  }
354  nwritten = 0;
355  do {
356  err = write(fd, src + nwritten, nbytes - nwritten);
357  if (err >= 0)
358  nwritten += err;
359  } while (err > 0 && nwritten < nbytes);
360  /* Account for extra byte */
361  nwritten++;
362  } /* if (err > 0) */
363 
364  /* Done with the dst buffer */
365  G_free(dst);
366 
367  /* If we didn't write all the data return an error */
368  if (err < 0)
369  return -2;
370 
371  return nwritten;
372 } /* G_write_compressed() */
373 
374 int G_write_uncompressed(int fd, const unsigned char *src, int nbytes)
375 {
376  int err, nwritten;
377  unsigned char compressed;
378 
379  /* Catch errors */
380  if (src == NULL || nbytes < 0)
381  return -1;
382 
383  /* Write the compression flag */
384  compressed = G_COMPRESSED_NO;
385  if (write(fd, &compressed, 1) != 1)
386  return -1;
387 
388  /* Now write the data */
389  nwritten = 0;
390  do {
391  err = write(fd, src + nwritten, nbytes - nwritten);
392  if (err > 0)
393  nwritten += err;
394  } while (err > 0 && nwritten < nbytes);
395 
396  if (err < 0 || nwritten != nbytes)
397  return -1;
398 
399  /* Account for extra compressed flag */
400  nwritten++;
401 
402  /* That's all */
403  return nwritten;
404 
405 } /* G_write_uncompressed() */
406 
407 
408 /* vim: set softtabstop=4 shiftwidth=4 expandtab: */
int G_default_compressor(void)
Definition: compress.c:125
int G_read_compressed(int fd, int rbytes, unsigned char *dst, int nbytes, int number)
Definition: compress.c:248
int G_strcasecmp(const char *x, const char *y)
String compare ignoring case (upper or lower)
Definition: strings.c:46
void G_free(void *buf)
Free allocated memory.
Definition: gis/alloc.c:149
struct compressor_list compressor[]
Definition: compress.h:54
int G_no_compress_bound(int src_sz)
Definition: compress.c:149
int G_compress(unsigned char *src, int src_sz, unsigned char *dst, int dst_sz, int number)
Definition: compress.c:221
compress_fn * compress
Definition: compress.h:37
char * G_compressor_name(int number)
Definition: compress.c:117
expand_fn * expand
Definition: compress.h:38
int G_expand(unsigned char *src, int src_sz, unsigned char *dst, int dst_sz, int number)
Definition: compress.c:237
char * dst
Definition: lz4.h:354
#define NULL
Definition: ccmath.h:32
fd
Definition: d/range.c:69
void G_fatal_error(const char *msg,...)
Print a fatal error message to stderr.
Definition: gis/error.c:159
SYMBOL * err(FILE *fp, SYMBOL *s, char *msg)
Definition: symbol/read.c:220
int G_no_expand(unsigned char *src, int src_sz, unsigned char *dst, int dst_sz)
Definition: compress.c:177
int G_write_uncompressed(int fd, const unsigned char *src, int nbytes)
Definition: compress.c:374
char * name
Definition: compress.h:40
double b
Definition: r_raster.c:39
#define G_COMPRESSED_NO
Definition: compress.c:93
int G_no_compress(unsigned char *src, int src_sz, unsigned char *dst, int dst_sz)
Definition: compress.c:155
bound_fn * bound
Definition: compress.h:39
#define G_COMPRESSED_YES
Definition: compress.c:94
int G_compress_bound(int src_sz, int number)
Definition: compress.c:204
int G_check_compressor(int number)
Definition: compress.c:139
int G_compressor_number(char *name)
Definition: compress.c:99
int G_write_compressed(int fd, unsigned char *src, int nbytes, int number)
Definition: compress.c:307
#define _(str)
Definition: glocale.h:13
const char * name
Definition: named_colr.c:7
void G_warning(const char *msg,...)
Print a warning message to stderr.
Definition: gis/error.c:203