Add option -d (decompress) to fpga_compress. Allows testing.

Improve zlib deflate (note: no change required to inflate).

diff --git a/client/Makefile b/client/Makefile
index aa1d33c59119ba8c57283bfc9c216b85e5776a1f..bc43408ab3c1ab44c152f5451e3b9983f2566d76 100644 (file)
--- a/client/Makefile
+++ b/client/Makefile
@@ -103,8 +103,8 @@ CMDSRCS =   nonce2key/crapto1.c\
                        aes.c\
                        protocols.c\
 
                        aes.c\
                        protocols.c\
 

-ZLIBSRCS = deflate.c adler32.c trees.c zutil.c
-ZLIB_FLAGS = -DZ_SOLO -DZ_PREFIX -DNO_GZIP -DZLIB_PM3_TUNED
+ZLIBSRCS = deflate.c adler32.c trees.c zutil.c inflate.c inffast.c inftrees.c
+ZLIB_FLAGS = -DZ_SOLO -DZ_PREFIX -DNO_GZIP -DZLIB_PM3_TUNED -DDEBUG -Dverbose=1

 
 
 COREOBJS = $(CORESRCS:%.c=$(OBJDIR)/%.o)
 
 
 COREOBJS = $(CORESRCS:%.c=$(OBJDIR)/%.o)

diff --git a/client/fpga_compress.c b/client/fpga_compress.c
index 5164f6d41bbb6612255d943a0c7cdf5b5cac6ebd..68e9103c9dfab8c0cf590dc3ba0bac74a0934388 100644 (file)
--- a/client/fpga_compress.c
+++ b/client/fpga_compress.c
@@ -45,14 +45,12 @@ static void usage(char *argv0)
 
 static voidpf fpga_deflate_malloc(voidpf opaque, uInt items, uInt size)
 {
 
 static voidpf fpga_deflate_malloc(voidpf opaque, uInt items, uInt size)
 {

-       fprintf(stderr, "zlib requested %d bytes\n", items*size);

        return malloc(items*size);
 }
 
 
 static void fpga_deflate_free(voidpf opaque, voidpf address)
 {
        return malloc(items*size);
 }
 
 
 static void fpga_deflate_free(voidpf opaque, voidpf address)
 {

-       fprintf(stderr, "zlib frees memory\n");

        return free(address);
 }
 
        return free(address);
 }
 


@@ -119,7 +117,6 @@ int zlib_compress(FILE *infile[], uint8_t num_infiles, FILE *outfile)
 
        // estimate the size of the compressed output
        unsigned int outsize_max = deflateBound(&compressed_fpga_stream, compressed_fpga_stream.avail_in);
 
        // estimate the size of the compressed output
        unsigned int outsize_max = deflateBound(&compressed_fpga_stream, compressed_fpga_stream.avail_in);

-       fprintf(stderr, "Allocating %ld bytes for output file (estimated upper bound)\n", outsize_max);

        uint8_t *outbuf = malloc(outsize_max);
        compressed_fpga_stream.next_out = outbuf;
        compressed_fpga_stream.avail_out = outsize_max;
        uint8_t *outbuf = malloc(outsize_max);
        compressed_fpga_stream.next_out = outbuf;
        compressed_fpga_stream.avail_out = outsize_max;


@@ -137,7 +134,7 @@ int zlib_compress(FILE *infile[], uint8_t num_infiles, FILE *outfile)
                ret = deflate(&compressed_fpga_stream, Z_FINISH);
        }
        
                ret = deflate(&compressed_fpga_stream, Z_FINISH);
        }
        

-       fprintf(stderr, "produced %d bytes of output\n", compressed_fpga_stream.total_out);
+       fprintf(stderr, "\ncompressed %d input bytes to %d output bytes\n", i, compressed_fpga_stream.total_out);

 
        if (ret != Z_STREAM_END) {
                fprintf(stderr, "Error in deflate(): %d %s\n", ret, compressed_fpga_stream.msg);
 
        if (ret != Z_STREAM_END) {
                fprintf(stderr, "Error in deflate(): %d %s\n", ret, compressed_fpga_stream.msg);


@@ -170,6 +167,72 @@ int zlib_compress(FILE *infile[], uint8_t num_infiles, FILE *outfile)
 }
 
 
 }
 
 

+int zlib_decompress(FILE *infile, FILE *outfile)
+{
+       #define DECOMPRESS_BUF_SIZE 1024
+       uint8_t outbuf[DECOMPRESS_BUF_SIZE];
+       uint8_t inbuf[DECOMPRESS_BUF_SIZE];
+       int ret;
+       
+       z_stream compressed_fpga_stream;
+       // initialize zlib structures
+       compressed_fpga_stream.next_in = inbuf;
+       compressed_fpga_stream.avail_in = 0;
+       compressed_fpga_stream.next_out = outbuf;
+       compressed_fpga_stream.avail_out = DECOMPRESS_BUF_SIZE;
+       compressed_fpga_stream.zalloc = fpga_deflate_malloc;
+       compressed_fpga_stream.zfree = fpga_deflate_free;
+       
+       ret = inflateInit2(&compressed_fpga_stream, 0);
+       
+       do {
+               if (compressed_fpga_stream.avail_in == 0) {
+                       compressed_fpga_stream.next_in = inbuf;
+                       uint16_t i = 0;
+                       do {
+                               uint8_t c = fgetc(infile);
+                               if (!feof(infile)) {
+                                       inbuf[i++] = c;
+                                       compressed_fpga_stream.avail_in++;
+                               } else {
+                                       break;
+                               }
+                       } while (i < DECOMPRESS_BUF_SIZE);
+               }
+
+               ret = inflate(&compressed_fpga_stream, Z_SYNC_FLUSH);
+
+               if (ret != Z_OK && ret != Z_STREAM_END) {
+                       break;
+               }
+
+               if (compressed_fpga_stream.avail_out == 0) {
+                       for (uint16_t i = 0; i < DECOMPRESS_BUF_SIZE; i++) {
+                               fputc(outbuf[i], outfile);
+                       }
+                       compressed_fpga_stream.avail_out = DECOMPRESS_BUF_SIZE;
+                       compressed_fpga_stream.next_out = outbuf;
+               }
+       } while (ret == Z_OK);
+
+       if (ret == Z_STREAM_END) {  // reached end of input
+               uint16_t i = 0;
+               while (compressed_fpga_stream.avail_out < DECOMPRESS_BUF_SIZE) {
+                       fputc(outbuf[i++], outfile);
+                       compressed_fpga_stream.avail_out++;
+               }
+               fclose(outfile);
+               fclose(infile);
+               return 0;
+       } else {
+               fprintf(stderr, "Error. Inflate() returned error %d, %s", ret, compressed_fpga_stream.msg);
+               fclose(outfile);
+               fclose(infile);
+               return -1;
+       }
+       
+}
+

 
 int main(int argc, char **argv)
 {
 
 int main(int argc, char **argv)
 {


@@ -181,6 +244,26 @@ int main(int argc, char **argv)
                return -1;
        }
        
                return -1;
        }
        

+       if (!strcmp(argv[1], "-d")) {
+               infiles = calloc(1, sizeof(FILE*));
+               if (argc != 4) {
+                       usage(argv[0]);
+                       return -1;
+               } 
+               infiles[0] = fopen(argv[2], "rb");
+               if (infiles[0] == NULL) {
+                       fprintf(stderr, "Error. Cannot open input file %s", argv[2]);
+                       return -1;
+               }
+               outfile = fopen(argv[3], "wb");
+               if (outfile == NULL) {
+                       fprintf(stderr, "Error. Cannot open output file %s", argv[3]);
+                       return -1;
+               }
+               return zlib_decompress(infiles[0], outfile);
+       }
+       
+

        infiles = calloc(argc-2, sizeof(FILE*));
        
        for (uint16_t i = 0; i < argc-2; i++) { 
        infiles = calloc(argc-2, sizeof(FILE*));
        
        for (uint16_t i = 0; i < argc-2; i++) { 


@@ -195,7 +278,7 @@ int main(int argc, char **argv)
        if (outfile == NULL) {
                fprintf(stderr, "Error. Cannot open output file %s", argv[argc-1]);
                return -1;
        if (outfile == NULL) {
                fprintf(stderr, "Error. Cannot open output file %s", argv[argc-1]);
                return -1;

-               }
+       }

                
        return zlib_compress(infiles, argc-2, outfile);
 }
                
        return zlib_compress(infiles, argc-2, outfile);
 }

diff --git a/zlib/deflate.c b/zlib/deflate.c
index 696957705b756b1457a18c7a23a91affafa17d91..7c22a72b4c6ecebc7859ee10e810f8308c3dd36c 100644 (file)
--- a/zlib/deflate.c
+++ b/zlib/deflate.c
@@ -1153,7 +1153,11 @@ local uInt longest_match(s, cur_match)
     register Bytef *scan = s->window + s->strstart; /* current string */
     register Bytef *match;                       /* matched string */
     register int len;                           /* length of current match */
     register Bytef *scan = s->window + s->strstart; /* current string */
     register Bytef *match;                       /* matched string */
     register int len;                           /* length of current match */

+#ifdef ZLIB_PM3_TUNED
+       int best_len =  MIN_MATCH-1;                            // lift the restriction on prev-length
+#else

     int best_len = s->prev_length;              /* best match length so far */
     int best_len = s->prev_length;              /* best match length so far */

+#endif

     int nice_match = s->nice_match;             /* stop if match long enough */
     IPos limit = s->strstart > (IPos)MAX_DIST(s) ?
         s->strstart - (IPos)MAX_DIST(s) : NIL;
     int nice_match = s->nice_match;             /* stop if match long enough */
     IPos limit = s->strstart > (IPos)MAX_DIST(s) ?
         s->strstart - (IPos)MAX_DIST(s) : NIL;


@@ -1721,6 +1725,104 @@ local block_state deflate_fast(s, flush)
     return block_done;
 }
 
     return block_done;
 }
 

+
+#ifdef ZLIB_PM3_TUNED
+local uInt try_harder(s, strstart, lookahead, hash_head, level)
+       deflate_state *s;
+       uInt strstart;
+       uInt lookahead;
+       IPos hash_head;
+       uInt level;
+{
+       uInt strstart_save = s->strstart;
+       s->strstart = strstart;
+       uInt lookahead_save = s->lookahead;
+       s->lookahead = lookahead;
+       uInt ins_h_save = s->ins_h;
+       uInt combined_gain;
+       uInt best_combined_gain = 0;
+       uInt match_length;
+       uInt prev_length = s->prev_length < MIN_MATCH ? 1 : s->prev_length;
+       uInt best_prev_length = prev_length;
+       uInt current_match_start = s->match_start;
+       uInt current_match_length = s->match_length;
+
+       do {
+        if (hash_head != NIL && s->strstart - hash_head <= MAX_DIST(s)) {
+                       match_length = longest_match (s, hash_head);
+                       /* longest_match() sets match_start */
+               } else {
+                       match_length = MIN_MATCH - 1;
+               }
+#if TOO_FAR <= 32767
+               if (match_length == MIN_MATCH && s->strstart - s->match_start > TOO_FAR) {
+                       match_length = MIN_MATCH-1;
+               }
+#endif
+               if (s->strstart == strstart) {                                          // store match at current position
+                       current_match_length = match_length;
+                       current_match_start = s->match_start;
+               }
+               if (s->strstart - strstart + 1 < MIN_MATCH) {           // previous match reduced to one or two literals
+                       combined_gain = 0;                                                              // need one literal per byte: no gain (assuming 8 bits per literal)
+               } else {
+                       combined_gain = s->strstart - strstart + 1 - MIN_MATCH;  // (possibly truncated) previous_length - 3 literals
+               }
+               if (level > 1 && s->strstart+1 <= s->window_size - MIN_LOOKAHEAD) {     // test one level more
+                       s->prev_length = match_length;
+                       uInt save_ins_h = s->ins_h;
+                       UPDATE_HASH(s, s->ins_h, s->window[(s->strstart+1) + (MIN_MATCH-1)]);
+                       combined_gain += try_harder(s, s->strstart+1, s->lookahead-1, s->head[s->ins_h], level-1);
+                       s->ins_h = save_ins_h;
+               } else {
+                       if (match_length < MIN_MATCH) { 
+                               combined_gain += 0;                                                     // no gain
+                       } else {
+                               combined_gain += match_length - MIN_MATCH;      // match_length bytes coded as approx three literals
+                       }
+               }
+               // if (combined_length > s->lookahead - 1) {
+                       // combined_length = s->lookahead;
+               // }
+               if (combined_gain >= best_combined_gain) {                      // in case of a tie we prefer the longer prev_length
+                       best_combined_gain = combined_gain;
+                       best_prev_length = s->strstart - strstart + 1;
+               }
+               s->strstart++;
+               s->lookahead--;
+               UPDATE_HASH(s, s->ins_h, s->window[(s->strstart) + (MIN_MATCH-1)]);
+               hash_head = s->head[s->ins_h];
+               // if (s->strstart - strstart + 1 == MIN_MATCH-1) {             // a match with length == 2 is not possible
+                       // s->strstart++;
+                       // s->lookahead--;
+                       // UPDATE_HASH(s, s->ins_h, s->window[(s->strstart) + (MIN_MATCH-1)]);
+                       // hash_head = s->head[s->ins_h];
+               // }
+       } while (s->strstart <= strstart-1 + prev_length                        // try to truncate the previous match to 1, 3, ... prev_length
+                       && s->strstart <= s->window_size - MIN_LOOKAHEAD);      // watch out for the end of the input
+       
+       s->strstart = strstart_save;
+       s->lookahead = lookahead_save;
+       s->ins_h = ins_h_save;
+       s->match_length = current_match_length;
+       s->match_start = current_match_start;
+       if (prev_length >= MIN_MATCH) {
+               if (best_prev_length != prev_length && best_prev_length >= MIN_MATCH) {
+                       printf("at %d, level %d: Reducing prev_length from %d to %d\n", s->strstart, level, prev_length, best_prev_length);
+               }
+       }
+       if (best_prev_length >= MIN_MATCH) {
+               s->prev_length = best_prev_length;
+               s->match_length = MIN_MATCH - 1;
+       } else {
+               s->prev_length = MIN_MATCH - 1;
+       }
+       return best_combined_gain;
+}
+#endif
+
+
+

 #ifndef FASTEST
 /* ===========================================================================
  * Same as above, but achieves better compression. We use a lazy
 #ifndef FASTEST
 /* ===========================================================================
  * Same as above, but achieves better compression. We use a lazy


@@ -1757,11 +1859,16 @@ local block_state deflate_slow(s, flush)
             INSERT_STRING(s, s->strstart, hash_head);
         }
 
             INSERT_STRING(s, s->strstart, hash_head);
         }
 

-        /* Find the longest match, discarding those <= prev_length.
-         */
+        /* Find the longest match, discarding those <= prev_length. */

         s->prev_length = s->match_length, s->prev_match = s->match_start;
         s->match_length = MIN_MATCH-1;
 
         s->prev_length = s->match_length, s->prev_match = s->match_start;
         s->match_length = MIN_MATCH-1;
 

+#ifdef ZLIB_PM3_TUNED
+               if (s->prev_length < s->max_lazy_match) {
+                       try_harder(s, s->strstart, s->lookahead, hash_head, 1);
+               }
+
+#else

         if (hash_head != NIL && s->prev_length < s->max_lazy_match &&
             s->strstart - hash_head <= MAX_DIST(s)) {
             /* To simplify the code, we prevent matches with the string
         if (hash_head != NIL && s->prev_length < s->max_lazy_match &&
             s->strstart - hash_head <= MAX_DIST(s)) {
             /* To simplify the code, we prevent matches with the string


@@ -1784,6 +1891,7 @@ local block_state deflate_slow(s, flush)
                 s->match_length = MIN_MATCH-1;
             }
         }
                 s->match_length = MIN_MATCH-1;
             }
         }

+#endif /* ZLIB_PM3_TUNED */

         /* If there was a match at the previous step and the current
          * match is not better, output the previous match:
          */
         /* If there was a match at the previous step and the current
          * match is not better, output the previous match:
          */

diff --git a/zlib/inflate.c b/zlib/inflate.c
index d2c666b0345244b70281de85eac7b0733909a66e..417b2c41bf93e434550e65efdadda4c32381f0dc 100644 (file)
--- a/zlib/inflate.c
+++ b/zlib/inflate.c
@@ -847,8 +847,8 @@ int flush;
                 break;
             case 1:                             /* fixed block */
 #ifdef ZLIB_PM3_TUNED
                 break;
             case 1:                             /* fixed block */
 #ifdef ZLIB_PM3_TUNED

-                               Dbprintf("FATAL error. Compressed FPGA files with fixed code blocks are not supported!");
-                               for(;;);
+                strm->msg = (char *)"fixed block coding not supported";
+                               state->mode = BAD;

 #else                          
                 fixedtables(state);
                 Tracev((stderr, "inflate:     fixed codes block%s\n",
 #else                          
                 fixedtables(state);
                 Tracev((stderr, "inflate:     fixed codes block%s\n",

diff --git a/zlib/trees.c b/zlib/trees.c
index a4f04057e7b38aa566c5a528689fcefc556a476e..8c52121124c81cf4570b97ede3e196c0e111960b 100644 (file)
--- a/zlib/trees.c
+++ b/zlib/trees.c
@@ -989,7 +989,9 @@ void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last)
 #ifdef DEBUG
         s->compressed_len += 3 + s->opt_len;
 #endif
 #ifdef DEBUG
         s->compressed_len += 3 + s->opt_len;
 #endif

+#ifndef ZLIB_PM3_TUNED 

     }
     }

+#endif 

     Assert (s->compressed_len == s->bits_sent, "bad compressed size");
     /* The above check is made mod 2^32, for files larger than 512 MB
      * and uLong implemented on 32 bits.
     Assert (s->compressed_len == s->bits_sent, "bad compressed size");
     /* The above check is made mod 2^32, for files larger than 512 MB
      * and uLong implemented on 32 bits.