From 4b3f6d79ea9174d97f3f82177fa9ded4ad485172 Mon Sep 17 00:00:00 2001 From: pwpiwi Date: Mon, 18 May 2015 08:49:38 +0200 Subject: [PATCH] Add option -d (decompress) to fpga_compress. Allows testing. Improve zlib deflate (note: no change required to inflate). --- client/Makefile | 4 +- client/fpga_compress.c | 93 ++++++++++++++++++++++++++++++++-- zlib/deflate.c | 112 ++++++++++++++++++++++++++++++++++++++++- zlib/inflate.c | 4 +- zlib/trees.c | 2 + 5 files changed, 204 insertions(+), 11 deletions(-) diff --git a/client/Makefile b/client/Makefile index aa1d33c5..bc43408a 100644 --- a/client/Makefile +++ b/client/Makefile @@ -103,8 +103,8 @@ CMDSRCS = nonce2key/crapto1.c\ aes.c\ protocols.c\ -ZLIBSRCS = deflate.c adler32.c trees.c zutil.c -ZLIB_FLAGS = -DZ_SOLO -DZ_PREFIX -DNO_GZIP -DZLIB_PM3_TUNED +ZLIBSRCS = deflate.c adler32.c trees.c zutil.c inflate.c inffast.c inftrees.c +ZLIB_FLAGS = -DZ_SOLO -DZ_PREFIX -DNO_GZIP -DZLIB_PM3_TUNED -DDEBUG -Dverbose=1 COREOBJS = $(CORESRCS:%.c=$(OBJDIR)/%.o) diff --git a/client/fpga_compress.c b/client/fpga_compress.c index 5164f6d4..68e9103c 100644 --- a/client/fpga_compress.c +++ b/client/fpga_compress.c @@ -45,14 +45,12 @@ static void usage(char *argv0) static voidpf fpga_deflate_malloc(voidpf opaque, uInt items, uInt size) { - fprintf(stderr, "zlib requested %d bytes\n", items*size); return malloc(items*size); } static void fpga_deflate_free(voidpf opaque, voidpf address) { - fprintf(stderr, "zlib frees memory\n"); return free(address); } @@ -119,7 +117,6 @@ int zlib_compress(FILE *infile[], uint8_t num_infiles, FILE *outfile) // estimate the size of the compressed output unsigned int outsize_max = deflateBound(&compressed_fpga_stream, compressed_fpga_stream.avail_in); - fprintf(stderr, "Allocating %ld bytes for output file (estimated upper bound)\n", outsize_max); uint8_t *outbuf = malloc(outsize_max); compressed_fpga_stream.next_out = outbuf; compressed_fpga_stream.avail_out = outsize_max; @@ -137,7 +134,7 @@ int zlib_compress(FILE *infile[], uint8_t num_infiles, FILE *outfile) ret = deflate(&compressed_fpga_stream, Z_FINISH); } - fprintf(stderr, "produced %d bytes of output\n", compressed_fpga_stream.total_out); + fprintf(stderr, "\ncompressed %d input bytes to %d output bytes\n", i, compressed_fpga_stream.total_out); if (ret != Z_STREAM_END) { fprintf(stderr, "Error in deflate(): %d %s\n", ret, compressed_fpga_stream.msg); @@ -170,6 +167,72 @@ int zlib_compress(FILE *infile[], uint8_t num_infiles, FILE *outfile) } +int zlib_decompress(FILE *infile, FILE *outfile) +{ + #define DECOMPRESS_BUF_SIZE 1024 + uint8_t outbuf[DECOMPRESS_BUF_SIZE]; + uint8_t inbuf[DECOMPRESS_BUF_SIZE]; + int ret; + + z_stream compressed_fpga_stream; + // initialize zlib structures + compressed_fpga_stream.next_in = inbuf; + compressed_fpga_stream.avail_in = 0; + compressed_fpga_stream.next_out = outbuf; + compressed_fpga_stream.avail_out = DECOMPRESS_BUF_SIZE; + compressed_fpga_stream.zalloc = fpga_deflate_malloc; + compressed_fpga_stream.zfree = fpga_deflate_free; + + ret = inflateInit2(&compressed_fpga_stream, 0); + + do { + if (compressed_fpga_stream.avail_in == 0) { + compressed_fpga_stream.next_in = inbuf; + uint16_t i = 0; + do { + uint8_t c = fgetc(infile); + if (!feof(infile)) { + inbuf[i++] = c; + compressed_fpga_stream.avail_in++; + } else { + break; + } + } while (i < DECOMPRESS_BUF_SIZE); + } + + ret = inflate(&compressed_fpga_stream, Z_SYNC_FLUSH); + + if (ret != Z_OK && ret != Z_STREAM_END) { + break; + } + + if (compressed_fpga_stream.avail_out == 0) { + for (uint16_t i = 0; i < DECOMPRESS_BUF_SIZE; i++) { + fputc(outbuf[i], outfile); + } + compressed_fpga_stream.avail_out = DECOMPRESS_BUF_SIZE; + compressed_fpga_stream.next_out = outbuf; + } + } while (ret == Z_OK); + + if (ret == Z_STREAM_END) { // reached end of input + uint16_t i = 0; + while (compressed_fpga_stream.avail_out < DECOMPRESS_BUF_SIZE) { + fputc(outbuf[i++], outfile); + compressed_fpga_stream.avail_out++; + } + fclose(outfile); + fclose(infile); + return 0; + } else { + fprintf(stderr, "Error. Inflate() returned error %d, %s", ret, compressed_fpga_stream.msg); + fclose(outfile); + fclose(infile); + return -1; + } + +} + int main(int argc, char **argv) { @@ -181,6 +244,26 @@ int main(int argc, char **argv) return -1; } + if (!strcmp(argv[1], "-d")) { + infiles = calloc(1, sizeof(FILE*)); + if (argc != 4) { + usage(argv[0]); + return -1; + } + infiles[0] = fopen(argv[2], "rb"); + if (infiles[0] == NULL) { + fprintf(stderr, "Error. Cannot open input file %s", argv[2]); + return -1; + } + outfile = fopen(argv[3], "wb"); + if (outfile == NULL) { + fprintf(stderr, "Error. Cannot open output file %s", argv[3]); + return -1; + } + return zlib_decompress(infiles[0], outfile); + } + + infiles = calloc(argc-2, sizeof(FILE*)); for (uint16_t i = 0; i < argc-2; i++) { @@ -195,7 +278,7 @@ int main(int argc, char **argv) if (outfile == NULL) { fprintf(stderr, "Error. Cannot open output file %s", argv[argc-1]); return -1; - } + } return zlib_compress(infiles, argc-2, outfile); } diff --git a/zlib/deflate.c b/zlib/deflate.c index 69695770..7c22a72b 100644 --- a/zlib/deflate.c +++ b/zlib/deflate.c @@ -1153,7 +1153,11 @@ local uInt longest_match(s, cur_match) register Bytef *scan = s->window + s->strstart; /* current string */ register Bytef *match; /* matched string */ register int len; /* length of current match */ +#ifdef ZLIB_PM3_TUNED + int best_len = MIN_MATCH-1; // lift the restriction on prev-length +#else int best_len = s->prev_length; /* best match length so far */ +#endif int nice_match = s->nice_match; /* stop if match long enough */ IPos limit = s->strstart > (IPos)MAX_DIST(s) ? s->strstart - (IPos)MAX_DIST(s) : NIL; @@ -1721,6 +1725,104 @@ local block_state deflate_fast(s, flush) return block_done; } + +#ifdef ZLIB_PM3_TUNED +local uInt try_harder(s, strstart, lookahead, hash_head, level) + deflate_state *s; + uInt strstart; + uInt lookahead; + IPos hash_head; + uInt level; +{ + uInt strstart_save = s->strstart; + s->strstart = strstart; + uInt lookahead_save = s->lookahead; + s->lookahead = lookahead; + uInt ins_h_save = s->ins_h; + uInt combined_gain; + uInt best_combined_gain = 0; + uInt match_length; + uInt prev_length = s->prev_length < MIN_MATCH ? 1 : s->prev_length; + uInt best_prev_length = prev_length; + uInt current_match_start = s->match_start; + uInt current_match_length = s->match_length; + + do { + if (hash_head != NIL && s->strstart - hash_head <= MAX_DIST(s)) { + match_length = longest_match (s, hash_head); + /* longest_match() sets match_start */ + } else { + match_length = MIN_MATCH - 1; + } +#if TOO_FAR <= 32767 + if (match_length == MIN_MATCH && s->strstart - s->match_start > TOO_FAR) { + match_length = MIN_MATCH-1; + } +#endif + if (s->strstart == strstart) { // store match at current position + current_match_length = match_length; + current_match_start = s->match_start; + } + if (s->strstart - strstart + 1 < MIN_MATCH) { // previous match reduced to one or two literals + combined_gain = 0; // need one literal per byte: no gain (assuming 8 bits per literal) + } else { + combined_gain = s->strstart - strstart + 1 - MIN_MATCH; // (possibly truncated) previous_length - 3 literals + } + if (level > 1 && s->strstart+1 <= s->window_size - MIN_LOOKAHEAD) { // test one level more + s->prev_length = match_length; + uInt save_ins_h = s->ins_h; + UPDATE_HASH(s, s->ins_h, s->window[(s->strstart+1) + (MIN_MATCH-1)]); + combined_gain += try_harder(s, s->strstart+1, s->lookahead-1, s->head[s->ins_h], level-1); + s->ins_h = save_ins_h; + } else { + if (match_length < MIN_MATCH) { + combined_gain += 0; // no gain + } else { + combined_gain += match_length - MIN_MATCH; // match_length bytes coded as approx three literals + } + } + // if (combined_length > s->lookahead - 1) { + // combined_length = s->lookahead; + // } + if (combined_gain >= best_combined_gain) { // in case of a tie we prefer the longer prev_length + best_combined_gain = combined_gain; + best_prev_length = s->strstart - strstart + 1; + } + s->strstart++; + s->lookahead--; + UPDATE_HASH(s, s->ins_h, s->window[(s->strstart) + (MIN_MATCH-1)]); + hash_head = s->head[s->ins_h]; + // if (s->strstart - strstart + 1 == MIN_MATCH-1) { // a match with length == 2 is not possible + // s->strstart++; + // s->lookahead--; + // UPDATE_HASH(s, s->ins_h, s->window[(s->strstart) + (MIN_MATCH-1)]); + // hash_head = s->head[s->ins_h]; + // } + } while (s->strstart <= strstart-1 + prev_length // try to truncate the previous match to 1, 3, ... prev_length + && s->strstart <= s->window_size - MIN_LOOKAHEAD); // watch out for the end of the input + + s->strstart = strstart_save; + s->lookahead = lookahead_save; + s->ins_h = ins_h_save; + s->match_length = current_match_length; + s->match_start = current_match_start; + if (prev_length >= MIN_MATCH) { + if (best_prev_length != prev_length && best_prev_length >= MIN_MATCH) { + printf("at %d, level %d: Reducing prev_length from %d to %d\n", s->strstart, level, prev_length, best_prev_length); + } + } + if (best_prev_length >= MIN_MATCH) { + s->prev_length = best_prev_length; + s->match_length = MIN_MATCH - 1; + } else { + s->prev_length = MIN_MATCH - 1; + } + return best_combined_gain; +} +#endif + + + #ifndef FASTEST /* =========================================================================== * Same as above, but achieves better compression. We use a lazy @@ -1757,11 +1859,16 @@ local block_state deflate_slow(s, flush) INSERT_STRING(s, s->strstart, hash_head); } - /* Find the longest match, discarding those <= prev_length. - */ + /* Find the longest match, discarding those <= prev_length. */ s->prev_length = s->match_length, s->prev_match = s->match_start; s->match_length = MIN_MATCH-1; +#ifdef ZLIB_PM3_TUNED + if (s->prev_length < s->max_lazy_match) { + try_harder(s, s->strstart, s->lookahead, hash_head, 1); + } + +#else if (hash_head != NIL && s->prev_length < s->max_lazy_match && s->strstart - hash_head <= MAX_DIST(s)) { /* To simplify the code, we prevent matches with the string @@ -1784,6 +1891,7 @@ local block_state deflate_slow(s, flush) s->match_length = MIN_MATCH-1; } } +#endif /* ZLIB_PM3_TUNED */ /* If there was a match at the previous step and the current * match is not better, output the previous match: */ diff --git a/zlib/inflate.c b/zlib/inflate.c index d2c666b0..417b2c41 100644 --- a/zlib/inflate.c +++ b/zlib/inflate.c @@ -847,8 +847,8 @@ int flush; break; case 1: /* fixed block */ #ifdef ZLIB_PM3_TUNED - Dbprintf("FATAL error. Compressed FPGA files with fixed code blocks are not supported!"); - for(;;); + strm->msg = (char *)"fixed block coding not supported"; + state->mode = BAD; #else fixedtables(state); Tracev((stderr, "inflate: fixed codes block%s\n", diff --git a/zlib/trees.c b/zlib/trees.c index a4f04057..8c521211 100644 --- a/zlib/trees.c +++ b/zlib/trees.c @@ -989,7 +989,9 @@ void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last) #ifdef DEBUG s->compressed_len += 3 + s->opt_len; #endif +#ifndef ZLIB_PM3_TUNED } +#endif Assert (s->compressed_len == s->bits_sent, "bad compressed size"); /* The above check is made mod 2^32, for files larger than 512 MB * and uLong implemented on 32 bits. -- 2.39.2