1 /* trees.c -- output deflated data using Huffman coding
2 * Copyright (C) 1995-2012 Jean-loup Gailly
3 * detect_data_type() function provided freely by Cosmin Truta, 2006
4 * For conditions of distribution and use, see copyright notice in zlib.h
10 * The "deflation" process uses several Huffman trees. The more
11 * common source values are represented by shorter bit sequences.
13 * Each code tree is stored in a compressed form which is itself
14 * a Huffman encoding of the lengths of all the code strings (in
15 * ascending order by source values). The actual code strings are
16 * reconstructed from the lengths in the inflate process, as described
17 * in the deflate specification.
21 * Deutsch, L.P.,"'Deflate' Compressed Data Format Specification".
22 * Available in ftp.uu.net:/pub/archiving/zip/doc/deflate-1.1.doc
25 * Data Compression: Methods and Theory, pp. 49-50.
26 * Computer Science Press, 1988. ISBN 0-7167-8156-5.
30 * Addison-Wesley, 1983. ISBN 0-201-06672-6.
35 //-----------------------------------------------------------------------------
36 // This version of zlib is modified for use within the Proxmark3 project.
37 // Files from the original distribution which are not required for this
38 // purpose are not included. All modifications can easily be found
39 // by searching for #ifdef ZLIB_PM3_TUNED and #ifndef ZLIB_PM3_TUNED.
40 //-----------------------------------------------------------------------------
42 /* #define GEN_TREES_H */
51 /* ===========================================================================
56 /* Bit length codes must not exceed MAX_BL_BITS bits */
59 /* end of block literal code */
62 /* repeat previous bit length 3-6 times (2 bits of repeat count) */
65 /* repeat a zero length 3-10 times (3 bits of repeat count) */
67 #define REPZ_11_138 18
68 /* repeat a zero length 11-138 times (7 bits of repeat count) */
70 local
const int extra_lbits
[LENGTH_CODES
] /* extra bits for each length code */
71 = {0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0};
73 local
const int extra_dbits
[D_CODES
] /* extra bits for each distance code */
74 = {0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13};
76 local
const int extra_blbits
[BL_CODES
]/* extra bits for each bit length code */
77 = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,3,7};
79 local
const uch bl_order
[BL_CODES
]
80 = {16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15};
81 /* The lengths of the bit length codes are sent in order of decreasing
82 * probability, to avoid transmitting the lengths for unused bit length codes.
85 /* ===========================================================================
86 * Local data. These are initialized only once.
89 #define DIST_CODE_LEN 512 /* see definition of array dist_code below */
91 #if defined(GEN_TREES_H) || !defined(STDC)
92 /* non ANSI compilers may not accept trees.h */
94 local ct_data static_ltree
[L_CODES
+2];
95 /* The static literal tree. Since the bit lengths are imposed, there is no
96 * need for the L_CODES extra codes used during heap construction. However
97 * The codes 286 and 287 are needed to build a canonical tree (see _tr_init
101 local ct_data static_dtree
[D_CODES
];
102 /* The static distance tree. (Actually a trivial tree since all codes use
106 uch _dist_code
[DIST_CODE_LEN
];
107 /* Distance codes. The first 256 values correspond to the distances
108 * 3 .. 258, the last 256 values correspond to the top 8 bits of
109 * the 15 bit distances.
112 uch _length_code
[MAX_MATCH
-MIN_MATCH
+1];
113 /* length code for each normalized match length (0 == MIN_MATCH) */
115 local
int base_length
[LENGTH_CODES
];
116 /* First normalized length for each code (0 = MIN_MATCH) */
118 local
int base_dist
[D_CODES
];
119 /* First normalized distance for each code (0 = distance of 1) */
123 #endif /* GEN_TREES_H */
125 struct static_tree_desc_s
{
126 const ct_data
*static_tree
; /* static tree or NULL */
127 const intf
*extra_bits
; /* extra bits for each code or NULL */
128 int extra_base
; /* base index for extra_bits */
129 int elems
; /* max number of elements in the tree */
130 int max_length
; /* max bit length for the codes */
133 local static_tree_desc static_l_desc
=
134 {static_ltree
, extra_lbits
, LITERALS
+1, L_CODES
, MAX_BITS
};
136 local static_tree_desc static_d_desc
=
137 {static_dtree
, extra_dbits
, 0, D_CODES
, MAX_BITS
};
139 local static_tree_desc static_bl_desc
=
140 {(const ct_data
*)0, extra_blbits
, 0, BL_CODES
, MAX_BL_BITS
};
142 /* ===========================================================================
143 * Local (static) routines in this file.
146 local
void tr_static_init
OF((void));
147 local
void init_block
OF((deflate_state
*s
));
148 local
void pqdownheap
OF((deflate_state
*s
, ct_data
*tree
, int k
));
149 local
void gen_bitlen
OF((deflate_state
*s
, tree_desc
*desc
));
150 local
void gen_codes
OF((ct_data
*tree
, int max_code
, ushf
*bl_count
));
151 local
void build_tree
OF((deflate_state
*s
, tree_desc
*desc
));
152 local
void scan_tree
OF((deflate_state
*s
, ct_data
*tree
, int max_code
));
153 local
void send_tree
OF((deflate_state
*s
, ct_data
*tree
, int max_code
));
154 local
int build_bl_tree
OF((deflate_state
*s
));
155 local
void send_all_trees
OF((deflate_state
*s
, int lcodes
, int dcodes
,
157 local
void compress_block
OF((deflate_state
*s
, const ct_data
*ltree
,
158 const ct_data
*dtree
));
159 local
int detect_data_type
OF((deflate_state
*s
));
160 local
unsigned bi_reverse
OF((unsigned value
, int length
));
161 local
void bi_windup
OF((deflate_state
*s
));
162 local
void bi_flush
OF((deflate_state
*s
));
163 local
void copy_block
OF((deflate_state
*s
, charf
*buf
, unsigned len
,
167 local
void gen_trees_header
OF((void));
171 # define send_code(s, c, tree) send_bits(s, tree[c].Code, tree[c].Len)
172 /* Send a code of the given tree. c and tree must not have side effects */
175 # define send_code(s, c, tree) \
176 { if (z_verbose>2) fprintf(stderr,"\ncd %3d ",(c)); \
177 send_bits(s, tree[c].Code, tree[c].Len); }
180 /* ===========================================================================
181 * Output a short LSB first on the stream.
182 * IN assertion: there is enough room in pendingBuf.
184 #define put_short(s, w) { \
185 put_byte(s, (uch)((w) & 0xff)); \
186 put_byte(s, (uch)((ush)(w) >> 8)); \
189 /* ===========================================================================
190 * Send a value on a given number of bits.
191 * IN assertion: length <= 16 and value fits in length bits.
194 local
void send_bits
OF((deflate_state
*s
, int value
, int length
));
196 local
void send_bits(s
, value
, length
)
198 int value
; /* value to send */
199 int length
; /* number of bits */
201 Tracevv((stderr
," l %2d v %4x ", length
, value
));
202 Assert(length
> 0 && length
<= 15, "invalid length");
203 s
->bits_sent
+= (ulg
)length
;
205 /* If not enough room in bi_buf, use (valid) bits from bi_buf and
206 * (16 - bi_valid) bits from value, leaving (width - (16-bi_valid))
207 * unused bits in value.
209 if (s
->bi_valid
> (int)Buf_size
- length
) {
210 s
->bi_buf
|= (ush
)value
<< s
->bi_valid
;
211 put_short(s
, s
->bi_buf
);
212 s
->bi_buf
= (ush
)value
>> (Buf_size
- s
->bi_valid
);
213 s
->bi_valid
+= length
- Buf_size
;
215 s
->bi_buf
|= (ush
)value
<< s
->bi_valid
;
216 s
->bi_valid
+= length
;
221 #define send_bits(s, value, length) \
223 if (s->bi_valid > (int)Buf_size - len) {\
225 s->bi_buf |= (ush)val << s->bi_valid;\
226 put_short(s, s->bi_buf);\
227 s->bi_buf = (ush)val >> (Buf_size - s->bi_valid);\
228 s->bi_valid += len - Buf_size;\
230 s->bi_buf |= (ush)(value) << s->bi_valid;\
237 /* the arguments must not have side effects */
239 /* ===========================================================================
240 * Initialize the various 'constant' tables.
242 local
void tr_static_init()
244 #if defined(GEN_TREES_H) || !defined(STDC)
245 static int static_init_done
= 0;
246 int n
; /* iterates over tree elements */
247 int bits
; /* bit counter */
248 int length
; /* length value */
249 int code
; /* code value */
250 int dist
; /* distance index */
251 ush bl_count
[MAX_BITS
+1];
252 /* number of codes at each bit length for an optimal tree */
254 if (static_init_done
) return;
256 /* For some embedded targets, global variables are not initialized: */
257 #ifdef NO_INIT_GLOBAL_POINTERS
258 static_l_desc
.static_tree
= static_ltree
;
259 static_l_desc
.extra_bits
= extra_lbits
;
260 static_d_desc
.static_tree
= static_dtree
;
261 static_d_desc
.extra_bits
= extra_dbits
;
262 static_bl_desc
.extra_bits
= extra_blbits
;
265 /* Initialize the mapping length (0..255) -> length code (0..28) */
267 for (code
= 0; code
< LENGTH_CODES
-1; code
++) {
268 base_length
[code
] = length
;
269 for (n
= 0; n
< (1<<extra_lbits
[code
]); n
++) {
270 _length_code
[length
++] = (uch
)code
;
273 Assert (length
== 256, "tr_static_init: length != 256");
274 /* Note that the length 255 (match length 258) can be represented
275 * in two different ways: code 284 + 5 bits or code 285, so we
276 * overwrite length_code[255] to use the best encoding:
278 _length_code
[length
-1] = (uch
)code
;
280 /* Initialize the mapping dist (0..32K) -> dist code (0..29) */
282 for (code
= 0 ; code
< 16; code
++) {
283 base_dist
[code
] = dist
;
284 for (n
= 0; n
< (1<<extra_dbits
[code
]); n
++) {
285 _dist_code
[dist
++] = (uch
)code
;
288 Assert (dist
== 256, "tr_static_init: dist != 256");
289 dist
>>= 7; /* from now on, all distances are divided by 128 */
290 for ( ; code
< D_CODES
; code
++) {
291 base_dist
[code
] = dist
<< 7;
292 for (n
= 0; n
< (1<<(extra_dbits
[code
]-7)); n
++) {
293 _dist_code
[256 + dist
++] = (uch
)code
;
296 Assert (dist
== 256, "tr_static_init: 256+dist != 512");
298 /* Construct the codes of the static literal tree */
299 for (bits
= 0; bits
<= MAX_BITS
; bits
++) bl_count
[bits
] = 0;
301 while (n
<= 143) static_ltree
[n
++].Len
= 8, bl_count
[8]++;
302 while (n
<= 255) static_ltree
[n
++].Len
= 9, bl_count
[9]++;
303 while (n
<= 279) static_ltree
[n
++].Len
= 7, bl_count
[7]++;
304 while (n
<= 287) static_ltree
[n
++].Len
= 8, bl_count
[8]++;
305 /* Codes 286 and 287 do not exist, but we must include them in the
306 * tree construction to get a canonical Huffman tree (longest code
309 gen_codes((ct_data
*)static_ltree
, L_CODES
+1, bl_count
);
311 /* The static distance tree is trivial: */
312 for (n
= 0; n
< D_CODES
; n
++) {
313 static_dtree
[n
].Len
= 5;
314 static_dtree
[n
].Code
= bi_reverse((unsigned)n
, 5);
316 static_init_done
= 1;
321 #endif /* defined(GEN_TREES_H) || !defined(STDC) */
324 /* ===========================================================================
325 * Genererate the file trees.h describing the static trees.
332 # define SEPARATOR(i, last, width) \
333 ((i) == (last)? "\n};\n\n" : \
334 ((i) % (width) == (width)-1 ? ",\n" : ", "))
336 void gen_trees_header()
338 FILE *header
= fopen("trees.h", "w");
341 Assert (header
!= NULL
, "Can't open trees.h");
343 "/* header created automatically with -DGEN_TREES_H */\n\n");
345 fprintf(header
, "local const ct_data static_ltree[L_CODES+2] = {\n");
346 for (i
= 0; i
< L_CODES
+2; i
++) {
347 fprintf(header
, "{{%3u},{%3u}}%s", static_ltree
[i
].Code
,
348 static_ltree
[i
].Len
, SEPARATOR(i
, L_CODES
+1, 5));
351 fprintf(header
, "local const ct_data static_dtree[D_CODES] = {\n");
352 for (i
= 0; i
< D_CODES
; i
++) {
353 fprintf(header
, "{{%2u},{%2u}}%s", static_dtree
[i
].Code
,
354 static_dtree
[i
].Len
, SEPARATOR(i
, D_CODES
-1, 5));
357 fprintf(header
, "const uch ZLIB_INTERNAL _dist_code[DIST_CODE_LEN] = {\n");
358 for (i
= 0; i
< DIST_CODE_LEN
; i
++) {
359 fprintf(header
, "%2u%s", _dist_code
[i
],
360 SEPARATOR(i
, DIST_CODE_LEN
-1, 20));
364 "const uch ZLIB_INTERNAL _length_code[MAX_MATCH-MIN_MATCH+1]= {\n");
365 for (i
= 0; i
< MAX_MATCH
-MIN_MATCH
+1; i
++) {
366 fprintf(header
, "%2u%s", _length_code
[i
],
367 SEPARATOR(i
, MAX_MATCH
-MIN_MATCH
, 20));
370 fprintf(header
, "local const int base_length[LENGTH_CODES] = {\n");
371 for (i
= 0; i
< LENGTH_CODES
; i
++) {
372 fprintf(header
, "%1u%s", base_length
[i
],
373 SEPARATOR(i
, LENGTH_CODES
-1, 20));
376 fprintf(header
, "local const int base_dist[D_CODES] = {\n");
377 for (i
= 0; i
< D_CODES
; i
++) {
378 fprintf(header
, "%5u%s", base_dist
[i
],
379 SEPARATOR(i
, D_CODES
-1, 10));
384 #endif /* GEN_TREES_H */
386 /* ===========================================================================
387 * Initialize the tree data structures for a new zlib stream.
389 void ZLIB_INTERNAL
_tr_init(s
)
394 s
->l_desc
.dyn_tree
= s
->dyn_ltree
;
395 s
->l_desc
.stat_desc
= &static_l_desc
;
397 s
->d_desc
.dyn_tree
= s
->dyn_dtree
;
398 s
->d_desc
.stat_desc
= &static_d_desc
;
400 s
->bl_desc
.dyn_tree
= s
->bl_tree
;
401 s
->bl_desc
.stat_desc
= &static_bl_desc
;
406 s
->compressed_len
= 0L;
410 /* Initialize the first block of the first file: */
414 /* ===========================================================================
415 * Initialize a new block.
417 local
void init_block(s
)
420 int n
; /* iterates over tree elements */
422 /* Initialize the trees. */
423 for (n
= 0; n
< L_CODES
; n
++) s
->dyn_ltree
[n
].Freq
= 0;
424 for (n
= 0; n
< D_CODES
; n
++) s
->dyn_dtree
[n
].Freq
= 0;
425 for (n
= 0; n
< BL_CODES
; n
++) s
->bl_tree
[n
].Freq
= 0;
427 s
->dyn_ltree
[END_BLOCK
].Freq
= 1;
428 s
->opt_len
= s
->static_len
= 0L;
429 s
->last_lit
= s
->matches
= 0;
433 /* Index within the heap array of least frequent node in the Huffman tree */
436 /* ===========================================================================
437 * Remove the smallest element from the heap and recreate the heap with
438 * one less element. Updates heap and heap_len.
440 #define pqremove(s, tree, top) \
442 top = s->heap[SMALLEST]; \
443 s->heap[SMALLEST] = s->heap[s->heap_len--]; \
444 pqdownheap(s, tree, SMALLEST); \
447 /* ===========================================================================
448 * Compares to subtrees, using the tree depth as tie breaker when
449 * the subtrees have equal frequency. This minimizes the worst case length.
451 #define smaller(tree, n, m, depth) \
452 (tree[n].Freq < tree[m].Freq || \
453 (tree[n].Freq == tree[m].Freq && depth[n] <= depth[m]))
455 /* ===========================================================================
456 * Restore the heap property by moving down the tree starting at node k,
457 * exchanging a node with the smallest of its two sons if necessary, stopping
458 * when the heap property is re-established (each father smaller than its
461 local
void pqdownheap(s
, tree
, k
)
463 ct_data
*tree
; /* the tree to restore */
464 int k
; /* node to move down */
467 int j
= k
<< 1; /* left son of k */
468 while (j
<= s
->heap_len
) {
469 /* Set j to the smallest of the two sons: */
470 if (j
< s
->heap_len
&&
471 smaller(tree
, s
->heap
[j
+1], s
->heap
[j
], s
->depth
)) {
474 /* Exit if v is smaller than both sons */
475 if (smaller(tree
, v
, s
->heap
[j
], s
->depth
)) break;
477 /* Exchange v with the smallest son */
478 s
->heap
[k
] = s
->heap
[j
]; k
= j
;
480 /* And continue down the tree, setting j to the left son of k */
486 /* ===========================================================================
487 * Compute the optimal bit lengths for a tree and update the total bit length
488 * for the current block.
489 * IN assertion: the fields freq and dad are set, heap[heap_max] and
490 * above are the tree nodes sorted by increasing frequency.
491 * OUT assertions: the field len is set to the optimal bit length, the
492 * array bl_count contains the frequencies for each bit length.
493 * The length opt_len is updated; static_len is also updated if stree is
496 local
void gen_bitlen(s
, desc
)
498 tree_desc
*desc
; /* the tree descriptor */
500 ct_data
*tree
= desc
->dyn_tree
;
501 int max_code
= desc
->max_code
;
502 const ct_data
*stree
= desc
->stat_desc
->static_tree
;
503 const intf
*extra
= desc
->stat_desc
->extra_bits
;
504 int base
= desc
->stat_desc
->extra_base
;
505 int max_length
= desc
->stat_desc
->max_length
;
506 int h
; /* heap index */
507 int n
, m
; /* iterate over the tree elements */
508 int bits
; /* bit length */
509 int xbits
; /* extra bits */
510 ush f
; /* frequency */
511 int overflow
= 0; /* number of elements with bit length too large */
513 for (bits
= 0; bits
<= MAX_BITS
; bits
++) s
->bl_count
[bits
] = 0;
515 /* In a first pass, compute the optimal bit lengths (which may
516 * overflow in the case of the bit length tree).
518 tree
[s
->heap
[s
->heap_max
]].Len
= 0; /* root of the heap */
520 for (h
= s
->heap_max
+1; h
< HEAP_SIZE
; h
++) {
522 bits
= tree
[tree
[n
].Dad
].Len
+ 1;
523 if (bits
> max_length
) bits
= max_length
, overflow
++;
524 tree
[n
].Len
= (ush
)bits
;
525 /* We overwrite tree[n].Dad which is no longer needed */
527 if (n
> max_code
) continue; /* not a leaf node */
531 if (n
>= base
) xbits
= extra
[n
-base
];
533 s
->opt_len
+= (ulg
)f
* (bits
+ xbits
);
534 if (stree
) s
->static_len
+= (ulg
)f
* (stree
[n
].Len
+ xbits
);
536 if (overflow
== 0) return;
538 Trace((stderr
,"\nbit length overflow\n"));
539 /* This happens for example on obj2 and pic of the Calgary corpus */
541 /* Find the first bit length which could increase: */
544 while (s
->bl_count
[bits
] == 0) bits
--;
545 s
->bl_count
[bits
]--; /* move one leaf down the tree */
546 s
->bl_count
[bits
+1] += 2; /* move one overflow item as its brother */
547 s
->bl_count
[max_length
]--;
548 /* The brother of the overflow item also moves one step up,
549 * but this does not affect bl_count[max_length]
552 } while (overflow
> 0);
554 /* Now recompute all bit lengths, scanning in increasing frequency.
555 * h is still equal to HEAP_SIZE. (It is simpler to reconstruct all
556 * lengths instead of fixing only the wrong ones. This idea is taken
557 * from 'ar' written by Haruhiko Okumura.)
559 for (bits
= max_length
; bits
!= 0; bits
--) {
560 n
= s
->bl_count
[bits
];
563 if (m
> max_code
) continue;
564 if ((unsigned) tree
[m
].Len
!= (unsigned) bits
) {
565 Trace((stderr
,"code %d bits %d->%d\n", m
, tree
[m
].Len
, bits
));
566 s
->opt_len
+= ((long)bits
- (long)tree
[m
].Len
)
568 tree
[m
].Len
= (ush
)bits
;
575 /* ===========================================================================
576 * Generate the codes for a given tree and bit counts (which need not be
578 * IN assertion: the array bl_count contains the bit length statistics for
579 * the given tree and the field len is set for all tree elements.
580 * OUT assertion: the field code is set for all tree elements of non
583 local
void gen_codes (tree
, max_code
, bl_count
)
584 ct_data
*tree
; /* the tree to decorate */
585 int max_code
; /* largest code with non zero frequency */
586 ushf
*bl_count
; /* number of codes at each bit length */
588 ush next_code
[MAX_BITS
+1]; /* next code value for each bit length */
589 ush code
= 0; /* running code value */
590 int bits
; /* bit index */
591 int n
; /* code index */
593 /* The distribution counts are first used to generate the code values
594 * without bit reversal.
596 for (bits
= 1; bits
<= MAX_BITS
; bits
++) {
597 next_code
[bits
] = code
= (code
+ bl_count
[bits
-1]) << 1;
599 /* Check that the bit counts in bl_count are consistent. The last code
602 Assert (code
+ bl_count
[MAX_BITS
]-1 == (1<<MAX_BITS
)-1,
603 "inconsistent bit counts");
604 Tracev((stderr
,"\ngen_codes: max_code %d ", max_code
));
606 for (n
= 0; n
<= max_code
; n
++) {
607 int len
= tree
[n
].Len
;
608 if (len
== 0) continue;
609 /* Now reverse the bits */
610 tree
[n
].Code
= bi_reverse(next_code
[len
]++, len
);
612 Tracecv(tree
!= static_ltree
, (stderr
,"\nn %3d %c l %2d c %4x (%x) ",
613 n
, (isgraph(n
) ? n
: ' '), len
, tree
[n
].Code
, next_code
[len
]-1));
617 /* ===========================================================================
618 * Construct one Huffman tree and assigns the code bit strings and lengths.
619 * Update the total bit length for the current block.
620 * IN assertion: the field freq is set for all tree elements.
621 * OUT assertions: the fields len and code are set to the optimal bit length
622 * and corresponding code. The length opt_len is updated; static_len is
623 * also updated if stree is not null. The field max_code is set.
625 local
void build_tree(s
, desc
)
627 tree_desc
*desc
; /* the tree descriptor */
629 ct_data
*tree
= desc
->dyn_tree
;
630 const ct_data
*stree
= desc
->stat_desc
->static_tree
;
631 int elems
= desc
->stat_desc
->elems
;
632 int n
, m
; /* iterate over heap elements */
633 int max_code
= -1; /* largest code with non zero frequency */
634 int node
; /* new node being created */
636 /* Construct the initial heap, with least frequent element in
637 * heap[SMALLEST]. The sons of heap[n] are heap[2*n] and heap[2*n+1].
638 * heap[0] is not used.
640 s
->heap_len
= 0, s
->heap_max
= HEAP_SIZE
;
642 for (n
= 0; n
< elems
; n
++) {
643 if (tree
[n
].Freq
!= 0) {
644 s
->heap
[++(s
->heap_len
)] = max_code
= n
;
651 /* The pkzip format requires that at least one distance code exists,
652 * and that at least one bit should be sent even if there is only one
653 * possible code. So to avoid special checks later on we force at least
654 * two codes of non zero frequency.
656 while (s
->heap_len
< 2) {
657 node
= s
->heap
[++(s
->heap_len
)] = (max_code
< 2 ? ++max_code
: 0);
660 s
->opt_len
--; if (stree
) s
->static_len
-= stree
[node
].Len
;
661 /* node is 0 or 1 so it does not have extra bits */
663 desc
->max_code
= max_code
;
665 /* The elements heap[heap_len/2+1 .. heap_len] are leaves of the tree,
666 * establish sub-heaps of increasing lengths:
668 for (n
= s
->heap_len
/2; n
>= 1; n
--) pqdownheap(s
, tree
, n
);
670 /* Construct the Huffman tree by repeatedly combining the least two
673 node
= elems
; /* next internal node of the tree */
675 pqremove(s
, tree
, n
); /* n = node of least frequency */
676 m
= s
->heap
[SMALLEST
]; /* m = node of next least frequency */
678 s
->heap
[--(s
->heap_max
)] = n
; /* keep the nodes sorted by frequency */
679 s
->heap
[--(s
->heap_max
)] = m
;
681 /* Create a new node father of n and m */
682 tree
[node
].Freq
= tree
[n
].Freq
+ tree
[m
].Freq
;
683 s
->depth
[node
] = (uch
)((s
->depth
[n
] >= s
->depth
[m
] ?
684 s
->depth
[n
] : s
->depth
[m
]) + 1);
685 tree
[n
].Dad
= tree
[m
].Dad
= (ush
)node
;
687 if (tree
== s
->bl_tree
) {
688 fprintf(stderr
,"\nnode %d(%d), sons %d(%d) %d(%d)",
689 node
, tree
[node
].Freq
, n
, tree
[n
].Freq
, m
, tree
[m
].Freq
);
692 /* and insert the new node in the heap */
693 s
->heap
[SMALLEST
] = node
++;
694 pqdownheap(s
, tree
, SMALLEST
);
696 } while (s
->heap_len
>= 2);
698 s
->heap
[--(s
->heap_max
)] = s
->heap
[SMALLEST
];
700 /* At this point, the fields freq and dad are set. We can now
701 * generate the bit lengths.
703 gen_bitlen(s
, (tree_desc
*)desc
);
705 /* The field len is now set, we can generate the bit codes */
706 gen_codes ((ct_data
*)tree
, max_code
, s
->bl_count
);
709 /* ===========================================================================
710 * Scan a literal or distance tree to determine the frequencies of the codes
711 * in the bit length tree.
713 local
void scan_tree (s
, tree
, max_code
)
715 ct_data
*tree
; /* the tree to be scanned */
716 int max_code
; /* and its largest code of non zero frequency */
718 int n
; /* iterates over all tree elements */
719 int prevlen
= -1; /* last emitted length */
720 int curlen
; /* length of current code */
721 int nextlen
= tree
[0].Len
; /* length of next code */
722 int count
= 0; /* repeat count of the current code */
723 int max_count
= 7; /* max repeat count */
724 int min_count
= 4; /* min repeat count */
726 if (nextlen
== 0) max_count
= 138, min_count
= 3;
727 tree
[max_code
+1].Len
= (ush
)0xffff; /* guard */
729 for (n
= 0; n
<= max_code
; n
++) {
730 curlen
= nextlen
; nextlen
= tree
[n
+1].Len
;
731 if (++count
< max_count
&& curlen
== nextlen
) {
733 } else if (count
< min_count
) {
734 s
->bl_tree
[curlen
].Freq
+= count
;
735 } else if (curlen
!= 0) {
736 if (curlen
!= prevlen
) s
->bl_tree
[curlen
].Freq
++;
737 s
->bl_tree
[REP_3_6
].Freq
++;
738 } else if (count
<= 10) {
739 s
->bl_tree
[REPZ_3_10
].Freq
++;
741 s
->bl_tree
[REPZ_11_138
].Freq
++;
743 count
= 0; prevlen
= curlen
;
745 max_count
= 138, min_count
= 3;
746 } else if (curlen
== nextlen
) {
747 max_count
= 6, min_count
= 3;
749 max_count
= 7, min_count
= 4;
754 /* ===========================================================================
755 * Send a literal or distance tree in compressed form, using the codes in
758 local
void send_tree (s
, tree
, max_code
)
760 ct_data
*tree
; /* the tree to be scanned */
761 int max_code
; /* and its largest code of non zero frequency */
763 int n
; /* iterates over all tree elements */
764 int prevlen
= -1; /* last emitted length */
765 int curlen
; /* length of current code */
766 int nextlen
= tree
[0].Len
; /* length of next code */
767 int count
= 0; /* repeat count of the current code */
768 int max_count
= 7; /* max repeat count */
769 int min_count
= 4; /* min repeat count */
771 /* tree[max_code+1].Len = -1; */ /* guard already set */
772 if (nextlen
== 0) max_count
= 138, min_count
= 3;
774 for (n
= 0; n
<= max_code
; n
++) {
775 curlen
= nextlen
; nextlen
= tree
[n
+1].Len
;
776 if (++count
< max_count
&& curlen
== nextlen
) {
778 } else if (count
< min_count
) {
779 do { send_code(s
, curlen
, s
->bl_tree
); } while (--count
!= 0);
781 } else if (curlen
!= 0) {
782 if (curlen
!= prevlen
) {
783 send_code(s
, curlen
, s
->bl_tree
); count
--;
785 Assert(count
>= 3 && count
<= 6, " 3_6?");
786 send_code(s
, REP_3_6
, s
->bl_tree
); send_bits(s
, count
-3, 2);
788 } else if (count
<= 10) {
789 send_code(s
, REPZ_3_10
, s
->bl_tree
); send_bits(s
, count
-3, 3);
792 send_code(s
, REPZ_11_138
, s
->bl_tree
); send_bits(s
, count
-11, 7);
794 count
= 0; prevlen
= curlen
;
796 max_count
= 138, min_count
= 3;
797 } else if (curlen
== nextlen
) {
798 max_count
= 6, min_count
= 3;
800 max_count
= 7, min_count
= 4;
805 /* ===========================================================================
806 * Construct the Huffman tree for the bit lengths and return the index in
807 * bl_order of the last bit length code to send.
809 local
int build_bl_tree(s
)
812 int max_blindex
; /* index of last bit length code of non zero freq */
814 /* Determine the bit length frequencies for literal and distance trees */
815 scan_tree(s
, (ct_data
*)s
->dyn_ltree
, s
->l_desc
.max_code
);
816 scan_tree(s
, (ct_data
*)s
->dyn_dtree
, s
->d_desc
.max_code
);
818 /* Build the bit length tree: */
819 build_tree(s
, (tree_desc
*)(&(s
->bl_desc
)));
820 /* opt_len now includes the length of the tree representations, except
821 * the lengths of the bit lengths codes and the 5+5+4 bits for the counts.
824 /* Determine the number of bit length codes to send. The pkzip format
825 * requires that at least 4 bit length codes be sent. (appnote.txt says
826 * 3 but the actual value used is 4.)
828 for (max_blindex
= BL_CODES
-1; max_blindex
>= 3; max_blindex
--) {
829 if (s
->bl_tree
[bl_order
[max_blindex
]].Len
!= 0) break;
831 /* Update opt_len to include the bit length tree and counts */
832 s
->opt_len
+= 3*(max_blindex
+1) + 5+5+4;
833 Tracev((stderr
, "\ndyn trees: dyn %ld, stat %ld",
834 s
->opt_len
, s
->static_len
));
839 /* ===========================================================================
840 * Send the header for a block using dynamic Huffman trees: the counts, the
841 * lengths of the bit length codes, the literal tree and the distance tree.
842 * IN assertion: lcodes >= 257, dcodes >= 1, blcodes >= 4.
844 local
void send_all_trees(s
, lcodes
, dcodes
, blcodes
)
846 int lcodes
, dcodes
, blcodes
; /* number of codes for each tree */
848 int rank
; /* index in bl_order */
850 Assert (lcodes
>= 257 && dcodes
>= 1 && blcodes
>= 4, "not enough codes");
851 Assert (lcodes
<= L_CODES
&& dcodes
<= D_CODES
&& blcodes
<= BL_CODES
,
853 Tracev((stderr
, "\nbl counts: "));
854 send_bits(s
, lcodes
-257, 5); /* not +255 as stated in appnote.txt */
855 send_bits(s
, dcodes
-1, 5);
856 send_bits(s
, blcodes
-4, 4); /* not -3 as stated in appnote.txt */
857 for (rank
= 0; rank
< blcodes
; rank
++) {
858 Tracev((stderr
, "\nbl code %2d ", bl_order
[rank
]));
859 send_bits(s
, s
->bl_tree
[bl_order
[rank
]].Len
, 3);
861 Tracev((stderr
, "\nbl tree: sent %ld", s
->bits_sent
));
863 send_tree(s
, (ct_data
*)s
->dyn_ltree
, lcodes
-1); /* literal tree */
864 Tracev((stderr
, "\nlit tree: sent %ld", s
->bits_sent
));
866 send_tree(s
, (ct_data
*)s
->dyn_dtree
, dcodes
-1); /* distance tree */
867 Tracev((stderr
, "\ndist tree: sent %ld", s
->bits_sent
));
870 /* ===========================================================================
871 * Send a stored block
873 void ZLIB_INTERNAL
_tr_stored_block(s
, buf
, stored_len
, last
)
875 charf
*buf
; /* input block */
876 ulg stored_len
; /* length of input block */
877 int last
; /* one if this is the last block for a file */
879 send_bits(s
, (STORED_BLOCK
<<1)+last
, 3); /* send block type */
881 s
->compressed_len
= (s
->compressed_len
+ 3 + 7) & (ulg
)~7L;
882 s
->compressed_len
+= (stored_len
+ 4) << 3;
884 copy_block(s
, buf
, (unsigned)stored_len
, 1); /* with header */
887 /* ===========================================================================
888 * Flush the bits in the bit buffer to pending output (leaves at most 7 bits)
890 void ZLIB_INTERNAL
_tr_flush_bits(s
)
896 /* ===========================================================================
897 * Send one empty static block to give enough lookahead for inflate.
898 * This takes 10 bits, of which 7 may remain in the bit buffer.
900 void ZLIB_INTERNAL
_tr_align(s
)
903 send_bits(s
, STATIC_TREES
<<1, 3);
904 send_code(s
, END_BLOCK
, static_ltree
);
906 s
->compressed_len
+= 10L; /* 3 for block type, 7 for EOB */
911 /* ===========================================================================
912 * Determine the best encoding for the current block: dynamic trees, static
913 * trees or store, and output the encoded block to the zip file.
915 void ZLIB_INTERNAL
_tr_flush_block(s
, buf
, stored_len
, last
)
917 charf
*buf
; /* input block, or NULL if too old */
918 ulg stored_len
; /* length of input block */
919 int last
; /* one if this is the last block for a file */
921 int max_blindex
= 0; /* index of last bit length code of non zero freq */
922 #ifndef ZLIB_PM3_TUNED
923 ulg opt_lenb
, static_lenb
; /* opt_len and static_len in bytes */
925 /* Build the Huffman trees unless a stored block is forced */
928 /* Check if the file is binary or text */
929 if (s
->strm
->data_type
== Z_UNKNOWN
)
930 s
->strm
->data_type
= detect_data_type(s
);
932 /* Construct the literal and distance trees */
933 build_tree(s
, (tree_desc
*)(&(s
->l_desc
)));
934 Tracev((stderr
, "\nlit data: dyn %ld, stat %ld", s
->opt_len
,
937 build_tree(s
, (tree_desc
*)(&(s
->d_desc
)));
938 Tracev((stderr
, "\ndist data: dyn %ld, stat %ld", s
->opt_len
,
940 /* At this point, opt_len and static_len are the total bit lengths of
941 * the compressed block data, excluding the tree representations.
944 /* Build the bit length tree for the above two trees, and get the index
945 * in bl_order of the last bit length code to send.
947 max_blindex
= build_bl_tree(s
);
949 #ifndef ZLIB_PM3_TUNED
950 /* Determine the best encoding. Compute the block lengths in bytes. */
951 opt_lenb
= (s
->opt_len
+3+7)>>3;
952 static_lenb
= (s
->static_len
+3+7)>>3;
954 Tracev((stderr
, "\nopt %lu(%lu) stat %lu(%lu) stored %lu lit %u ",
955 opt_lenb
, s
->opt_len
, static_lenb
, s
->static_len
, stored_len
,
958 if (static_lenb
<= opt_lenb
) opt_lenb
= static_lenb
;
961 Assert(buf
!= (char*)0, "lost buf");
962 opt_lenb
= static_lenb
= stored_len
+ 5; /* force a stored block */
966 if (buf
!= (char*)0) { /* force stored block */
968 if (stored_len
+4 <= opt_lenb
&& buf
!= (char*)0) {
969 /* 4: two words for the lengths */
971 /* The test buf != NULL is only necessary if LIT_BUFSIZE > WSIZE.
972 * Otherwise we can't have processed more than WSIZE input bytes since
973 * the last block flush, because compression would have been
974 * successful. If LIT_BUFSIZE <= WSIZE, it is never too late to
975 * transform a block into a stored block.
977 _tr_stored_block(s
, buf
, stored_len
, last
);
980 } else if (static_lenb
>= 0) { /* force static trees */
982 } else if (s
->strategy
== Z_FIXED
|| static_lenb
== opt_lenb
) {
984 send_bits(s
, (STATIC_TREES
<<1)+last
, 3);
985 compress_block(s
, (const ct_data
*)static_ltree
,
986 (const ct_data
*)static_dtree
);
988 s
->compressed_len
+= 3 + s
->static_len
;
991 #endif /* !ZLIB_PM3_TUNED */
992 send_bits(s
, (DYN_TREES
<<1)+last
, 3);
993 send_all_trees(s
, s
->l_desc
.max_code
+1, s
->d_desc
.max_code
+1,
995 compress_block(s
, (const ct_data
*)s
->dyn_ltree
,
996 (const ct_data
*)s
->dyn_dtree
);
998 s
->compressed_len
+= 3 + s
->opt_len
;
1000 #ifndef ZLIB_PM3_TUNED
1003 Assert (s
->compressed_len
== s
->bits_sent
, "bad compressed size");
1004 /* The above check is made mod 2^32, for files larger than 512 MB
1005 * and uLong implemented on 32 bits.
1012 s
->compressed_len
+= 7; /* align on byte boundary */
1015 Tracev((stderr
,"\ncomprlen %lu(%lu) ", s
->compressed_len
>>3,
1016 s
->compressed_len
-7*last
));
1019 /* ===========================================================================
1020 * Save the match info and tally the frequency counts. Return true if
1021 * the current block must be flushed.
1023 int ZLIB_INTERNAL
_tr_tally (s
, dist
, lc
)
1025 unsigned dist
; /* distance of matched string */
1026 unsigned lc
; /* match length-MIN_MATCH or unmatched char (if dist==0) */
1028 s
->d_buf
[s
->last_lit
] = (ush
)dist
;
1029 s
->l_buf
[s
->last_lit
++] = (uch
)lc
;
1031 /* lc is the unmatched char */
1032 s
->dyn_ltree
[lc
].Freq
++;
1035 /* Here, lc is the match length - MIN_MATCH */
1036 dist
--; /* dist = match distance - 1 */
1037 Assert((ush
)dist
< (ush
)MAX_DIST(s
) &&
1038 (ush
)lc
<= (ush
)(MAX_MATCH
-MIN_MATCH
) &&
1039 (ush
)d_code(dist
) < (ush
)D_CODES
, "_tr_tally: bad match");
1041 s
->dyn_ltree
[_length_code
[lc
]+LITERALS
+1].Freq
++;
1042 s
->dyn_dtree
[d_code(dist
)].Freq
++;
1045 #ifdef TRUNCATE_BLOCK
1046 /* Try to guess if it is profitable to stop the current block here */
1047 if ((s
->last_lit
& 0x1fff) == 0 && s
->level
> 2) {
1048 /* Compute an upper bound for the compressed length */
1049 ulg out_length
= (ulg
)s
->last_lit
*8L;
1050 ulg in_length
= (ulg
)((long)s
->strstart
- s
->block_start
);
1052 for (dcode
= 0; dcode
< D_CODES
; dcode
++) {
1053 out_length
+= (ulg
)s
->dyn_dtree
[dcode
].Freq
*
1054 (5L+extra_dbits
[dcode
]);
1057 Tracev((stderr
,"\nlast_lit %u, in %ld, out ~%ld(%ld%%) ",
1058 s
->last_lit
, in_length
, out_length
,
1059 100L - out_length
*100L/in_length
));
1060 if (s
->matches
< s
->last_lit
/2 && out_length
< in_length
/2) return 1;
1063 return (s
->last_lit
== s
->lit_bufsize
-1);
1064 /* We avoid equality with lit_bufsize because of wraparound at 64K
1065 * on 16 bit machines and because stored blocks are restricted to
1070 /* ===========================================================================
1071 * Send the block data compressed using the given Huffman trees
1073 local
void compress_block(s
, ltree
, dtree
)
1075 const ct_data
*ltree
; /* literal tree */
1076 const ct_data
*dtree
; /* distance tree */
1078 unsigned dist
; /* distance of matched string */
1079 int lc
; /* match length or unmatched char (if dist == 0) */
1080 unsigned lx
= 0; /* running index in l_buf */
1081 unsigned code
; /* the code to send */
1082 int extra
; /* number of extra bits to send */
1084 if (s
->last_lit
!= 0) do {
1085 dist
= s
->d_buf
[lx
];
1086 lc
= s
->l_buf
[lx
++];
1088 send_code(s
, lc
, ltree
); /* send a literal byte */
1089 Tracecv(isgraph(lc
), (stderr
," '%c' ", lc
));
1091 /* Here, lc is the match length - MIN_MATCH */
1092 code
= _length_code
[lc
];
1093 send_code(s
, code
+LITERALS
+1, ltree
); /* send the length code */
1094 extra
= extra_lbits
[code
];
1096 lc
-= base_length
[code
];
1097 send_bits(s
, lc
, extra
); /* send the extra length bits */
1099 dist
--; /* dist is now the match distance - 1 */
1100 code
= d_code(dist
);
1101 Assert (code
< D_CODES
, "bad d_code");
1103 send_code(s
, code
, dtree
); /* send the distance code */
1104 extra
= extra_dbits
[code
];
1106 dist
-= base_dist
[code
];
1107 send_bits(s
, dist
, extra
); /* send the extra distance bits */
1109 } /* literal or match pair ? */
1111 /* Check that the overlay between pending_buf and d_buf+l_buf is ok: */
1112 Assert((uInt
)(s
->pending
) < s
->lit_bufsize
+ 2*lx
,
1113 "pendingBuf overflow");
1115 } while (lx
< s
->last_lit
);
1117 send_code(s
, END_BLOCK
, ltree
);
1120 /* ===========================================================================
1121 * Check if the data type is TEXT or BINARY, using the following algorithm:
1122 * - TEXT if the two conditions below are satisfied:
1123 * a) There are no non-portable control characters belonging to the
1124 * "black list" (0..6, 14..25, 28..31).
1125 * b) There is at least one printable character belonging to the
1126 * "white list" (9 {TAB}, 10 {LF}, 13 {CR}, 32..255).
1127 * - BINARY otherwise.
1128 * - The following partially-portable control characters form a
1129 * "gray list" that is ignored in this detection algorithm:
1130 * (7 {BEL}, 8 {BS}, 11 {VT}, 12 {FF}, 26 {SUB}, 27 {ESC}).
1131 * IN assertion: the fields Freq of dyn_ltree are set.
1133 local
int detect_data_type(s
)
1136 /* black_mask is the bit mask of black-listed bytes
1137 * set bits 0..6, 14..25, and 28..31
1138 * 0xf3ffc07f = binary 11110011111111111100000001111111
1140 unsigned long black_mask
= 0xf3ffc07fUL
;
1143 /* Check for non-textual ("black-listed") bytes. */
1144 for (n
= 0; n
<= 31; n
++, black_mask
>>= 1)
1145 if ((black_mask
& 1) && (s
->dyn_ltree
[n
].Freq
!= 0))
1148 /* Check for textual ("white-listed") bytes. */
1149 if (s
->dyn_ltree
[9].Freq
!= 0 || s
->dyn_ltree
[10].Freq
!= 0
1150 || s
->dyn_ltree
[13].Freq
!= 0)
1152 for (n
= 32; n
< LITERALS
; n
++)
1153 if (s
->dyn_ltree
[n
].Freq
!= 0)
1156 /* There are no "black-listed" or "white-listed" bytes:
1157 * this stream either is empty or has tolerated ("gray-listed") bytes only.
1162 /* ===========================================================================
1163 * Reverse the first len bits of a code, using straightforward code (a faster
1164 * method would use a table)
1165 * IN assertion: 1 <= len <= 15
1167 local
unsigned bi_reverse(code
, len
)
1168 unsigned code
; /* the value to invert */
1169 int len
; /* its bit length */
1171 register unsigned res
= 0;
1174 code
>>= 1, res
<<= 1;
1175 } while (--len
> 0);
1179 /* ===========================================================================
1180 * Flush the bit buffer, keeping at most 7 bits in it.
1182 local
void bi_flush(s
)
1185 if (s
->bi_valid
== 16) {
1186 put_short(s
, s
->bi_buf
);
1189 } else if (s
->bi_valid
>= 8) {
1190 put_byte(s
, (Byte
)s
->bi_buf
);
1196 /* ===========================================================================
1197 * Flush the bit buffer and align the output on a byte boundary
1199 local
void bi_windup(s
)
1202 if (s
->bi_valid
> 8) {
1203 put_short(s
, s
->bi_buf
);
1204 } else if (s
->bi_valid
> 0) {
1205 put_byte(s
, (Byte
)s
->bi_buf
);
1210 s
->bits_sent
= (s
->bits_sent
+7) & ~7;
1214 /* ===========================================================================
1215 * Copy a stored block, storing first the length and its
1216 * one's complement if requested.
1218 local
void copy_block(s
, buf
, len
, header
)
1220 charf
*buf
; /* the input data */
1221 unsigned len
; /* its length */
1222 int header
; /* true if block header must be written */
1224 bi_windup(s
); /* align on byte boundary */
1227 put_short(s
, (ush
)len
);
1228 put_short(s
, (ush
)~len
);
1230 s
->bits_sent
+= 2*16;
1234 s
->bits_sent
+= (ulg
)len
<<3;
1237 put_byte(s
, *buf
++);