#pragma prototyped12/*3* compress/zcat discipline snarfed from BSD zopen by4* Glenn Fowler5* AT&T Research6* 1999-06-237*/89/*-10* Copyright (c) 1985, 1986, 1992, 199311* The Regents of the University of California. All rights reserved.12*13* This code is derived from software contributed to Berkeley by14* Diomidis Spinellis and James A. Woods, derived from original15* work by Spencer Thomas and Joseph Orost.16*17* Redistribution and use in source and binary forms, with or without18* modification, are permitted provided that the following conditions19* are met:20* 1. Redistributions of source code must retain the above copyright21* notice, this list of conditions and the following disclaimer.22* 2. Redistributions in binary form must reproduce the above copyright23* notice, this list of conditions and the following disclaimer in the24* documentation and/or other materials provided with the distribution.25* 3. Neither the name of the University nor the names of its contributors26* may be used to endorse or promote products derived from this software27* without specific prior written permission.28*29* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND30* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE31* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE32* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE33* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL34* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS35* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)36* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT37* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY38* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF39* SUCH DAMAGE.40*/4142/*-43* fcompress.c - File compression ala IEEE Computer, June 1984.44*45* Compress authors:46* Spencer W. Thomas (decvax!utah-cs!thomas)47* Jim McKie (decvax!mcvax!jim)48* Steve Davies (decvax!vax135!petsd!peora!srd)49* Ken Turkowski (decvax!decwrl!turtlevax!ken)50* James A. Woods (decvax!ihnp4!ames!jaw)51* Joe Orost (decvax!vax135!petsd!joe)52*53* Cleaned up and converted to library returning I/O streams by54* Diomidis Spinellis <[email protected]>.55*/5657#include <sfio_t.h>58#include <ast.h>59#include <sfdcgzip.h>6061#define BITS 16 /* Default bits. */62#define HSIZE 69001 /* 95% occupancy */6364/* A code_int must be able to hold 2**BITS values of type int, and also -1. */65typedef long code_int;66typedef long count_int;6768typedef u_char char_type;69static char_type magic_header[] =70{ 0x1f, 0x9d };7172#define BIT_MASK 0x1f /* Defines for third byte of header. */73#define BLOCK_MASK 0x807475/*76* Masks 0x40 and 0x20 are free. I think 0x20 should mean that there is77* a fourth header byte (for expansion).78*/79#define INIT_BITS 9 /* Initial number of bits/code. */8081#define MAXCODE(n_bits) ((1 << (n_bits)) - 1)8283typedef struct s_zstate {84Sfdisc_t disc; /* sfio discipline */85enum {86S_START, S_MIDDLE, S_EOF87} zs_state; /* State of computation */88int zs_n_bits; /* Number of bits/code. */89int zs_maxbits; /* User settable max # bits/code. */90code_int zs_maxcode; /* Maximum code, given n_bits. */91code_int zs_maxmaxcode; /* Should NEVER generate this code. */92count_int zs_htab [HSIZE];93u_short zs_codetab [HSIZE];94code_int zs_hsize; /* For dynamic table sizing. */95code_int zs_free_ent; /* First unused entry. */96/*97* Block compression parameters -- after all codes are used up,98* and compression rate changes, start over.99*/100int zs_block_compress;101int zs_clear_flg;102long zs_ratio;103count_int zs_checkpoint;104int zs_offset;105long zs_in_count; /* Length of input. */106long zs_bytes_out; /* Length of compressed output. */107long zs_out_count; /* # of codes output (for debugging). */108char_type zs_buf[BITS];109union {110struct {111long zs_fcode;112code_int zs_ent;113code_int zs_hsize_reg;114int zs_hshift;115} w; /* Write paramenters */116struct {117char_type *zs_stackp;118int zs_finchar;119code_int zs_code, zs_oldcode, zs_incode;120int zs_roffset, zs_size;121char_type zs_gbuf[BITS];122} r; /* Read parameters */123} u;124} LZW_t;125126/* Definitions to retain old variable names */127#define state zs->zs_state128#define n_bits zs->zs_n_bits129#define maxbits zs->zs_maxbits130#define maxcode zs->zs_maxcode131#define maxmaxcode zs->zs_maxmaxcode132#define htab zs->zs_htab133#define codetab zs->zs_codetab134#define hsize zs->zs_hsize135#define free_ent zs->zs_free_ent136#define block_compress zs->zs_block_compress137#define clear_flg zs->zs_clear_flg138#define ratio zs->zs_ratio139#define checkpoint zs->zs_checkpoint140#define offset zs->zs_offset141#define in_count zs->zs_in_count142#define bytes_out zs->zs_bytes_out143#define out_count zs->zs_out_count144#define buf zs->zs_buf145#define fcode zs->u.w.zs_fcode146#define hsize_reg zs->u.w.zs_hsize_reg147#define ent zs->u.w.zs_ent148#define hshift zs->u.w.zs_hshift149#define stackp zs->u.r.zs_stackp150#define finchar zs->u.r.zs_finchar151#define code zs->u.r.zs_code152#define oldcode zs->u.r.zs_oldcode153#define incode zs->u.r.zs_incode154#define roffset zs->u.r.zs_roffset155#define size zs->u.r.zs_size156#define gbuf zs->u.r.zs_gbuf157158/*159* To save much memory, we overlay the table used by compress() with those160* used by decompress(). The tab_prefix table is the same size and type as161* the codetab. The tab_suffix table needs 2**BITS characters. We get this162* from the beginning of htab. The output stack uses the rest of htab, and163* contains characters. There is plenty of room for any possible stack164* (stack used to be 8000 characters).165*/166167#define htabof(i) htab[i]168#define codetabof(i) codetab[i]169170#define tab_prefixof(i) codetabof(i)171#define tab_suffixof(i) ((char_type *)(htab))[i]172#define de_stack ((char_type *)&tab_suffixof(1 << BITS))173174#define CHECK_GAP 10000 /* Ratio check interval. */175176/*177* the next two codes should not be changed lightly, as they must not178* lie within the contiguous general code space.179*/180#define FIRST 257 /* First free entry. */181#define CLEAR 256 /* Table clear output code. */182183/*-184* Algorithm from "A Technique for High Performance Data Compression",185* Terry A. Welch, IEEE Computer Vol 17, No 6 (June 1984), pp 8-19.186*187* Algorithm:188* Modified Lempel-Ziv method (LZW). Basically finds common189* substrings and replaces them with a variable size code. This is190* deterministic, and can be done on the fly. Thus, the decompression191* procedure needs no input table, but tracks the way the table was built.192*/193194/*-195* Output the given code.196* Inputs:197* code: A n_bits-bit integer. If == -1, then EOF. This assumes198* that n_bits =< (long)wordsize - 1.199* Outputs:200* Outputs code to the file.201* Assumptions:202* Chars are 8 bits long.203* Algorithm:204* Maintain a BITS character long buffer (so that 8 codes will205* fit in it exactly). Use the VAX insv instruction to insert each206* code in turn. When the buffer fills up empty it and start over.207*/208209static char_type lmask[9] =210{0xff, 0xfe, 0xfc, 0xf8, 0xf0, 0xe0, 0xc0, 0x80, 0x00};211static char_type rmask[9] =212{0x00, 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f, 0x7f, 0xff};213214static int215output(LZW_t* zs, Sfio_t* f, code_int ocode, Sfdisc_t* dp)216{217register int bits, r_off;218register char_type *bp;219220r_off = offset;221bits = n_bits;222bp = buf;223if (ocode >= 0) {224/* Get to the first byte. */225bp += (r_off >> 3);226r_off &= 7;227/*228* Since ocode is always >= 8 bits, only need to mask the first229* hunk on the left.230*/231*bp = (*bp & rmask[r_off]) | ((ocode << r_off) & lmask[r_off]);232bp++;233bits -= (8 - r_off);234ocode >>= 8 - r_off;235/* Get any 8 bit parts in the middle (<=1 for up to 16 bits). */236if (bits >= 8) {237*bp++ = ocode;238ocode >>= 8;239bits -= 8;240}241/* Last bits. */242if (bits)243*bp = ocode;244offset += n_bits;245if (offset == (n_bits << 3)) {246bp = buf;247bits = n_bits;248bytes_out += bits;249if (sfwr(f, bp, bits, dp) != bits)250return (-1);251bp += bits;252bits = 0;253offset = 0;254}255/*256* If the next entry is going to be too big for the ocode size,257* then increase it, if possible.258*/259if (free_ent > maxcode || (clear_flg > 0)) {260/*261* Write the whole buffer, because the input side won't262* discover the size increase until after it has read it.263*/264if (offset > 0) {265if (sfwr(f, buf, n_bits, dp) != n_bits)266return (-1);267bytes_out += n_bits;268}269offset = 0;270271if (clear_flg) {272maxcode = MAXCODE(n_bits = INIT_BITS);273clear_flg = 0;274} else {275n_bits++;276if (n_bits == maxbits)277maxcode = maxmaxcode;278else279maxcode = MAXCODE(n_bits);280}281}282} else {283/* At EOF, write the rest of the buffer. */284if (offset > 0) {285offset = (offset + 7) / 8;286if (sfwr(f, buf, offset, dp) != offset)287return (-1);288bytes_out += offset;289}290offset = 0;291}292return (0);293}294295/*-296* Read one code from the standard input. If EOF, return -1.297* Inputs:298* stdin299* Outputs:300* code or -1 is returned.301*/302static code_int303getcode(LZW_t* zs, Sfio_t* f, Sfdisc_t* dp)304{305register code_int gcode;306register int r_off, bits;307register char_type *bp;308309bp = gbuf;310if (clear_flg > 0 || roffset >= size || free_ent > maxcode) {311/*312* If the next entry will be too big for the current gcode313* size, then we must increase the size. This implies reading314* a new buffer full, too.315*/316if (free_ent > maxcode) {317n_bits++;318if (n_bits == maxbits) /* Won't get any bigger now. */319maxcode = maxmaxcode;320else321maxcode = MAXCODE(n_bits);322}323if (clear_flg > 0) {324maxcode = MAXCODE(n_bits = INIT_BITS);325clear_flg = 0;326}327size = sfrd(f, gbuf, n_bits, dp);328if (size <= 0) /* End of file. */329return (-1);330roffset = 0;331/* Round size down to integral number of codes. */332size = (size << 3) - (n_bits - 1);333}334r_off = roffset;335bits = n_bits;336337/* Get to the first byte. */338bp += (r_off >> 3);339r_off &= 7;340341/* Get first part (low order bits). */342gcode = (*bp++ >> r_off);343bits -= (8 - r_off);344r_off = 8 - r_off; /* Now, roffset into gcode word. */345346/* Get any 8 bit parts in the middle (<=1 for up to 16 bits). */347if (bits >= 8) {348gcode |= *bp++ << r_off;349r_off += 8;350bits -= 8;351}352353/* High order bits. */354gcode |= (*bp & rmask[bits]) << r_off;355roffset += n_bits;356357return (gcode);358}359360static void361cl_hash(LZW_t* zs, register count_int cl_hsize) /* Reset code table. */362{363register count_int *htab_p;364register long i, m1;365366m1 = -1;367htab_p = htab + cl_hsize;368i = cl_hsize - 16;369do { /* Might use Sys V memset(3) here. */370*(htab_p - 16) = m1;371*(htab_p - 15) = m1;372*(htab_p - 14) = m1;373*(htab_p - 13) = m1;374*(htab_p - 12) = m1;375*(htab_p - 11) = m1;376*(htab_p - 10) = m1;377*(htab_p - 9) = m1;378*(htab_p - 8) = m1;379*(htab_p - 7) = m1;380*(htab_p - 6) = m1;381*(htab_p - 5) = m1;382*(htab_p - 4) = m1;383*(htab_p - 3) = m1;384*(htab_p - 2) = m1;385*(htab_p - 1) = m1;386htab_p -= 16;387} while ((i -= 16) >= 0);388for (i += 16; i > 0; i--)389*--htab_p = m1;390}391392static int393cl_block(LZW_t* zs, Sfio_t* f, Sfdisc_t* dp)/* Table clear for block compress. */394{395register long rat;396397checkpoint = in_count + CHECK_GAP;398399if (in_count > 0x007fffff) { /* Shift will overflow. */400rat = bytes_out >> 8;401if (rat == 0) /* Don't divide by zero. */402rat = 0x7fffffff;403else404rat = in_count / rat;405} else406rat = (in_count << 8) / bytes_out; /* 8 fractional bits. */407if (rat > ratio)408ratio = rat;409else {410ratio = 0;411cl_hash(zs, (count_int) hsize);412free_ent = FIRST;413clear_flg = 1;414if (output(zs, f, (code_int) CLEAR, dp) == -1)415return (-1);416}417return (0);418}419420/*421* lzw sync/flush/getpos422* only sync is implemented right now423* see sfdcgzip() for full implementation424*/425426static Sfoff_t427lzw_sync(LZW_t* zs, Sfio_t* f, Sfoff_t off, Sfdisc_t* dp)428{429#if 1430static int nosync;431432if (off == -1)433{434if (!nosync)435nosync = getenv("SFDCLZW_nosync") ? 1 : -1;436if (nosync > 0)437return 0;438}439#endif440if (zs->disc.writef && cl_block(zs, f, dp))441return -1;442if (off == -1)443return 0;444return -1;445}446447/*448* lzw exception handler449* free on close450*/451452static int453lzw_except(Sfio_t* f, int op, void* val, Sfdisc_t* dp)454{455register LZW_t* zs = (LZW_t*)dp;456int flags;457int r;458459NoP(f);460switch (op)461{462case SF_ATEXIT:463sfdisc(f, SF_POPDISC);464return 0;465case SF_CLOSING:466case SF_DPOP:467case SF_FINAL:468r = 0;469if (dp->writef)470{471SFDCNEXT(f, flags);472if (output(zs, f, (code_int) ent, dp) == -1)473r = -1;474else475{476out_count++;477if (output(zs, f, (code_int) -1, dp) == -1)478r = -1;479}480SFDCPREV(f, flags);481}482if (op != SF_CLOSING)483free(dp);484return r;485case SF_DBUFFER:486return 1;487case SF_READ:488case SF_WRITE:489return *((ssize_t*)val) < 0 ? -1 : 0;490case SF_SYNC:491return val ? 0 : lzw_sync(zs, f, -1, dp) == -1 ? -1 : 0;492case SFGZ_HANDLE:493return (*((LZW_t**)val) = zs) ? 1 : -1;494case SFGZ_GETPOS:495return (*((Sfoff_t*)val) = lzw_sync(zs, f, -1, dp)) == -1 ? -1 : 0;496case SFGZ_SETPOS:497return lzw_sync(zs, f, *((Sfoff_t*)val), dp) == -1 ? -1 : 0;498}499return 0;500}501502/*-503* compress write504*505* Algorithm: use open addressing double hashing (no chaining) on the506* prefix code / next character combination. We do a variant of Knuth's507* algorithm D (vol. 3, sec. 6.4) along with G. Knott's relatively-prime508* secondary probe. Here, the modular division first probe is gives way509* to a faster exclusive-or manipulation. Also do block compression with510* an adaptive reset, whereby the code table is cleared when the compression511* ratio decreases, but after the table fills. The variable-length output512* codes are re-sized at this point, and a special CLEAR code is generated513* for the decompressor. Late addition: construct the table according to514* file size for noticeable speed improvement on small files. Please direct515* questions about this implementation to ames!jaw.516*/517518static ssize_t519lzw_write(Sfio_t* f, const Void_t* wbp, size_t num, Sfdisc_t* dp)520{521register code_int i;522register int c, disp;523LZW_t *zs;524const u_char *bp;525u_char tmp;526int count;527528if (num == 0)529return (0);530531zs = (LZW_t*)dp;532count = num;533bp = (u_char *)wbp;534if (state == S_MIDDLE)535goto middle;536state = S_MIDDLE;537538maxmaxcode = 1L << maxbits;539if (sfwr(f, magic_header, sizeof(magic_header), dp) != sizeof(magic_header))540return (-1);541tmp = (u_char)((maxbits) | block_compress);542if (sfwr(f, &tmp, sizeof(tmp), dp) != sizeof(tmp))543return (-1);544545offset = 0;546bytes_out = 3; /* Includes 3-byte header mojo. */547out_count = 0;548clear_flg = 0;549ratio = 0;550in_count = 1;551checkpoint = CHECK_GAP;552maxcode = MAXCODE(n_bits = INIT_BITS);553free_ent = ((block_compress) ? FIRST : 256);554555ent = *bp++;556--count;557558hshift = 0;559for (fcode = (long)hsize; fcode < 65536L; fcode *= 2L)560hshift++;561hshift = 8 - hshift; /* Set hash code range bound. */562563hsize_reg = hsize;564cl_hash(zs, (count_int)hsize_reg); /* Clear hash table. */565566middle: for (i = 0; count--;) {567c = *bp++;568in_count++;569fcode = (long)(((long)c << maxbits) + ent);570i = ((c << hshift) ^ ent); /* Xor hashing. */571572if (htabof(i) == fcode) {573ent = codetabof(i);574continue;575} else if ((long)htabof(i) < 0) /* Empty slot. */576goto nomatch;577disp = hsize_reg - i; /* Secondary hash (after G. Knott). */578if (i == 0)579disp = 1;580probe: if ((i -= disp) < 0)581i += hsize_reg;582583if (htabof(i) == fcode) {584ent = codetabof(i);585continue;586}587if ((long)htabof(i) >= 0)588goto probe;589nomatch: if (output(zs, f, (code_int) ent, dp) == -1)590return (-1);591out_count++;592ent = c;593if (free_ent < maxmaxcode) {594codetabof(i) = free_ent++; /* code -> hashtable */595htabof(i) = fcode;596} else if ((count_int)in_count >=597checkpoint && block_compress) {598if (cl_block(zs, f, dp) == -1)599return (-1);600}601}602return (num);603}604605/*606* Decompress read. This routine adapts to the codes in the file building607* the "string" table on-the-fly; requiring no table to be stored in the608* compressed file. The tables used herein are shared with those of the609* compress() routine. See the definitions above.610*/611static ssize_t612lzw_read(Sfio_t* f, Void_t* rbp, size_t num, Sfdisc_t* dp)613{614register u_int count;615LZW_t *zs;616u_char *bp, header[3];617618if (num == 0)619return (0);620621zs = (LZW_t*)dp;622count = num;623bp = (u_char *)rbp;624switch (state) {625case S_START:626state = S_MIDDLE;627break;628case S_MIDDLE:629goto middle;630case S_EOF:631goto eof;632}633634/* Check the magic number */635if (sfrd(f, header, sizeof(header), dp) != sizeof(header) ||636memcmp(header, magic_header, sizeof(magic_header)) != 0)637return (-1);638maxbits = header[2]; /* Set -b from file. */639block_compress = maxbits & BLOCK_MASK;640maxbits &= BIT_MASK;641maxmaxcode = 1L << maxbits;642if (maxbits > BITS)643return (-1);644/* As above, initialize the first 256 entries in the table. */645maxcode = MAXCODE(n_bits = INIT_BITS);646for (code = 255; code >= 0; code--) {647tab_prefixof(code) = 0;648tab_suffixof(code) = (char_type) code;649}650free_ent = block_compress ? FIRST : 256;651652finchar = oldcode = getcode(zs, f, dp);653if (oldcode == -1) /* EOF already? */654return (0); /* Get out of here */655656/* First code must be 8 bits = char. */657*bp++ = (u_char)finchar;658count--;659stackp = de_stack;660661while ((code = getcode(zs, f, dp)) > -1) {662663if ((code == CLEAR) && block_compress) {664for (code = 255; code >= 0; code--)665tab_prefixof(code) = 0;666clear_flg = 1;667free_ent = FIRST - 1;668if ((code = getcode(zs, f, dp)) == -1) /* O, untimely death! */669break;670}671incode = code;672673/* Special case for KwKwK string. */674if (code >= free_ent) {675*stackp++ = finchar;676code = oldcode;677}678679/* Generate output characters in reverse order. */680while (code >= 256) {681*stackp++ = tab_suffixof(code);682code = tab_prefixof(code);683}684*stackp++ = finchar = tab_suffixof(code);685686/* And put them out in forward order. */687middle: do {688if (count-- == 0)689return (num);690*bp++ = *--stackp;691} while (stackp > de_stack);692693/* Generate the new entry. */694if ((code = free_ent) < maxmaxcode) {695tab_prefixof(code) = (u_short) oldcode;696tab_suffixof(code) = finchar;697free_ent = code + 1;698}699700/* Remember previous code. */701oldcode = incode;702}703state = S_EOF;704eof: return (num - count);705}706707/*708* create and push the sfio lzw discipline709* for SF_READ streams only710*711* (flags&SFGZ_VERIFY) return712* >0 is an lzw file713* 0 not an lzw file714* <0 error715* otherwise return716* >0 discipline pushed (gzip or lzw)717* 0 discipline not needed718* <0 error719*/720721int722sfdclzw(Sfio_t* f, int flags)723{724LZW_t* zs;725726if (sfset(f, 0, 0) & SF_READ)727{728register unsigned char* s;729register int n;730731/*732* peek the first 2 bytes to verify the magic733*734* 0x1f8b sfdcgzip gzip735* 0x1f9d sfdclzw compress736*/737738if (!(n = sfset(f, 0, 0) & SF_SHARE))739sfset(f, SF_SHARE, 1);740s = (unsigned char*)sfreserve(f, 2, 1);741if (!n)742sfset(f, SF_SHARE, 0);743if (!s)744return -1;745if (*s != 0x1f)746n = -1;747else if (*(s + 1) == 0x8b)748n = 1;749else if (*(s + 1) == 0x9d)750n = 0;751else752n = -1;753sfread(f, s, 0);754if (n < 0)755return 0;756if (flags & SFGZ_VERIFY)757return n >= 0;758if (n > 0)759return sfdcgzip(f, flags);760}761if (!(zs = newof(0, LZW_t, 1, 0)))762return -1;763zs->disc.exceptf = lzw_except;764if (sfset(f, 0, 0) & SF_READ)765zs->disc.readf = lzw_read;766else767zs->disc.writef = lzw_write;768769maxbits = BITS; /* fixed max # bits/code. */770maxmaxcode = 1L << maxbits; /* Should NEVER generate this code. */771hsize = HSIZE; /* For dynamic table sizing. */772free_ent = 0; /* First unused entry. */773block_compress = BLOCK_MASK;774clear_flg = 0;775ratio = 0;776checkpoint = CHECK_GAP;777in_count = 1; /* Length of input. */778out_count = 0; /* # of codes output (for debugging). */779state = S_START;780roffset = 0;781size = 0;782783sfset(f, SF_SHARE|SF_PUBLIC, 0);784if (sfdisc(f, &zs->disc) != &zs->disc)785{786free(zs);787return -1;788}789return 1;790}791792793