View Javadoc
1   /*
2    * Copyright 2013 The Netty Project
3    *
4    * The Netty Project licenses this file to you under the Apache License,
5    * version 2.0 (the "License"); you may not use this file except in compliance
6    * with the License. You may obtain a copy of the License at:
7    *
8    *   https://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13   * License for the specific language governing permissions and limitations
14   * under the License.
15   */
16  package io.netty5.handler.codec.compression;
17  
18  import io.netty5.buffer.api.Buffer;
19  import io.netty5.buffer.api.BufferAllocator;
20  
21  import java.nio.ByteBuffer;
22  import java.util.function.Supplier;
23  import java.util.zip.CRC32;
24  import java.util.zip.DataFormatException;
25  import java.util.zip.Deflater;
26  import java.util.zip.Inflater;
27  
28  import static java.util.Objects.requireNonNull;
29  
30  /**
31   * Decompress a {@link Buffer} using the inflate algorithm.
32   */
33  public final class ZlibDecompressor implements Decompressor {
34      private static final int FHCRC = 0x02;
35      private static final int FEXTRA = 0x04;
36      private static final int FNAME = 0x08;
37      private static final int FCOMMENT = 0x10;
38      private static final int FRESERVED = 0xE0;
39  
40      private Inflater inflater;
41      private final byte[] dictionary;
42  
43      // GZIP related
44      private final BufferChecksum crc;
45      private final boolean decompressConcatenated;
46  
47      /**
48       * Maximum allowed size of the decompression buffer.
49       */
50      private final int maxAllocation;
51  
52      private enum GzipState {
53          HEADER_START,
54          HEADER_END,
55          FLG_READ,
56          XLEN_READ,
57          SKIP_FNAME,
58          SKIP_COMMENT,
59          PROCESS_FHCRC,
60          FOOTER_START,
61      }
62  
63      private GzipState gzipState = GzipState.HEADER_START;
64      private int flags = -1;
65      private int xlen = -1;
66  
67      private boolean finished;
68      private boolean closed;
69  
70      private boolean decideZlibOrNone;
71  
72      private ZlibDecompressor(ZlibWrapper wrapper, byte[] dictionary, boolean decompressConcatenated,
73                               int maxAllocation) {
74          this.maxAllocation = maxAllocation;
75          this.decompressConcatenated = decompressConcatenated;
76          switch (wrapper) {
77              case GZIP:
78                  inflater = new Inflater(true);
79                  crc = new BufferChecksum(new CRC32());
80                  break;
81              case NONE:
82                  inflater = new Inflater(true);
83                  crc = null;
84                  break;
85              case ZLIB:
86                  inflater = new Inflater();
87                  crc = null;
88                  break;
89              case ZLIB_OR_NONE:
90                  // Postpone the decision until decode(...) is called.
91                  decideZlibOrNone = true;
92                  crc = null;
93                  break;
94              default:
95                  throw new IllegalArgumentException("Only GZIP or ZLIB is supported, but you used " + wrapper);
96          }
97          this.dictionary = dictionary;
98      }
99  
100     /**
101      * Creates a zlib decompressor factory with the default wrapper ({@link ZlibWrapper#ZLIB}).
102      *
103      * @return the factory.
104      */
105     public static Supplier<ZlibDecompressor> newFactory() {
106         return newFactory(ZlibWrapper.ZLIB, null, false, 0);
107     }
108 
109     /**
110      * Creates a zlib decompressor factory with the default wrapper ({@link ZlibWrapper#ZLIB})
111      * and the specified maximum buffer allocation.
112      *
113      * @param maxAllocation
114      *          Maximum size of the decompression buffer. Must be &gt;= 0.
115      *          If zero, maximum size is decided by the {@link BufferAllocator}.
116      * @return the factory.
117      */
118     public static Supplier<ZlibDecompressor> newFactory(int maxAllocation) {
119         return newFactory(ZlibWrapper.ZLIB, null, false, maxAllocation);
120     }
121 
122     /**
123      * Creates a zlib decompressor factory with the specified preset dictionary. The wrapper
124      * is always {@link ZlibWrapper#ZLIB} because it is the only format that
125      * supports the preset dictionary.
126      *
127      * @return the factory.
128      */
129     public static Supplier<ZlibDecompressor> newFactory(byte[] dictionary) {
130         return newFactory(ZlibWrapper.ZLIB, dictionary, false, 0);
131     }
132 
133     /**
134      * Creates zlib decompressor factory with the specified preset dictionary and maximum buffer allocation.
135      * The wrapper is always {@link ZlibWrapper#ZLIB} because it is the only format that
136      * supports the preset dictionary.
137      *
138      * @param maxAllocation
139      *          Maximum size of the decompression buffer. Must be &gt;= 0.
140      *          If zero, maximum size is decided by the {@link BufferAllocator}.
141      * @return the factory.
142      */
143     public static Supplier<ZlibDecompressor> newFactory(byte[] dictionary, int maxAllocation) {
144         return newFactory(ZlibWrapper.ZLIB, dictionary, false, maxAllocation);
145     }
146 
147     /**
148      * Creates zlib decompressor factory with the specified wrapper.
149      * Be aware that only {@link ZlibWrapper#GZIP}, {@link ZlibWrapper#ZLIB} and {@link ZlibWrapper#NONE} are
150      * supported atm.
151      *
152      * @return the factory.
153      */
154     public static Supplier<ZlibDecompressor> newFactory(ZlibWrapper wrapper) {
155         return newFactory(wrapper, null, false, 0);
156     }
157 
158     /**
159      * Creates zlib decompressor factory with the specified wrapper and maximum buffer allocation.
160      * Be aware that only {@link ZlibWrapper#GZIP}, {@link ZlibWrapper#ZLIB} and {@link ZlibWrapper#NONE} are
161      * supported atm.
162      *
163      * @param maxAllocation
164      *          Maximum size of the decompression buffer. Must be &gt;= 0.
165      *          If zero, maximum size is decided by the {@link BufferAllocator}.
166      * @return the factory.
167      */
168     public static Supplier<ZlibDecompressor> newFactory(ZlibWrapper wrapper, int maxAllocation) {
169         return newFactory(wrapper, null, false, maxAllocation);
170     }
171 
172     public static Supplier<ZlibDecompressor> newFactory(ZlibWrapper wrapper, boolean decompressConcatenated) {
173         return newFactory(wrapper, null, decompressConcatenated, 0);
174     }
175 
176     public static Supplier<ZlibDecompressor> newFactory(
177             ZlibWrapper wrapper, boolean decompressConcatenated, int maxAllocation) {
178         return newFactory(wrapper, null, decompressConcatenated, maxAllocation);
179     }
180 
181     public static Supplier<ZlibDecompressor> newFactory(boolean decompressConcatenated) {
182         return newFactory(ZlibWrapper.GZIP, null, decompressConcatenated, 0);
183     }
184 
185     public static Supplier<ZlibDecompressor> newFactory(boolean decompressConcatenated, int maxAllocation) {
186         return newFactory(ZlibWrapper.GZIP, null, decompressConcatenated, maxAllocation);
187     }
188 
189     private static Supplier<ZlibDecompressor> newFactory(ZlibWrapper wrapper, byte[] dictionary,
190                                                          boolean decompressConcatenated, int maxAllocation) {
191         requireNonNull(wrapper, "wrapper");
192         return () -> new ZlibDecompressor(wrapper, dictionary, decompressConcatenated, maxAllocation);
193     }
194 
195     @Override
196     public Buffer decompress(Buffer in, BufferAllocator allocator) throws DecompressionException {
197         if (closed) {
198             throw new DecompressionException("Decompressor closed");
199         }
200         if (finished) {
201             return allocator.allocate(0);
202         }
203 
204         int readableBytes = in.readableBytes();
205         if (readableBytes == 0) {
206             return null;
207         }
208 
209         if (decideZlibOrNone) {
210             // First two bytes are needed to decide if it's a ZLIB stream.
211             if (readableBytes < 2) {
212                 return null;
213             }
214 
215             boolean nowrap = !looksLikeZlib(in.getShort(in.readerOffset()));
216             inflater = new Inflater(nowrap);
217             decideZlibOrNone = false;
218         }
219 
220         if (crc != null) {
221             if (gzipState != GzipState.HEADER_END) {
222                 if (gzipState == GzipState.FOOTER_START) {
223                     if (!handleGzipFooter(in)) {
224                         // Either there was not enough data or the input is finished.
225                         return null;
226                     }
227                     // If we consumed the footer we will start with the header again.
228                     assert gzipState == GzipState.HEADER_START;
229                 }
230                 if (!readGZIPHeader(in)) {
231                     // There was not enough data readable to read the GZIP header.
232                     return null;
233                 }
234                 // Some bytes may have been consumed, and so we must re-set the number of readable bytes.
235                 readableBytes = in.readableBytes();
236                 if (readableBytes == 0) {
237                     return null;
238                 }
239             }
240         }
241 
242         if (inflater.needsInput()) {
243             try (var readableIteration = in.forEachReadable()) {
244                 var readableComponent = readableIteration.first();
245                 if (readableComponent.hasReadableArray()) {
246                     inflater.setInput(readableComponent.readableArray(),
247                             readableComponent.readableArrayOffset(), readableComponent.readableBytes());
248                 } else {
249                     inflater.setInput(readableComponent.readableBuffer());
250                 }
251             }
252         }
253 
254         Buffer decompressed = prepareDecompressBuffer(allocator, null, inflater.getRemaining() << 1);
255         try {
256             boolean readFooter = false;
257             while (!inflater.needsInput()) {
258                 int writableComponents = decompressed.countWritableComponents();
259                 if (writableComponents == 0) {
260                     break;
261                 } else if (writableComponents > 1) {
262                     throw new IllegalStateException(
263                             "Decompress buffer must have array or exactly 1 NIO buffer: " + decompressed);
264                 }
265                 try (var writableIteration = decompressed.forEachWritable()) {
266                     var writableComponent = writableIteration.first();
267                     int writerIndex = decompressed.writerOffset();
268                     int writable = decompressed.writableBytes();
269                     int outputLength;
270                     if (writableComponent.hasWritableArray()) {
271                         byte[] outArray = writableComponent.writableArray();
272                         int outIndex = writableComponent.writableArrayOffset();
273                         outputLength = inflater.inflate(outArray, outIndex, writable);
274                     } else {
275                         ByteBuffer buffer = writableComponent.writableBuffer();
276                         outputLength = inflater.inflate(buffer);
277                     }
278                     if (outputLength > 0) {
279                         writableComponent.skipWritableBytes(outputLength);
280                         if (crc != null) {
281                             crc.update(decompressed, writerIndex, outputLength);
282                         }
283                     } else if (inflater.needsDictionary()) {
284                         if (dictionary == null) {
285                             throw new DecompressionException(
286                                     "decompression failure, unable to set dictionary as non was specified");
287                         }
288                         inflater.setDictionary(dictionary);
289                     }
290 
291                     if (inflater.finished()) {
292                         if (crc == null) {
293                             finished = true; // Do not decode anymore.
294                         } else {
295                             readFooter = true;
296                         }
297                         break;
298                     }
299                 }
300                 decompressed = prepareDecompressBuffer(allocator, decompressed, inflater.getRemaining() << 1);
301             }
302 
303             int remaining = inflater.getRemaining();
304             in.skipReadableBytes(readableBytes - remaining);
305 
306             if (readFooter) {
307                 gzipState = GzipState.FOOTER_START;
308                 handleGzipFooter(in);
309             }
310 
311             if (decompressed.readableBytes() > 0) {
312                 return decompressed;
313             } else {
314                 decompressed.close();
315                 return null;
316             }
317         } catch (DataFormatException e) {
318             decompressed.close();
319             throw new DecompressionException("decompression failure", e);
320         } catch (Throwable cause) {
321             decompressed.close();
322             throw cause;
323         }
324     }
325 
326     private boolean handleGzipFooter(Buffer in) {
327         if (readGZIPFooter(in)) {
328             finished = !decompressConcatenated;
329 
330             if (!finished) {
331                 inflater.reset();
332                 crc.reset();
333                 gzipState = GzipState.HEADER_START;
334                 return true;
335             }
336         }
337         return false;
338     }
339 
340     private boolean readGZIPHeader(Buffer in) {
341         switch (gzipState) {
342             case HEADER_START:
343                 if (in.readableBytes() < 10) {
344                     return false;
345                 }
346                 // read magic numbers
347                 int magic0 = in.readByte();
348                 int magic1 = in.readByte();
349 
350                 if (magic0 != 31) {
351                     throw new DecompressionException("Input is not in the GZIP format");
352                 }
353                 crc.update(magic0);
354                 crc.update(magic1);
355 
356                 int method = in.readUnsignedByte();
357                 if (method != Deflater.DEFLATED) {
358                     throw new DecompressionException("Unsupported compression method "
359                             + method + " in the GZIP header");
360                 }
361                 crc.update(method);
362 
363                 flags = in.readUnsignedByte();
364                 crc.update(flags);
365 
366                 if ((flags & FRESERVED) != 0) {
367                     throw new DecompressionException(
368                             "Reserved flags are set in the GZIP header");
369                 }
370 
371                 // mtime (int)
372                 crc.update(in, in.readerOffset(), 4);
373                 in.skipReadableBytes(4);
374 
375                 crc.update(in.readUnsignedByte()); // extra flags
376                 crc.update(in.readUnsignedByte()); // operating system
377 
378                 gzipState = GzipState.FLG_READ;
379                 // fall through
380             case FLG_READ:
381                 if ((flags & FEXTRA) != 0) {
382                     if (in.readableBytes() < 2) {
383                         return false;
384                     }
385                     int xlen1 = in.readUnsignedByte();
386                     int xlen2 = in.readUnsignedByte();
387                     crc.update(xlen1);
388                     crc.update(xlen2);
389 
390                     xlen |= xlen1 << 8 | xlen2;
391                 }
392                 gzipState = GzipState.XLEN_READ;
393                 // fall through
394             case XLEN_READ:
395                 if (xlen != -1) {
396                     if (in.readableBytes() < xlen) {
397                         return false;
398                     }
399                     crc.update(in, in.readerOffset(), xlen);
400                     in.skipReadableBytes(xlen);
401                 }
402                 gzipState = GzipState.SKIP_FNAME;
403                 // fall through
404             case SKIP_FNAME:
405                 if (!skipIfNeeded(in, FNAME)) {
406                     return false;
407                 }
408                 gzipState = GzipState.SKIP_COMMENT;
409                 // fall through
410             case SKIP_COMMENT:
411                 if (!skipIfNeeded(in, FCOMMENT)) {
412                     return false;
413                 }
414                 gzipState = GzipState.PROCESS_FHCRC;
415                 // fall through
416             case PROCESS_FHCRC:
417                 if ((flags & FHCRC) != 0) {
418                     if (!verifyCrc16(in)) {
419                         return false;
420                     }
421                 }
422                 crc.reset();
423                 gzipState = GzipState.HEADER_END;
424                 // fall through
425             case HEADER_END:
426                 return true;
427             default:
428                 throw new IllegalStateException();
429         }
430     }
431 
432     /**
433      * Skip bytes in the input if needed until we find the end marker {@code 0x00}.
434      * @param   in the input
435      * @param   flagMask the mask that should be present in the {@code flags} when we need to skip bytes.
436      * @return  {@code true} if the operation is complete and we can move to the next state, {@code false} if
437      * we need to retry again once we have more readable bytes.
438      */
439     private boolean skipIfNeeded(Buffer in, int flagMask) {
440         if ((flags & flagMask) != 0) {
441             for (;;) {
442                 if (in.readableBytes() == 0) {
443                     // We didnt find the end yet, need to retry again once more data is readable
444                     return false;
445                 }
446                 int b = in.readUnsignedByte();
447                 crc.update(b);
448                 if (b == 0x00) {
449                     break;
450                 }
451             }
452         }
453         // Skip is handled, we can move to the next processing state.
454         return true;
455     }
456 
457     /**
458      * Read the GZIP footer.
459      *
460      * @param   in the input.
461      * @return  {@code true} if the footer could be read, {@code false} if the read could not be performed as
462      *          the input {@link Buffer} doesn't have enough readable bytes (8 bytes).
463      */
464     private boolean readGZIPFooter(Buffer in) {
465         if (in.readableBytes() < 8) {
466             return false;
467         }
468 
469         boolean enoughData = verifyCrc(in);
470         assert enoughData;
471 
472         // read ISIZE and verify
473         int dataLength = 0;
474         for (int i = 0; i < 4; ++i) {
475             dataLength |= in.readUnsignedByte() << i * 8;
476         }
477         int readLength = inflater.getTotalOut();
478         if (dataLength != readLength) {
479             throw new DecompressionException(
480                     "Number of bytes mismatch. Expected: " + dataLength + ", Got: " + readLength);
481         }
482         return true;
483     }
484 
485     /**
486      * Verifies CRC.
487      *
488      * @param   in the input.
489      * @return  {@code true} if verification could be performed, {@code false} if verification could not be
490      * performed as the input {@link Buffer} doesn't have enough readable bytes (4 bytes).
491      */
492     private boolean verifyCrc(Buffer in) {
493         if (in.readableBytes() < 4) {
494             return false;
495         }
496         long crcValue = 0;
497         for (int i = 0; i < 4; ++i) {
498             crcValue |= (long) in.readUnsignedByte() << i * 8;
499         }
500         long readCrc = crc.getValue();
501         if (crcValue != readCrc) {
502             throw new DecompressionException(
503                     "CRC value mismatch. Expected: " + crcValue + ", Got: " + readCrc);
504         }
505         return true;
506     }
507 
508     private boolean verifyCrc16(Buffer in) {
509         if (in.readableBytes() < 2) {
510             return false;
511         }
512         long readCrc32 = crc.getValue();
513         long crc16Value = 0;
514         long readCrc16 = 0; // the two least significant bytes from the CRC32
515         for (int i = 0; i < 2; ++i) {
516             crc16Value |= (long) in.readUnsignedByte() << (i * 8);
517             readCrc16 |= ((readCrc32 >> (i * 8)) & 0xff) << (i * 8);
518         }
519 
520         if (crc16Value != readCrc16) {
521             throw new DecompressionException(
522                     "CRC16 value mismatch. Expected: " + crc16Value + ", Got: " + readCrc16);
523         }
524         return true;
525     }
526 
527     /*
528      * Returns true if the cmf_flg parameter (think: first two bytes of a zlib stream)
529      * indicates that this is a zlib stream.
530      * <p>
531      * You can lookup the details in the ZLIB RFC:
532      * <a href="https://tools.ietf.org/html/rfc1950#section-2.2">RFC 1950</a>.
533      */
534     private static boolean looksLikeZlib(short cmf_flg) {
535         return (cmf_flg & 0x7800) == 0x7800 &&
536                 cmf_flg % 31 == 0;
537     }
538 
539     /**
540      * Allocate or expand the decompression buffer, without exceeding the maximum allocation.
541      * Calls {@link #decompressionBufferExhausted(Buffer)} if the buffer is full and cannot be expanded further.
542      */
543     protected Buffer prepareDecompressBuffer(BufferAllocator allocator, Buffer buffer, int preferredSize) {
544         if (buffer == null) {
545             if (maxAllocation == 0) {
546                 return allocator.allocate(preferredSize);
547             }
548 
549             Buffer buf = allocator.allocate(Math.min(preferredSize, maxAllocation));
550             buf.implicitCapacityLimit(maxAllocation);
551             return buf;
552         }
553 
554         if (buffer.implicitCapacityLimit() < preferredSize) {
555             decompressionBufferExhausted(buffer);
556             buffer.skipReadableBytes(buffer.readableBytes());
557             throw new DecompressionException(
558                     "Decompression buffer has reached maximum size: " + buffer.implicitCapacityLimit());
559         }
560         buffer.ensureWritable(preferredSize);
561         return buffer;
562     }
563 
564     protected void decompressionBufferExhausted(Buffer buffer) {
565         finished = true;
566     }
567 
568     @Override
569     public boolean isFinished() {
570         return finished;
571     }
572 
573     @Override
574     public void close() {
575         closed = true;
576         finished = true;
577         if (inflater != null) {
578             inflater.end();
579         }
580     }
581 
582     @Override
583     public boolean isClosed() {
584         return closed;
585     }
586 }