View Javadoc
1   /*
2    * Copyright 2013 The Netty Project
3    *
4    * The Netty Project licenses this file to you under the Apache License,
5    * version 2.0 (the "License"); you may not use this file except in compliance
6    * with the License. You may obtain a copy of the License at:
7    *
8    *   https://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13   * License for the specific language governing permissions and limitations
14   * under the License.
15   */
16  package io.netty.handler.codec.compression;
17  
18  import io.netty.buffer.ByteBuf;
19  import io.netty.buffer.ByteBufAllocator;
20  import io.netty.channel.ChannelHandlerContext;
21  import io.netty.util.internal.ObjectUtil;
22  
23  import java.util.List;
24  import java.util.zip.CRC32;
25  import java.util.zip.DataFormatException;
26  import java.util.zip.Deflater;
27  import java.util.zip.Inflater;
28  
29  /**
30   * Decompress a {@link ByteBuf} using the inflate algorithm.
31   */
32  public class JdkZlibDecoder extends ZlibDecoder {
33      private static final int FHCRC = 0x02;
34      private static final int FEXTRA = 0x04;
35      private static final int FNAME = 0x08;
36      private static final int FCOMMENT = 0x10;
37      private static final int FRESERVED = 0xE0;
38  
39      private Inflater inflater;
40      private final byte[] dictionary;
41  
42      // GZIP related
43      private final ByteBufChecksum crc;
44      private final boolean decompressConcatenated;
45  
46      private enum GzipState {
47          HEADER_START,
48          HEADER_END,
49          FLG_READ,
50          XLEN_READ,
51          SKIP_FNAME,
52          SKIP_COMMENT,
53          PROCESS_FHCRC,
54          FOOTER_START,
55      }
56  
57      private GzipState gzipState = GzipState.HEADER_START;
58      private int flags = -1;
59      private int xlen = -1;
60      private boolean needsRead;
61  
62      private static final int DEFAULT_MAX_FORWARD_BYTES = CompressionUtil.DEFAULT_MAX_FORWARD_BYTES;
63      private final int maxForwardBytes;
64  
65      private volatile boolean finished;
66  
67      private boolean decideZlibOrNone;
68  
69      /**
70       * Creates a new instance with the default wrapper ({@link ZlibWrapper#ZLIB}).
71       *
72       * @deprecated Use {@link JdkZlibDecoder#JdkZlibDecoder(int)}.
73       */
74      @Deprecated
75      public JdkZlibDecoder() {
76          this(ZlibWrapper.ZLIB, null, false, 0);
77      }
78  
79      /**
80       * Creates a new instance with the default wrapper ({@link ZlibWrapper#ZLIB})
81       * and the specified maximum buffer allocation.
82       *
83       * @param maxAllocation
84       *          Maximum size of the decompression buffer. Must be >= 0.
85       *          If zero, maximum size is decided by the {@link ByteBufAllocator}.
86       */
87      public JdkZlibDecoder(int maxAllocation) {
88          this(ZlibWrapper.ZLIB, null, false, maxAllocation);
89      }
90  
91      /**
92       * Creates a new instance with the specified preset dictionary. The wrapper
93       * is always {@link ZlibWrapper#ZLIB} because it is the only format that
94       * supports the preset dictionary.
95       *
96       * @deprecated Use {@link JdkZlibDecoder#JdkZlibDecoder(byte[], int)}.
97       */
98      @Deprecated
99      public JdkZlibDecoder(byte[] dictionary) {
100         this(ZlibWrapper.ZLIB, dictionary, false, 0);
101     }
102 
103     /**
104      * Creates a new instance with the specified preset dictionary and maximum buffer allocation.
105      * The wrapper is always {@link ZlibWrapper#ZLIB} because it is the only format that
106      * supports the preset dictionary.
107      *
108      * @param maxAllocation
109      *          Maximum size of the decompression buffer. Must be >= 0.
110      *          If zero, maximum size is decided by the {@link ByteBufAllocator}.
111      */
112     public JdkZlibDecoder(byte[] dictionary, int maxAllocation) {
113         this(ZlibWrapper.ZLIB, dictionary, false, maxAllocation);
114     }
115 
116     /**
117      * Creates a new instance with the specified wrapper.
118      * Be aware that only {@link ZlibWrapper#GZIP}, {@link ZlibWrapper#ZLIB} and {@link ZlibWrapper#NONE} are
119      * supported atm.
120      *
121      * @deprecated Use {@link JdkZlibDecoder#JdkZlibDecoder(ZlibWrapper, int)}.
122      */
123     @Deprecated
124     public JdkZlibDecoder(ZlibWrapper wrapper) {
125         this(wrapper, null, false, 0);
126     }
127 
128     /**
129      * Creates a new instance with the specified wrapper and maximum buffer allocation.
130      * Be aware that only {@link ZlibWrapper#GZIP}, {@link ZlibWrapper#ZLIB} and {@link ZlibWrapper#NONE} are
131      * supported atm.
132      *
133      * @param maxAllocation
134      *          Maximum size of the decompression buffer. Must be >= 0.
135      *          If zero, maximum size is decided by the {@link ByteBufAllocator}.
136      */
137     public JdkZlibDecoder(ZlibWrapper wrapper, int maxAllocation) {
138         this(wrapper, null, false, maxAllocation);
139     }
140 
141     /**
142      * @deprecated Use {@link JdkZlibDecoder#JdkZlibDecoder(ZlibWrapper, boolean, int)}.
143      */
144     @Deprecated
145     public JdkZlibDecoder(ZlibWrapper wrapper, boolean decompressConcatenated) {
146         this(wrapper, null, decompressConcatenated, 0);
147     }
148 
149     public JdkZlibDecoder(ZlibWrapper wrapper, boolean decompressConcatenated, int maxAllocation) {
150         this(wrapper, null, decompressConcatenated, maxAllocation);
151     }
152 
153     /**
154      * @deprecated Use {@link JdkZlibDecoder#JdkZlibDecoder(boolean, int)}.
155      */
156     @Deprecated
157     public JdkZlibDecoder(boolean decompressConcatenated) {
158         this(ZlibWrapper.GZIP, null, decompressConcatenated, 0);
159     }
160 
161     public JdkZlibDecoder(boolean decompressConcatenated, int maxAllocation) {
162         this(ZlibWrapper.GZIP, null, decompressConcatenated, maxAllocation);
163     }
164 
165     private JdkZlibDecoder(ZlibWrapper wrapper, byte[] dictionary, boolean decompressConcatenated, int maxAllocation) {
166         super(maxAllocation);
167         this.maxForwardBytes = maxAllocation > 0 ? maxAllocation : DEFAULT_MAX_FORWARD_BYTES;
168 
169         ObjectUtil.checkNotNull(wrapper, "wrapper");
170 
171         this.decompressConcatenated = decompressConcatenated;
172         switch (wrapper) {
173             case GZIP:
174                 inflater = new Inflater(true);
175                 crc = ByteBufChecksum.wrapChecksum(new CRC32());
176                 break;
177             case NONE:
178                 inflater = new Inflater(true);
179                 crc = null;
180                 break;
181             case ZLIB:
182                 inflater = new Inflater();
183                 crc = null;
184                 break;
185             case ZLIB_OR_NONE:
186                 // Postpone the decision until decode(...) is called.
187                 decideZlibOrNone = true;
188                 crc = null;
189                 break;
190             default:
191                 throw new IllegalArgumentException("Only GZIP or ZLIB is supported, but you used " + wrapper);
192         }
193         this.dictionary = dictionary;
194     }
195 
196     @Override
197     public boolean isClosed() {
198         return finished;
199     }
200 
201     @Override
202     protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception {
203         needsRead = true;
204         if (finished) {
205             // Skip data received after finished.
206             in.skipBytes(in.readableBytes());
207             return;
208         }
209 
210         int readableBytes = in.readableBytes();
211         if (readableBytes == 0) {
212             return;
213         }
214 
215         if (decideZlibOrNone) {
216             // First two bytes are needed to decide if it's a ZLIB stream.
217             if (readableBytes < 2) {
218                 return;
219             }
220 
221             boolean nowrap = !looksLikeZlib(in.getShort(in.readerIndex()));
222             inflater = new Inflater(nowrap);
223             decideZlibOrNone = false;
224         }
225 
226         if (crc != null) {
227             if (gzipState != GzipState.HEADER_END) {
228                 if (gzipState == GzipState.FOOTER_START) {
229                     if (!handleGzipFooter(in)) {
230                         // Either there was not enough data or the input is finished.
231                         return;
232                     }
233                     // If we consumed the footer we will start with the header again.
234                     assert gzipState == GzipState.HEADER_START;
235                 }
236                 if (!readGZIPHeader(in)) {
237                     // There was not enough data readable to read the GZIP header.
238                     return;
239                 }
240                 // Some bytes may have been consumed, and so we must re-set the number of readable bytes.
241                 readableBytes = in.readableBytes();
242                 if (readableBytes == 0) {
243                     return;
244                 }
245             }
246         }
247 
248         if (inflater.needsInput()) {
249             if (in.hasArray()) {
250                 inflater.setInput(in.array(), in.arrayOffset() + in.readerIndex(), readableBytes);
251             } else {
252                 byte[] array = new byte[readableBytes];
253                 in.getBytes(in.readerIndex(), array);
254                 inflater.setInput(array);
255             }
256         }
257 
258         ByteBuf decompressed = prepareDecompressBuffer(ctx, null, inflater.getRemaining() << 1);
259         try {
260             boolean readFooter = false;
261             while (!inflater.needsInput()) {
262                 byte[] outArray = decompressed.array();
263                 int writerIndex = decompressed.writerIndex();
264                 int outIndex = decompressed.arrayOffset() + writerIndex;
265                 int writable = decompressed.writableBytes();
266                 int outputLength = inflater.inflate(outArray, outIndex, writable);
267                 if (outputLength > 0) {
268                     decompressed.writerIndex(writerIndex + outputLength);
269                     if (crc != null) {
270                         crc.update(outArray, outIndex, outputLength);
271                     }
272                     if (maxAllocation == 0 && decompressed.readableBytes() >= maxForwardBytes) {
273                         // Forward the buffer once it exceeds the threshold to bound memory
274                         // while avoiding excessive fireChannelRead calls.
275                         ByteBuf buffer = decompressed;
276                         decompressed = null;
277                         needsRead = false;
278                         ctx.fireChannelRead(buffer);
279                     }
280                 } else if (inflater.needsDictionary()) {
281                     if (dictionary == null) {
282                         throw new DecompressionException(
283                                 "decompression failure, unable to set dictionary as non was specified");
284                     }
285                     inflater.setDictionary(dictionary);
286                 }
287 
288                 if (inflater.finished()) {
289                     if (crc == null) {
290                         finished = true; // Do not decode anymore.
291                     } else {
292                         readFooter = true;
293                     }
294                     break;
295                 } else {
296                     decompressed = prepareDecompressBuffer(ctx, decompressed, inflater.getRemaining() << 1);
297                 }
298             }
299 
300             in.skipBytes(readableBytes - inflater.getRemaining());
301 
302             if (readFooter) {
303                 gzipState = GzipState.FOOTER_START;
304                 handleGzipFooter(in);
305             }
306         } catch (DataFormatException e) {
307             throw new DecompressionException("decompression failure", e);
308         } finally {
309             if (decompressed != null) {
310                 if (decompressed.isReadable()) {
311                     needsRead = false;
312                     ctx.fireChannelRead(decompressed);
313                 } else {
314                     decompressed.release();
315                 }
316             }
317         }
318     }
319 
320     private boolean handleGzipFooter(ByteBuf in) {
321         if (readGZIPFooter(in)) {
322             finished = !decompressConcatenated;
323 
324             if (!finished) {
325                 inflater.reset();
326                 crc.reset();
327                 xlen = -1;
328                 gzipState = GzipState.HEADER_START;
329                 return true;
330             }
331         }
332         return false;
333     }
334 
335     @Override
336     protected void decompressionBufferExhausted(ByteBuf buffer) {
337         finished = true;
338     }
339 
340     @Override
341     protected void handlerRemoved0(ChannelHandlerContext ctx) throws Exception {
342         super.handlerRemoved0(ctx);
343         if (inflater != null) {
344             inflater.end();
345         }
346     }
347 
348     private boolean readGZIPHeader(ByteBuf in) {
349         switch (gzipState) {
350             case HEADER_START:
351                 if (in.readableBytes() < 10) {
352                     return false;
353                 }
354                 // read magic numbers
355                 int magic0 = in.readByte();
356                 int magic1 = in.readByte();
357 
358                 if (magic0 != 31) {
359                     throw new DecompressionException("Input is not in the GZIP format");
360                 }
361                 crc.update(magic0);
362                 crc.update(magic1);
363 
364                 int method = in.readUnsignedByte();
365                 if (method != Deflater.DEFLATED) {
366                     throw new DecompressionException("Unsupported compression method "
367                             + method + " in the GZIP header");
368                 }
369                 crc.update(method);
370 
371                 flags = in.readUnsignedByte();
372                 crc.update(flags);
373 
374                 if ((flags & FRESERVED) != 0) {
375                     throw new DecompressionException(
376                             "Reserved flags are set in the GZIP header");
377                 }
378 
379                 // mtime (int)
380                 crc.update(in, in.readerIndex(), 4);
381                 in.skipBytes(4);
382 
383                 crc.update(in.readUnsignedByte()); // extra flags
384                 crc.update(in.readUnsignedByte()); // operating system
385 
386                 gzipState = GzipState.FLG_READ;
387                 // fall through
388             case FLG_READ:
389                 if ((flags & FEXTRA) != 0) {
390                     if (in.readableBytes() < 2) {
391                         return false;
392                     }
393                     int xlen1 = in.readUnsignedByte();
394                     int xlen2 = in.readUnsignedByte();
395                     crc.update(xlen1);
396                     crc.update(xlen2);
397 
398                     // XLEN is a little-endian unsigned 16-bit value (RFC 1952), so xlen1 is the
399                     // low byte and xlen2 the high byte. This must be an assignment, not |=: xlen
400                     // starts at the -1 "no extra field" sentinel, and OR-ing into -1 (0xFFFFFFFF)
401                     // would leave it -1, so the extra field was never skipped.
402                     xlen = xlen2 << 8 | xlen1;
403                 }
404                 gzipState = GzipState.XLEN_READ;
405                 // fall through
406             case XLEN_READ:
407                 if (xlen != -1) {
408                     if (in.readableBytes() < xlen) {
409                         return false;
410                     }
411                     crc.update(in, in.readerIndex(), xlen);
412                     in.skipBytes(xlen);
413                 }
414                 gzipState = GzipState.SKIP_FNAME;
415                 // fall through
416             case SKIP_FNAME:
417                 if (!skipIfNeeded(in, FNAME)) {
418                     return false;
419                 }
420                 gzipState = GzipState.SKIP_COMMENT;
421                 // fall through
422             case SKIP_COMMENT:
423                 if (!skipIfNeeded(in, FCOMMENT)) {
424                     return false;
425                 }
426                 gzipState = GzipState.PROCESS_FHCRC;
427                 // fall through
428             case PROCESS_FHCRC:
429                 if ((flags & FHCRC) != 0) {
430                     if (!verifyCrc16(in)) {
431                         return false;
432                     }
433                 }
434                 crc.reset();
435                 gzipState = GzipState.HEADER_END;
436                 // fall through
437             case HEADER_END:
438                 return true;
439             default:
440                 throw new IllegalStateException();
441         }
442     }
443 
444     /**
445      * Skip bytes in the input if needed until we find the end marker {@code 0x00}.
446      * @param   in the input
447      * @param   flagMask the mask that should be present in the {@code flags} when we need to skip bytes.
448      * @return  {@code true} if the operation is complete and we can move to the next state, {@code false} if we need
449      *          the retry again once we have more readable bytes.
450      */
451     private boolean skipIfNeeded(ByteBuf in, int flagMask) {
452         if ((flags & flagMask) != 0) {
453             for (;;) {
454                 if (!in.isReadable()) {
455                     // We didnt find the end yet, need to retry again once more data is readable
456                     return false;
457                 }
458                 int b = in.readUnsignedByte();
459                 crc.update(b);
460                 if (b == 0x00) {
461                     break;
462                 }
463             }
464         }
465         // Skip is handled, we can move to the next processing state.
466         return true;
467     }
468 
469     /**
470      * Read the GZIP footer.
471      *
472      * @param   in the input.
473      * @return  {@code true} if the footer could be read, {@code false} if the read could not be performed as
474      *          the input {@link ByteBuf} doesn't have enough readable bytes (8 bytes).
475      */
476     private boolean readGZIPFooter(ByteBuf in) {
477         if (in.readableBytes() < 8) {
478             return false;
479         }
480 
481         boolean enoughData = verifyCrc(in);
482         assert enoughData;
483 
484         // read ISIZE and verify
485         int dataLength = in.readIntLE();
486         int readLength = inflater.getTotalOut();
487         if (dataLength != readLength) {
488             throw new DecompressionException(
489                     "Number of bytes mismatch. Expected: " + dataLength + ", Got: " + readLength);
490         }
491         return true;
492     }
493 
494     /**
495      * Verifies CRC.
496      *
497      * @param   in the input.
498      * @return  {@code true} if verification could be performed, {@code false} if verification could not be performed as
499      *          the input {@link ByteBuf} doesn't have enough readable bytes (4 bytes).
500      */
501     private boolean verifyCrc(ByteBuf in) {
502         if (in.readableBytes() < 4) {
503             return false;
504         }
505         long crcValue = in.readUnsignedIntLE();
506 
507         long readCrc = crc.getValue();
508         if (crcValue != readCrc) {
509             throw new DecompressionException(
510                     "CRC value mismatch. Expected: " + crcValue + ", Got: " + readCrc);
511         }
512         return true;
513     }
514 
515     private boolean verifyCrc16(ByteBuf in) {
516         if (in.readableBytes() < 2) {
517             return false;
518         }
519 
520         int crc16Value = in.readUnsignedShortLE();
521         // the two least significant bytes from the CRC32
522         int readCrc16 = (int) (crc.getValue() & 0xFFFF);
523 
524         if (crc16Value != readCrc16) {
525             throw new DecompressionException(
526                     "CRC16 value mismatch. Expected: " + crc16Value + ", Got: " + readCrc16);
527         }
528         return true;
529     }
530 
531     /*
532      * Returns true if the cmf_flg parameter (think: first two bytes of a zlib stream)
533      * indicates that this is a zlib stream.
534      * <p>
535      * You can lookup the details in the ZLIB RFC:
536      * <a href="https://tools.ietf.org/html/rfc1950#section-2.2">RFC 1950</a>.
537      */
538     private static boolean looksLikeZlib(short cmf_flg) {
539         return (cmf_flg & 0x7800) == 0x7800 &&
540                 cmf_flg % 31 == 0;
541     }
542 
543     @Override
544     public void channelReadComplete(ChannelHandlerContext ctx) throws Exception {
545         // Discard bytes of the cumulation buffer if needed.
546         discardSomeReadBytes();
547 
548         if (needsRead && !ctx.channel().config().isAutoRead()) {
549             ctx.read();
550         }
551         ctx.fireChannelReadComplete();
552     }
553 }