View Javadoc
1   /*
2    * Copyright 2013 The Netty Project
3    *
4    * The Netty Project licenses this file to you under the Apache License,
5    * version 2.0 (the "License"); you may not use this file except in compliance
6    * with the License. You may obtain a copy of the License at:
7    *
8    *   https://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13   * License for the specific language governing permissions and limitations
14   * under the License.
15   */
16  package io.netty.handler.codec.compression;
17  
18  import io.netty.buffer.ByteBuf;
19  import io.netty.buffer.ByteBufAllocator;
20  import io.netty.channel.ChannelHandlerContext;
21  import io.netty.util.internal.ObjectUtil;
22  
23  import java.util.List;
24  import java.util.zip.CRC32;
25  import java.util.zip.DataFormatException;
26  import java.util.zip.Deflater;
27  import java.util.zip.Inflater;
28  
29  /**
30   * Decompress a {@link ByteBuf} using the inflate algorithm.
31   */
32  public class JdkZlibDecoder extends ZlibDecoder {
33      private static final int FHCRC = 0x02;
34      private static final int FEXTRA = 0x04;
35      private static final int FNAME = 0x08;
36      private static final int FCOMMENT = 0x10;
37      private static final int FRESERVED = 0xE0;
38  
39      private Inflater inflater;
40      private final byte[] dictionary;
41  
42      // GZIP related
43      private final ByteBufChecksum crc;
44      private final boolean decompressConcatenated;
45  
46      private enum GzipState {
47          HEADER_START,
48          HEADER_END,
49          FLG_READ,
50          XLEN_READ,
51          SKIP_FNAME,
52          SKIP_COMMENT,
53          PROCESS_FHCRC,
54          FOOTER_START,
55      }
56  
57      private GzipState gzipState = GzipState.HEADER_START;
58      private int flags = -1;
59      private int xlen = -1;
60      private boolean needsRead;
61  
62      private static final int DEFAULT_MAX_FORWARD_BYTES = CompressionUtil.DEFAULT_MAX_FORWARD_BYTES;
63      private final int maxForwardBytes;
64  
65      private volatile boolean finished;
66  
67      private boolean decideZlibOrNone;
68  
69      /**
70       * Creates a new instance with the default wrapper ({@link ZlibWrapper#ZLIB}).
71       *
72       * @deprecated Use {@link JdkZlibDecoder#JdkZlibDecoder(int)}.
73       */
74      @Deprecated
75      public JdkZlibDecoder() {
76          this(ZlibWrapper.ZLIB, null, false, 0);
77      }
78  
79      /**
80       * Creates a new instance with the default wrapper ({@link ZlibWrapper#ZLIB})
81       * and the specified maximum buffer allocation.
82       *
83       * @param maxAllocation
84       *          Maximum size of the decompression buffer. Must be >= 0.
85       *          If zero, maximum size is decided by the {@link ByteBufAllocator}.
86       */
87      public JdkZlibDecoder(int maxAllocation) {
88          this(ZlibWrapper.ZLIB, null, false, maxAllocation);
89      }
90  
91      /**
92       * Creates a new instance with the specified preset dictionary. The wrapper
93       * is always {@link ZlibWrapper#ZLIB} because it is the only format that
94       * supports the preset dictionary.
95       *
96       * @deprecated Use {@link JdkZlibDecoder#JdkZlibDecoder(byte[], int)}.
97       */
98      @Deprecated
99      public JdkZlibDecoder(byte[] dictionary) {
100         this(ZlibWrapper.ZLIB, dictionary, false, 0);
101     }
102 
103     /**
104      * Creates a new instance with the specified preset dictionary and maximum buffer allocation.
105      * The wrapper is always {@link ZlibWrapper#ZLIB} because it is the only format that
106      * supports the preset dictionary.
107      *
108      * @param maxAllocation
109      *          Maximum size of the decompression buffer. Must be >= 0.
110      *          If zero, maximum size is decided by the {@link ByteBufAllocator}.
111      */
112     public JdkZlibDecoder(byte[] dictionary, int maxAllocation) {
113         this(ZlibWrapper.ZLIB, dictionary, false, maxAllocation);
114     }
115 
116     /**
117      * Creates a new instance with the specified wrapper.
118      * Be aware that only {@link ZlibWrapper#GZIP}, {@link ZlibWrapper#ZLIB} and {@link ZlibWrapper#NONE} are
119      * supported atm.
120      *
121      * @deprecated Use {@link JdkZlibDecoder#JdkZlibDecoder(ZlibWrapper, int)}.
122      */
123     @Deprecated
124     public JdkZlibDecoder(ZlibWrapper wrapper) {
125         this(wrapper, null, false, 0);
126     }
127 
128     /**
129      * Creates a new instance with the specified wrapper and maximum buffer allocation.
130      * Be aware that only {@link ZlibWrapper#GZIP}, {@link ZlibWrapper#ZLIB} and {@link ZlibWrapper#NONE} are
131      * supported atm.
132      *
133      * @param maxAllocation
134      *          Maximum size of the decompression buffer. Must be >= 0.
135      *          If zero, maximum size is decided by the {@link ByteBufAllocator}.
136      */
137     public JdkZlibDecoder(ZlibWrapper wrapper, int maxAllocation) {
138         this(wrapper, null, false, maxAllocation);
139     }
140 
141     /**
142      * @deprecated Use {@link JdkZlibDecoder#JdkZlibDecoder(ZlibWrapper, boolean, int)}.
143      */
144     @Deprecated
145     public JdkZlibDecoder(ZlibWrapper wrapper, boolean decompressConcatenated) {
146         this(wrapper, null, decompressConcatenated, 0);
147     }
148 
149     public JdkZlibDecoder(ZlibWrapper wrapper, boolean decompressConcatenated, int maxAllocation) {
150         this(wrapper, null, decompressConcatenated, maxAllocation);
151     }
152 
153     /**
154      * @deprecated Use {@link JdkZlibDecoder#JdkZlibDecoder(boolean, int)}.
155      */
156     @Deprecated
157     public JdkZlibDecoder(boolean decompressConcatenated) {
158         this(ZlibWrapper.GZIP, null, decompressConcatenated, 0);
159     }
160 
161     public JdkZlibDecoder(boolean decompressConcatenated, int maxAllocation) {
162         this(ZlibWrapper.GZIP, null, decompressConcatenated, maxAllocation);
163     }
164 
165     private JdkZlibDecoder(ZlibWrapper wrapper, byte[] dictionary, boolean decompressConcatenated, int maxAllocation) {
166         super(maxAllocation);
167         this.maxForwardBytes = maxAllocation > 0 ? maxAllocation : DEFAULT_MAX_FORWARD_BYTES;
168 
169         ObjectUtil.checkNotNull(wrapper, "wrapper");
170 
171         this.decompressConcatenated = decompressConcatenated;
172         switch (wrapper) {
173             case GZIP:
174                 inflater = new Inflater(true);
175                 crc = ByteBufChecksum.wrapChecksum(new CRC32());
176                 break;
177             case NONE:
178                 inflater = new Inflater(true);
179                 crc = null;
180                 break;
181             case ZLIB:
182                 inflater = new Inflater();
183                 crc = null;
184                 break;
185             case ZLIB_OR_NONE:
186                 // Postpone the decision until decode(...) is called.
187                 decideZlibOrNone = true;
188                 crc = null;
189                 break;
190             default:
191                 throw new IllegalArgumentException("Only GZIP or ZLIB is supported, but you used " + wrapper);
192         }
193         this.dictionary = dictionary;
194     }
195 
196     @Override
197     public boolean isClosed() {
198         return finished;
199     }
200 
201     @Override
202     protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception {
203         needsRead = true;
204         if (finished) {
205             // Skip data received after finished.
206             in.skipBytes(in.readableBytes());
207             return;
208         }
209 
210         int readableBytes = in.readableBytes();
211         if (readableBytes == 0) {
212             return;
213         }
214 
215         if (decideZlibOrNone) {
216             // First two bytes are needed to decide if it's a ZLIB stream.
217             if (readableBytes < 2) {
218                 return;
219             }
220 
221             boolean nowrap = !looksLikeZlib(in.getShort(in.readerIndex()));
222             inflater = new Inflater(nowrap);
223             decideZlibOrNone = false;
224         }
225 
226         if (crc != null) {
227             if (gzipState != GzipState.HEADER_END) {
228                 if (gzipState == GzipState.FOOTER_START) {
229                     if (!handleGzipFooter(in)) {
230                         // Either there was not enough data or the input is finished.
231                         return;
232                     }
233                     // If we consumed the footer we will start with the header again.
234                     assert gzipState == GzipState.HEADER_START;
235                 }
236                 if (!readGZIPHeader(in)) {
237                     // There was not enough data readable to read the GZIP header.
238                     return;
239                 }
240                 // Some bytes may have been consumed, and so we must re-set the number of readable bytes.
241                 readableBytes = in.readableBytes();
242                 if (readableBytes == 0) {
243                     return;
244                 }
245             }
246         }
247 
248         if (inflater.needsInput()) {
249             if (in.hasArray()) {
250                 inflater.setInput(in.array(), in.arrayOffset() + in.readerIndex(), readableBytes);
251             } else {
252                 byte[] array = new byte[readableBytes];
253                 in.getBytes(in.readerIndex(), array);
254                 inflater.setInput(array);
255             }
256         }
257 
258         ByteBuf decompressed = prepareDecompressBuffer(ctx, null, inflater.getRemaining() << 1);
259         try {
260             boolean readFooter = false;
261             while (!inflater.needsInput()) {
262                 byte[] outArray = decompressed.array();
263                 int writerIndex = decompressed.writerIndex();
264                 int outIndex = decompressed.arrayOffset() + writerIndex;
265                 int writable = decompressed.writableBytes();
266                 int outputLength = inflater.inflate(outArray, outIndex, writable);
267                 if (outputLength > 0) {
268                     decompressed.writerIndex(writerIndex + outputLength);
269                     if (crc != null) {
270                         crc.update(outArray, outIndex, outputLength);
271                     }
272                     if (maxAllocation == 0 && decompressed.readableBytes() >= maxForwardBytes) {
273                         // Forward the buffer once it exceeds the threshold to bound memory
274                         // while avoiding excessive fireChannelRead calls.
275                         ByteBuf buffer = decompressed;
276                         decompressed = null;
277                         needsRead = false;
278                         ctx.fireChannelRead(buffer);
279                     }
280                 } else if (inflater.needsDictionary()) {
281                     if (dictionary == null) {
282                         throw new DecompressionException(
283                                 "decompression failure, unable to set dictionary as non was specified");
284                     }
285                     inflater.setDictionary(dictionary);
286                 }
287 
288                 if (inflater.finished()) {
289                     if (crc == null) {
290                         finished = true; // Do not decode anymore.
291                     } else {
292                         readFooter = true;
293                     }
294                     break;
295                 } else {
296                     decompressed = prepareDecompressBuffer(ctx, decompressed, inflater.getRemaining() << 1);
297                 }
298             }
299 
300             in.skipBytes(readableBytes - inflater.getRemaining());
301 
302             if (readFooter) {
303                 gzipState = GzipState.FOOTER_START;
304                 handleGzipFooter(in);
305             }
306         } catch (DataFormatException e) {
307             throw new DecompressionException("decompression failure", e);
308         } finally {
309             if (decompressed != null) {
310                 if (decompressed.isReadable()) {
311                     needsRead = false;
312                     ctx.fireChannelRead(decompressed);
313                 } else {
314                     decompressed.release();
315                 }
316             }
317         }
318     }
319 
320     private boolean handleGzipFooter(ByteBuf in) {
321         if (readGZIPFooter(in)) {
322             finished = !decompressConcatenated;
323 
324             if (!finished) {
325                 inflater.reset();
326                 crc.reset();
327                 gzipState = GzipState.HEADER_START;
328                 return true;
329             }
330         }
331         return false;
332     }
333 
334     @Override
335     protected void decompressionBufferExhausted(ByteBuf buffer) {
336         finished = true;
337     }
338 
339     @Override
340     protected void handlerRemoved0(ChannelHandlerContext ctx) throws Exception {
341         super.handlerRemoved0(ctx);
342         if (inflater != null) {
343             inflater.end();
344         }
345     }
346 
347     private boolean readGZIPHeader(ByteBuf in) {
348         switch (gzipState) {
349             case HEADER_START:
350                 if (in.readableBytes() < 10) {
351                     return false;
352                 }
353                 // read magic numbers
354                 int magic0 = in.readByte();
355                 int magic1 = in.readByte();
356 
357                 if (magic0 != 31) {
358                     throw new DecompressionException("Input is not in the GZIP format");
359                 }
360                 crc.update(magic0);
361                 crc.update(magic1);
362 
363                 int method = in.readUnsignedByte();
364                 if (method != Deflater.DEFLATED) {
365                     throw new DecompressionException("Unsupported compression method "
366                             + method + " in the GZIP header");
367                 }
368                 crc.update(method);
369 
370                 flags = in.readUnsignedByte();
371                 crc.update(flags);
372 
373                 if ((flags & FRESERVED) != 0) {
374                     throw new DecompressionException(
375                             "Reserved flags are set in the GZIP header");
376                 }
377 
378                 // mtime (int)
379                 crc.update(in, in.readerIndex(), 4);
380                 in.skipBytes(4);
381 
382                 crc.update(in.readUnsignedByte()); // extra flags
383                 crc.update(in.readUnsignedByte()); // operating system
384 
385                 gzipState = GzipState.FLG_READ;
386                 // fall through
387             case FLG_READ:
388                 if ((flags & FEXTRA) != 0) {
389                     if (in.readableBytes() < 2) {
390                         return false;
391                     }
392                     int xlen1 = in.readUnsignedByte();
393                     int xlen2 = in.readUnsignedByte();
394                     crc.update(xlen1);
395                     crc.update(xlen2);
396 
397                     xlen |= xlen1 << 8 | xlen2;
398                 }
399                 gzipState = GzipState.XLEN_READ;
400                 // fall through
401             case XLEN_READ:
402                 if (xlen != -1) {
403                     if (in.readableBytes() < xlen) {
404                         return false;
405                     }
406                     crc.update(in, in.readerIndex(), xlen);
407                     in.skipBytes(xlen);
408                 }
409                 gzipState = GzipState.SKIP_FNAME;
410                 // fall through
411             case SKIP_FNAME:
412                 if (!skipIfNeeded(in, FNAME)) {
413                     return false;
414                 }
415                 gzipState = GzipState.SKIP_COMMENT;
416                 // fall through
417             case SKIP_COMMENT:
418                 if (!skipIfNeeded(in, FCOMMENT)) {
419                     return false;
420                 }
421                 gzipState = GzipState.PROCESS_FHCRC;
422                 // fall through
423             case PROCESS_FHCRC:
424                 if ((flags & FHCRC) != 0) {
425                     if (!verifyCrc16(in)) {
426                         return false;
427                     }
428                 }
429                 crc.reset();
430                 gzipState = GzipState.HEADER_END;
431                 // fall through
432             case HEADER_END:
433                 return true;
434             default:
435                 throw new IllegalStateException();
436         }
437     }
438 
439     /**
440      * Skip bytes in the input if needed until we find the end marker {@code 0x00}.
441      * @param   in the input
442      * @param   flagMask the mask that should be present in the {@code flags} when we need to skip bytes.
443      * @return  {@code true} if the operation is complete and we can move to the next state, {@code false} if we need
444      *          the retry again once we have more readable bytes.
445      */
446     private boolean skipIfNeeded(ByteBuf in, int flagMask) {
447         if ((flags & flagMask) != 0) {
448             for (;;) {
449                 if (!in.isReadable()) {
450                     // We didnt find the end yet, need to retry again once more data is readable
451                     return false;
452                 }
453                 int b = in.readUnsignedByte();
454                 crc.update(b);
455                 if (b == 0x00) {
456                     break;
457                 }
458             }
459         }
460         // Skip is handled, we can move to the next processing state.
461         return true;
462     }
463 
464     /**
465      * Read the GZIP footer.
466      *
467      * @param   in the input.
468      * @return  {@code true} if the footer could be read, {@code false} if the read could not be performed as
469      *          the input {@link ByteBuf} doesn't have enough readable bytes (8 bytes).
470      */
471     private boolean readGZIPFooter(ByteBuf in) {
472         if (in.readableBytes() < 8) {
473             return false;
474         }
475 
476         boolean enoughData = verifyCrc(in);
477         assert enoughData;
478 
479         // read ISIZE and verify
480         int dataLength = in.readIntLE();
481         int readLength = inflater.getTotalOut();
482         if (dataLength != readLength) {
483             throw new DecompressionException(
484                     "Number of bytes mismatch. Expected: " + dataLength + ", Got: " + readLength);
485         }
486         return true;
487     }
488 
489     /**
490      * Verifies CRC.
491      *
492      * @param   in the input.
493      * @return  {@code true} if verification could be performed, {@code false} if verification could not be performed as
494      *          the input {@link ByteBuf} doesn't have enough readable bytes (4 bytes).
495      */
496     private boolean verifyCrc(ByteBuf in) {
497         if (in.readableBytes() < 4) {
498             return false;
499         }
500         long crcValue = in.readUnsignedIntLE();
501 
502         long readCrc = crc.getValue();
503         if (crcValue != readCrc) {
504             throw new DecompressionException(
505                     "CRC value mismatch. Expected: " + crcValue + ", Got: " + readCrc);
506         }
507         return true;
508     }
509 
510     private boolean verifyCrc16(ByteBuf in) {
511         if (in.readableBytes() < 2) {
512             return false;
513         }
514 
515         int crc16Value = in.readUnsignedShortLE();
516         // the two least significant bytes from the CRC32
517         int readCrc16 = (int) (crc.getValue() & 0xFFFF);
518 
519         if (crc16Value != readCrc16) {
520             throw new DecompressionException(
521                     "CRC16 value mismatch. Expected: " + crc16Value + ", Got: " + readCrc16);
522         }
523         return true;
524     }
525 
526     /*
527      * Returns true if the cmf_flg parameter (think: first two bytes of a zlib stream)
528      * indicates that this is a zlib stream.
529      * <p>
530      * You can lookup the details in the ZLIB RFC:
531      * <a href="https://tools.ietf.org/html/rfc1950#section-2.2">RFC 1950</a>.
532      */
533     private static boolean looksLikeZlib(short cmf_flg) {
534         return (cmf_flg & 0x7800) == 0x7800 &&
535                 cmf_flg % 31 == 0;
536     }
537 
538     @Override
539     public void channelReadComplete(ChannelHandlerContext ctx) throws Exception {
540         // Discard bytes of the cumulation buffer if needed.
541         discardSomeReadBytes();
542 
543         if (needsRead && !ctx.channel().config().isAutoRead()) {
544             ctx.read();
545         }
546         ctx.fireChannelReadComplete();
547     }
548 }