View Javadoc
1   /*
2    * Copyright 2013 The Netty Project
3    *
4    * The Netty Project licenses this file to you under the Apache License,
5    * version 2.0 (the "License"); you may not use this file except in compliance
6    * with the License. You may obtain a copy of the License at:
7    *
8    *   https://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13   * License for the specific language governing permissions and limitations
14   * under the License.
15   */
16  package io.netty.handler.codec.compression;
17  
18  import io.netty.buffer.ByteBuf;
19  import io.netty.buffer.ByteBufAllocator;
20  import io.netty.channel.ChannelHandlerContext;
21  import io.netty.util.internal.ObjectUtil;
22  
23  import java.util.List;
24  import java.util.zip.CRC32;
25  import java.util.zip.DataFormatException;
26  import java.util.zip.Deflater;
27  import java.util.zip.Inflater;
28  
29  /**
30   * Decompress a {@link ByteBuf} using the inflate algorithm.
31   */
32  public class JdkZlibDecoder extends ZlibDecoder {
33      private static final int FHCRC = 0x02;
34      private static final int FEXTRA = 0x04;
35      private static final int FNAME = 0x08;
36      private static final int FCOMMENT = 0x10;
37      private static final int FRESERVED = 0xE0;
38  
39      private Inflater inflater;
40      private final byte[] dictionary;
41  
42      // GZIP related
43      private final ByteBufChecksum crc;
44      private final boolean decompressConcatenated;
45  
46      private enum GzipState {
47          HEADER_START,
48          HEADER_END,
49          FLG_READ,
50          XLEN_READ,
51          SKIP_FNAME,
52          SKIP_COMMENT,
53          PROCESS_FHCRC,
54          FOOTER_START,
55      }
56  
57      private GzipState gzipState = GzipState.HEADER_START;
58      private int flags = -1;
59      private int xlen = -1;
60  
61      private volatile boolean finished;
62  
63      private boolean decideZlibOrNone;
64  
65      /**
66       * Creates a new instance with the default wrapper ({@link ZlibWrapper#ZLIB}).
67       *
68       * @deprecated Use {@link JdkZlibDecoder#JdkZlibDecoder(int)}.
69       */
70      @Deprecated
71      public JdkZlibDecoder() {
72          this(ZlibWrapper.ZLIB, null, false, 0);
73      }
74  
75      /**
76       * Creates a new instance with the default wrapper ({@link ZlibWrapper#ZLIB})
77       * and the specified maximum buffer allocation.
78       *
79       * @param maxAllocation
80       *          Maximum size of the decompression buffer. Must be >= 0.
81       *          If zero, maximum size is decided by the {@link ByteBufAllocator}.
82       */
83      public JdkZlibDecoder(int maxAllocation) {
84          this(ZlibWrapper.ZLIB, null, false, maxAllocation);
85      }
86  
87      /**
88       * Creates a new instance with the specified preset dictionary. The wrapper
89       * is always {@link ZlibWrapper#ZLIB} because it is the only format that
90       * supports the preset dictionary.
91       *
92       * @deprecated Use {@link JdkZlibDecoder#JdkZlibDecoder(byte[], int)}.
93       */
94      @Deprecated
95      public JdkZlibDecoder(byte[] dictionary) {
96          this(ZlibWrapper.ZLIB, dictionary, false, 0);
97      }
98  
99      /**
100      * Creates a new instance with the specified preset dictionary and maximum buffer allocation.
101      * The wrapper is always {@link ZlibWrapper#ZLIB} because it is the only format that
102      * supports the preset dictionary.
103      *
104      * @param maxAllocation
105      *          Maximum size of the decompression buffer. Must be >= 0.
106      *          If zero, maximum size is decided by the {@link ByteBufAllocator}.
107      */
108     public JdkZlibDecoder(byte[] dictionary, int maxAllocation) {
109         this(ZlibWrapper.ZLIB, dictionary, false, maxAllocation);
110     }
111 
112     /**
113      * Creates a new instance with the specified wrapper.
114      * Be aware that only {@link ZlibWrapper#GZIP}, {@link ZlibWrapper#ZLIB} and {@link ZlibWrapper#NONE} are
115      * supported atm.
116      *
117      * @deprecated Use {@link JdkZlibDecoder#JdkZlibDecoder(ZlibWrapper, int)}.
118      */
119     @Deprecated
120     public JdkZlibDecoder(ZlibWrapper wrapper) {
121         this(wrapper, null, false, 0);
122     }
123 
124     /**
125      * Creates a new instance with the specified wrapper and maximum buffer allocation.
126      * Be aware that only {@link ZlibWrapper#GZIP}, {@link ZlibWrapper#ZLIB} and {@link ZlibWrapper#NONE} are
127      * supported atm.
128      *
129      * @param maxAllocation
130      *          Maximum size of the decompression buffer. Must be >= 0.
131      *          If zero, maximum size is decided by the {@link ByteBufAllocator}.
132      */
133     public JdkZlibDecoder(ZlibWrapper wrapper, int maxAllocation) {
134         this(wrapper, null, false, maxAllocation);
135     }
136 
137     /**
138      * @deprecated Use {@link JdkZlibDecoder#JdkZlibDecoder(ZlibWrapper, boolean, int)}.
139      */
140     @Deprecated
141     public JdkZlibDecoder(ZlibWrapper wrapper, boolean decompressConcatenated) {
142         this(wrapper, null, decompressConcatenated, 0);
143     }
144 
145     public JdkZlibDecoder(ZlibWrapper wrapper, boolean decompressConcatenated, int maxAllocation) {
146         this(wrapper, null, decompressConcatenated, maxAllocation);
147     }
148 
149     /**
150      * @deprecated Use {@link JdkZlibDecoder#JdkZlibDecoder(boolean, int)}.
151      */
152     @Deprecated
153     public JdkZlibDecoder(boolean decompressConcatenated) {
154         this(ZlibWrapper.GZIP, null, decompressConcatenated, 0);
155     }
156 
157     public JdkZlibDecoder(boolean decompressConcatenated, int maxAllocation) {
158         this(ZlibWrapper.GZIP, null, decompressConcatenated, maxAllocation);
159     }
160 
161     private JdkZlibDecoder(ZlibWrapper wrapper, byte[] dictionary, boolean decompressConcatenated, int maxAllocation) {
162         super(maxAllocation);
163 
164         ObjectUtil.checkNotNull(wrapper, "wrapper");
165 
166         this.decompressConcatenated = decompressConcatenated;
167         switch (wrapper) {
168             case GZIP:
169                 inflater = new Inflater(true);
170                 crc = ByteBufChecksum.wrapChecksum(new CRC32());
171                 break;
172             case NONE:
173                 inflater = new Inflater(true);
174                 crc = null;
175                 break;
176             case ZLIB:
177                 inflater = new Inflater();
178                 crc = null;
179                 break;
180             case ZLIB_OR_NONE:
181                 // Postpone the decision until decode(...) is called.
182                 decideZlibOrNone = true;
183                 crc = null;
184                 break;
185             default:
186                 throw new IllegalArgumentException("Only GZIP or ZLIB is supported, but you used " + wrapper);
187         }
188         this.dictionary = dictionary;
189     }
190 
191     @Override
192     public boolean isClosed() {
193         return finished;
194     }
195 
196     @Override
197     protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception {
198         if (finished) {
199             // Skip data received after finished.
200             in.skipBytes(in.readableBytes());
201             return;
202         }
203 
204         int readableBytes = in.readableBytes();
205         if (readableBytes == 0) {
206             return;
207         }
208 
209         if (decideZlibOrNone) {
210             // First two bytes are needed to decide if it's a ZLIB stream.
211             if (readableBytes < 2) {
212                 return;
213             }
214 
215             boolean nowrap = !looksLikeZlib(in.getShort(in.readerIndex()));
216             inflater = new Inflater(nowrap);
217             decideZlibOrNone = false;
218         }
219 
220         if (crc != null) {
221             if (gzipState != GzipState.HEADER_END) {
222                 if (gzipState == GzipState.FOOTER_START) {
223                     if (!handleGzipFooter(in)) {
224                         // Either there was not enough data or the input is finished.
225                         return;
226                     }
227                     // If we consumed the footer we will start with the header again.
228                     assert gzipState == GzipState.HEADER_START;
229                 }
230                 if (!readGZIPHeader(in)) {
231                     // There was not enough data readable to read the GZIP header.
232                     return;
233                 }
234                 // Some bytes may have been consumed, and so we must re-set the number of readable bytes.
235                 readableBytes = in.readableBytes();
236                 if (readableBytes == 0) {
237                     return;
238                 }
239             }
240         }
241 
242         if (inflater.needsInput()) {
243             if (in.hasArray()) {
244                 inflater.setInput(in.array(), in.arrayOffset() + in.readerIndex(), readableBytes);
245             } else {
246                 byte[] array = new byte[readableBytes];
247                 in.getBytes(in.readerIndex(), array);
248                 inflater.setInput(array);
249             }
250         }
251 
252         ByteBuf decompressed = prepareDecompressBuffer(ctx, null, inflater.getRemaining() << 1);
253         try {
254             boolean readFooter = false;
255             while (!inflater.needsInput()) {
256                 byte[] outArray = decompressed.array();
257                 int writerIndex = decompressed.writerIndex();
258                 int outIndex = decompressed.arrayOffset() + writerIndex;
259                 int writable = decompressed.writableBytes();
260                 int outputLength = inflater.inflate(outArray, outIndex, writable);
261                 if (outputLength > 0) {
262                     decompressed.writerIndex(writerIndex + outputLength);
263                     if (crc != null) {
264                         crc.update(outArray, outIndex, outputLength);
265                     }
266                 } else  if (inflater.needsDictionary()) {
267                     if (dictionary == null) {
268                         throw new DecompressionException(
269                                 "decompression failure, unable to set dictionary as non was specified");
270                     }
271                     inflater.setDictionary(dictionary);
272                 }
273 
274                 if (inflater.finished()) {
275                     if (crc == null) {
276                         finished = true; // Do not decode anymore.
277                     } else {
278                         readFooter = true;
279                     }
280                     break;
281                 } else {
282                     decompressed = prepareDecompressBuffer(ctx, decompressed, inflater.getRemaining() << 1);
283                 }
284             }
285 
286             in.skipBytes(readableBytes - inflater.getRemaining());
287 
288             if (readFooter) {
289                 gzipState = GzipState.FOOTER_START;
290                 handleGzipFooter(in);
291             }
292         } catch (DataFormatException e) {
293             throw new DecompressionException("decompression failure", e);
294         } finally {
295             if (decompressed.isReadable()) {
296                 out.add(decompressed);
297             } else {
298                 decompressed.release();
299             }
300         }
301     }
302 
303     private boolean handleGzipFooter(ByteBuf in) {
304         if (readGZIPFooter(in)) {
305             finished = !decompressConcatenated;
306 
307             if (!finished) {
308                 inflater.reset();
309                 crc.reset();
310                 gzipState = GzipState.HEADER_START;
311                 return true;
312             }
313         }
314         return false;
315     }
316 
317     @Override
318     protected void decompressionBufferExhausted(ByteBuf buffer) {
319         finished = true;
320     }
321 
322     @Override
323     protected void handlerRemoved0(ChannelHandlerContext ctx) throws Exception {
324         super.handlerRemoved0(ctx);
325         if (inflater != null) {
326             inflater.end();
327         }
328     }
329 
330     private boolean readGZIPHeader(ByteBuf in) {
331         switch (gzipState) {
332             case HEADER_START:
333                 if (in.readableBytes() < 10) {
334                     return false;
335                 }
336                 // read magic numbers
337                 int magic0 = in.readByte();
338                 int magic1 = in.readByte();
339 
340                 if (magic0 != 31) {
341                     throw new DecompressionException("Input is not in the GZIP format");
342                 }
343                 crc.update(magic0);
344                 crc.update(magic1);
345 
346                 int method = in.readUnsignedByte();
347                 if (method != Deflater.DEFLATED) {
348                     throw new DecompressionException("Unsupported compression method "
349                             + method + " in the GZIP header");
350                 }
351                 crc.update(method);
352 
353                 flags = in.readUnsignedByte();
354                 crc.update(flags);
355 
356                 if ((flags & FRESERVED) != 0) {
357                     throw new DecompressionException(
358                             "Reserved flags are set in the GZIP header");
359                 }
360 
361                 // mtime (int)
362                 crc.update(in, in.readerIndex(), 4);
363                 in.skipBytes(4);
364 
365                 crc.update(in.readUnsignedByte()); // extra flags
366                 crc.update(in.readUnsignedByte()); // operating system
367 
368                 gzipState = GzipState.FLG_READ;
369                 // fall through
370             case FLG_READ:
371                 if ((flags & FEXTRA) != 0) {
372                     if (in.readableBytes() < 2) {
373                         return false;
374                     }
375                     int xlen1 = in.readUnsignedByte();
376                     int xlen2 = in.readUnsignedByte();
377                     crc.update(xlen1);
378                     crc.update(xlen2);
379 
380                     xlen |= xlen1 << 8 | xlen2;
381                 }
382                 gzipState = GzipState.XLEN_READ;
383                 // fall through
384             case XLEN_READ:
385                 if (xlen != -1) {
386                     if (in.readableBytes() < xlen) {
387                         return false;
388                     }
389                     crc.update(in, in.readerIndex(), xlen);
390                     in.skipBytes(xlen);
391                 }
392                 gzipState = GzipState.SKIP_FNAME;
393                 // fall through
394             case SKIP_FNAME:
395                 if (!skipIfNeeded(in, FNAME)) {
396                     return false;
397                 }
398                 gzipState = GzipState.SKIP_COMMENT;
399                 // fall through
400             case SKIP_COMMENT:
401                 if (!skipIfNeeded(in, FCOMMENT)) {
402                     return false;
403                 }
404                 gzipState = GzipState.PROCESS_FHCRC;
405                 // fall through
406             case PROCESS_FHCRC:
407                 if ((flags & FHCRC) != 0) {
408                     if (!verifyCrc16(in)) {
409                         return false;
410                     }
411                 }
412                 crc.reset();
413                 gzipState = GzipState.HEADER_END;
414                 // fall through
415             case HEADER_END:
416                 return true;
417             default:
418                 throw new IllegalStateException();
419         }
420     }
421 
422     /**
423      * Skip bytes in the input if needed until we find the end marker {@code 0x00}.
424      * @param   in the input
425      * @param   flagMask the mask that should be present in the {@code flags} when we need to skip bytes.
426      * @return  {@code true} if the operation is complete and we can move to the next state, {@code false} if we need
427      *          the retry again once we have more readable bytes.
428      */
429     private boolean skipIfNeeded(ByteBuf in, int flagMask) {
430         if ((flags & flagMask) != 0) {
431             for (;;) {
432                 if (!in.isReadable()) {
433                     // We didnt find the end yet, need to retry again once more data is readable
434                     return false;
435                 }
436                 int b = in.readUnsignedByte();
437                 crc.update(b);
438                 if (b == 0x00) {
439                     break;
440                 }
441             }
442         }
443         // Skip is handled, we can move to the next processing state.
444         return true;
445     }
446 
447     /**
448      * Read the GZIP footer.
449      *
450      * @param   in the input.
451      * @return  {@code true} if the footer could be read, {@code false} if the read could not be performed as
452      *          the input {@link ByteBuf} doesn't have enough readable bytes (8 bytes).
453      */
454     private boolean readGZIPFooter(ByteBuf in) {
455         if (in.readableBytes() < 8) {
456             return false;
457         }
458 
459         boolean enoughData = verifyCrc(in);
460         assert enoughData;
461 
462         // read ISIZE and verify
463         int dataLength = 0;
464         for (int i = 0; i < 4; ++i) {
465             dataLength |= in.readUnsignedByte() << i * 8;
466         }
467         int readLength = inflater.getTotalOut();
468         if (dataLength != readLength) {
469             throw new DecompressionException(
470                     "Number of bytes mismatch. Expected: " + dataLength + ", Got: " + readLength);
471         }
472         return true;
473     }
474 
475     /**
476      * Verifies CRC.
477      *
478      * @param   in the input.
479      * @return  {@code true} if verification could be performed, {@code false} if verification could not be performed as
480      *          the input {@link ByteBuf} doesn't have enough readable bytes (4 bytes).
481      */
482     private boolean verifyCrc(ByteBuf in) {
483         if (in.readableBytes() < 4) {
484             return false;
485         }
486         long crcValue = 0;
487         for (int i = 0; i < 4; ++i) {
488             crcValue |= (long) in.readUnsignedByte() << i * 8;
489         }
490         long readCrc = crc.getValue();
491         if (crcValue != readCrc) {
492             throw new DecompressionException(
493                     "CRC value mismatch. Expected: " + crcValue + ", Got: " + readCrc);
494         }
495         return true;
496     }
497 
498     private boolean verifyCrc16(ByteBuf in) {
499         if (in.readableBytes() < 2) {
500             return false;
501         }
502         long readCrc32 = crc.getValue();
503         long crc16Value = 0;
504         long readCrc16 = 0; // the two least significant bytes from the CRC32
505         for (int i = 0; i < 2; ++i) {
506             crc16Value |= (long) in.readUnsignedByte() << (i * 8);
507             readCrc16 |= ((readCrc32 >> (i * 8)) & 0xff) << (i * 8);
508         }
509 
510         if (crc16Value != readCrc16) {
511             throw new DecompressionException(
512                     "CRC16 value mismatch. Expected: " + crc16Value + ", Got: " + readCrc16);
513         }
514         return true;
515     }
516 
517     /*
518      * Returns true if the cmf_flg parameter (think: first two bytes of a zlib stream)
519      * indicates that this is a zlib stream.
520      * <p>
521      * You can lookup the details in the ZLIB RFC:
522      * <a href="https://tools.ietf.org/html/rfc1950#section-2.2">RFC 1950</a>.
523      */
524     private static boolean looksLikeZlib(short cmf_flg) {
525         return (cmf_flg & 0x7800) == 0x7800 &&
526                 cmf_flg % 31 == 0;
527     }
528 }