View Javadoc
1   /*
2    * Copyright 2013 The Netty Project
3    *
4    * The Netty Project licenses this file to you under the Apache License,
5    * version 2.0 (the "License"); you may not use this file except in compliance
6    * with the License. You may obtain a copy of the License at:
7    *
8    *   https://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13   * License for the specific language governing permissions and limitations
14   * under the License.
15   */
16  package io.netty.handler.codec.compression;
17  
18  import io.netty.buffer.ByteBuf;
19  import io.netty.buffer.ByteBufAllocator;
20  import io.netty.channel.ChannelHandlerContext;
21  import io.netty.util.internal.ObjectUtil;
22  
23  import java.util.List;
24  import java.util.zip.CRC32;
25  import java.util.zip.DataFormatException;
26  import java.util.zip.Deflater;
27  import java.util.zip.Inflater;
28  
29  /**
30   * Decompress a {@link ByteBuf} using the inflate algorithm.
31   */
32  public class JdkZlibDecoder extends ZlibDecoder {
33      private static final int FHCRC = 0x02;
34      private static final int FEXTRA = 0x04;
35      private static final int FNAME = 0x08;
36      private static final int FCOMMENT = 0x10;
37      private static final int FRESERVED = 0xE0;
38  
39      private Inflater inflater;
40      private final byte[] dictionary;
41  
42      // GZIP related
43      private final ByteBufChecksum crc;
44      private final boolean decompressConcatenated;
45  
46      private enum GzipState {
47          HEADER_START,
48          HEADER_END,
49          FLG_READ,
50          XLEN_READ,
51          SKIP_FNAME,
52          SKIP_COMMENT,
53          PROCESS_FHCRC,
54          FOOTER_START,
55      }
56  
57      private GzipState gzipState = GzipState.HEADER_START;
58      private int flags = -1;
59      private int xlen = -1;
60      private boolean needsRead;
61  
62      private volatile boolean finished;
63  
64      private boolean decideZlibOrNone;
65  
66      /**
67       * Creates a new instance with the default wrapper ({@link ZlibWrapper#ZLIB}).
68       *
69       * @deprecated Use {@link JdkZlibDecoder#JdkZlibDecoder(int)}.
70       */
71      @Deprecated
72      public JdkZlibDecoder() {
73          this(ZlibWrapper.ZLIB, null, false, 0);
74      }
75  
76      /**
77       * Creates a new instance with the default wrapper ({@link ZlibWrapper#ZLIB})
78       * and the specified maximum buffer allocation.
79       *
80       * @param maxAllocation
81       *          Maximum size of the decompression buffer. Must be >= 0.
82       *          If zero, maximum size is decided by the {@link ByteBufAllocator}.
83       */
84      public JdkZlibDecoder(int maxAllocation) {
85          this(ZlibWrapper.ZLIB, null, false, maxAllocation);
86      }
87  
88      /**
89       * Creates a new instance with the specified preset dictionary. The wrapper
90       * is always {@link ZlibWrapper#ZLIB} because it is the only format that
91       * supports the preset dictionary.
92       *
93       * @deprecated Use {@link JdkZlibDecoder#JdkZlibDecoder(byte[], int)}.
94       */
95      @Deprecated
96      public JdkZlibDecoder(byte[] dictionary) {
97          this(ZlibWrapper.ZLIB, dictionary, false, 0);
98      }
99  
100     /**
101      * Creates a new instance with the specified preset dictionary and maximum buffer allocation.
102      * The wrapper is always {@link ZlibWrapper#ZLIB} because it is the only format that
103      * supports the preset dictionary.
104      *
105      * @param maxAllocation
106      *          Maximum size of the decompression buffer. Must be >= 0.
107      *          If zero, maximum size is decided by the {@link ByteBufAllocator}.
108      */
109     public JdkZlibDecoder(byte[] dictionary, int maxAllocation) {
110         this(ZlibWrapper.ZLIB, dictionary, false, maxAllocation);
111     }
112 
113     /**
114      * Creates a new instance with the specified wrapper.
115      * Be aware that only {@link ZlibWrapper#GZIP}, {@link ZlibWrapper#ZLIB} and {@link ZlibWrapper#NONE} are
116      * supported atm.
117      *
118      * @deprecated Use {@link JdkZlibDecoder#JdkZlibDecoder(ZlibWrapper, int)}.
119      */
120     @Deprecated
121     public JdkZlibDecoder(ZlibWrapper wrapper) {
122         this(wrapper, null, false, 0);
123     }
124 
125     /**
126      * Creates a new instance with the specified wrapper and maximum buffer allocation.
127      * Be aware that only {@link ZlibWrapper#GZIP}, {@link ZlibWrapper#ZLIB} and {@link ZlibWrapper#NONE} are
128      * supported atm.
129      *
130      * @param maxAllocation
131      *          Maximum size of the decompression buffer. Must be >= 0.
132      *          If zero, maximum size is decided by the {@link ByteBufAllocator}.
133      */
134     public JdkZlibDecoder(ZlibWrapper wrapper, int maxAllocation) {
135         this(wrapper, null, false, maxAllocation);
136     }
137 
138     /**
139      * @deprecated Use {@link JdkZlibDecoder#JdkZlibDecoder(ZlibWrapper, boolean, int)}.
140      */
141     @Deprecated
142     public JdkZlibDecoder(ZlibWrapper wrapper, boolean decompressConcatenated) {
143         this(wrapper, null, decompressConcatenated, 0);
144     }
145 
146     public JdkZlibDecoder(ZlibWrapper wrapper, boolean decompressConcatenated, int maxAllocation) {
147         this(wrapper, null, decompressConcatenated, maxAllocation);
148     }
149 
150     /**
151      * @deprecated Use {@link JdkZlibDecoder#JdkZlibDecoder(boolean, int)}.
152      */
153     @Deprecated
154     public JdkZlibDecoder(boolean decompressConcatenated) {
155         this(ZlibWrapper.GZIP, null, decompressConcatenated, 0);
156     }
157 
158     public JdkZlibDecoder(boolean decompressConcatenated, int maxAllocation) {
159         this(ZlibWrapper.GZIP, null, decompressConcatenated, maxAllocation);
160     }
161 
162     private JdkZlibDecoder(ZlibWrapper wrapper, byte[] dictionary, boolean decompressConcatenated, int maxAllocation) {
163         super(maxAllocation);
164 
165         ObjectUtil.checkNotNull(wrapper, "wrapper");
166 
167         this.decompressConcatenated = decompressConcatenated;
168         switch (wrapper) {
169             case GZIP:
170                 inflater = new Inflater(true);
171                 crc = ByteBufChecksum.wrapChecksum(new CRC32());
172                 break;
173             case NONE:
174                 inflater = new Inflater(true);
175                 crc = null;
176                 break;
177             case ZLIB:
178                 inflater = new Inflater();
179                 crc = null;
180                 break;
181             case ZLIB_OR_NONE:
182                 // Postpone the decision until decode(...) is called.
183                 decideZlibOrNone = true;
184                 crc = null;
185                 break;
186             default:
187                 throw new IllegalArgumentException("Only GZIP or ZLIB is supported, but you used " + wrapper);
188         }
189         this.dictionary = dictionary;
190     }
191 
192     @Override
193     public boolean isClosed() {
194         return finished;
195     }
196 
197     @Override
198     protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception {
199         needsRead = true;
200         if (finished) {
201             // Skip data received after finished.
202             in.skipBytes(in.readableBytes());
203             return;
204         }
205 
206         int readableBytes = in.readableBytes();
207         if (readableBytes == 0) {
208             return;
209         }
210 
211         if (decideZlibOrNone) {
212             // First two bytes are needed to decide if it's a ZLIB stream.
213             if (readableBytes < 2) {
214                 return;
215             }
216 
217             boolean nowrap = !looksLikeZlib(in.getShort(in.readerIndex()));
218             inflater = new Inflater(nowrap);
219             decideZlibOrNone = false;
220         }
221 
222         if (crc != null) {
223             if (gzipState != GzipState.HEADER_END) {
224                 if (gzipState == GzipState.FOOTER_START) {
225                     if (!handleGzipFooter(in)) {
226                         // Either there was not enough data or the input is finished.
227                         return;
228                     }
229                     // If we consumed the footer we will start with the header again.
230                     assert gzipState == GzipState.HEADER_START;
231                 }
232                 if (!readGZIPHeader(in)) {
233                     // There was not enough data readable to read the GZIP header.
234                     return;
235                 }
236                 // Some bytes may have been consumed, and so we must re-set the number of readable bytes.
237                 readableBytes = in.readableBytes();
238                 if (readableBytes == 0) {
239                     return;
240                 }
241             }
242         }
243 
244         if (inflater.needsInput()) {
245             if (in.hasArray()) {
246                 inflater.setInput(in.array(), in.arrayOffset() + in.readerIndex(), readableBytes);
247             } else {
248                 byte[] array = new byte[readableBytes];
249                 in.getBytes(in.readerIndex(), array);
250                 inflater.setInput(array);
251             }
252         }
253 
254         ByteBuf decompressed = prepareDecompressBuffer(ctx, null, inflater.getRemaining() << 1);
255         try {
256             boolean readFooter = false;
257             while (!inflater.needsInput()) {
258                 byte[] outArray = decompressed.array();
259                 int writerIndex = decompressed.writerIndex();
260                 int outIndex = decompressed.arrayOffset() + writerIndex;
261                 int writable = decompressed.writableBytes();
262                 int outputLength = inflater.inflate(outArray, outIndex, writable);
263                 if (outputLength > 0) {
264                     decompressed.writerIndex(writerIndex + outputLength);
265                     if (crc != null) {
266                         crc.update(outArray, outIndex, outputLength);
267                     }
268                     if (maxAllocation == 0) {
269                         // If we don't limit the maximum allocations we should just
270                         // forward the buffer directly.
271                         ByteBuf buffer = decompressed;
272                         decompressed = null;
273                         needsRead = false;
274                         ctx.fireChannelRead(buffer);
275                     }
276                 } else if (inflater.needsDictionary()) {
277                     if (dictionary == null) {
278                         throw new DecompressionException(
279                                 "decompression failure, unable to set dictionary as non was specified");
280                     }
281                     inflater.setDictionary(dictionary);
282                 }
283 
284                 if (inflater.finished()) {
285                     if (crc == null) {
286                         finished = true; // Do not decode anymore.
287                     } else {
288                         readFooter = true;
289                     }
290                     break;
291                 } else {
292                     decompressed = prepareDecompressBuffer(ctx, decompressed, inflater.getRemaining() << 1);
293                 }
294             }
295 
296             in.skipBytes(readableBytes - inflater.getRemaining());
297 
298             if (readFooter) {
299                 gzipState = GzipState.FOOTER_START;
300                 handleGzipFooter(in);
301             }
302         } catch (DataFormatException e) {
303             throw new DecompressionException("decompression failure", e);
304         } finally {
305             if (decompressed != null) {
306                 if (decompressed.isReadable()) {
307                     needsRead = false;
308                     ctx.fireChannelRead(decompressed);
309                 } else {
310                     decompressed.release();
311                 }
312             }
313         }
314     }
315 
316     private boolean handleGzipFooter(ByteBuf in) {
317         if (readGZIPFooter(in)) {
318             finished = !decompressConcatenated;
319 
320             if (!finished) {
321                 inflater.reset();
322                 crc.reset();
323                 gzipState = GzipState.HEADER_START;
324                 return true;
325             }
326         }
327         return false;
328     }
329 
330     @Override
331     protected void decompressionBufferExhausted(ByteBuf buffer) {
332         finished = true;
333     }
334 
335     @Override
336     protected void handlerRemoved0(ChannelHandlerContext ctx) throws Exception {
337         super.handlerRemoved0(ctx);
338         if (inflater != null) {
339             inflater.end();
340         }
341     }
342 
343     private boolean readGZIPHeader(ByteBuf in) {
344         switch (gzipState) {
345             case HEADER_START:
346                 if (in.readableBytes() < 10) {
347                     return false;
348                 }
349                 // read magic numbers
350                 int magic0 = in.readByte();
351                 int magic1 = in.readByte();
352 
353                 if (magic0 != 31) {
354                     throw new DecompressionException("Input is not in the GZIP format");
355                 }
356                 crc.update(magic0);
357                 crc.update(magic1);
358 
359                 int method = in.readUnsignedByte();
360                 if (method != Deflater.DEFLATED) {
361                     throw new DecompressionException("Unsupported compression method "
362                             + method + " in the GZIP header");
363                 }
364                 crc.update(method);
365 
366                 flags = in.readUnsignedByte();
367                 crc.update(flags);
368 
369                 if ((flags & FRESERVED) != 0) {
370                     throw new DecompressionException(
371                             "Reserved flags are set in the GZIP header");
372                 }
373 
374                 // mtime (int)
375                 crc.update(in, in.readerIndex(), 4);
376                 in.skipBytes(4);
377 
378                 crc.update(in.readUnsignedByte()); // extra flags
379                 crc.update(in.readUnsignedByte()); // operating system
380 
381                 gzipState = GzipState.FLG_READ;
382                 // fall through
383             case FLG_READ:
384                 if ((flags & FEXTRA) != 0) {
385                     if (in.readableBytes() < 2) {
386                         return false;
387                     }
388                     int xlen1 = in.readUnsignedByte();
389                     int xlen2 = in.readUnsignedByte();
390                     crc.update(xlen1);
391                     crc.update(xlen2);
392 
393                     xlen |= xlen1 << 8 | xlen2;
394                 }
395                 gzipState = GzipState.XLEN_READ;
396                 // fall through
397             case XLEN_READ:
398                 if (xlen != -1) {
399                     if (in.readableBytes() < xlen) {
400                         return false;
401                     }
402                     crc.update(in, in.readerIndex(), xlen);
403                     in.skipBytes(xlen);
404                 }
405                 gzipState = GzipState.SKIP_FNAME;
406                 // fall through
407             case SKIP_FNAME:
408                 if (!skipIfNeeded(in, FNAME)) {
409                     return false;
410                 }
411                 gzipState = GzipState.SKIP_COMMENT;
412                 // fall through
413             case SKIP_COMMENT:
414                 if (!skipIfNeeded(in, FCOMMENT)) {
415                     return false;
416                 }
417                 gzipState = GzipState.PROCESS_FHCRC;
418                 // fall through
419             case PROCESS_FHCRC:
420                 if ((flags & FHCRC) != 0) {
421                     if (!verifyCrc16(in)) {
422                         return false;
423                     }
424                 }
425                 crc.reset();
426                 gzipState = GzipState.HEADER_END;
427                 // fall through
428             case HEADER_END:
429                 return true;
430             default:
431                 throw new IllegalStateException();
432         }
433     }
434 
435     /**
436      * Skip bytes in the input if needed until we find the end marker {@code 0x00}.
437      * @param   in the input
438      * @param   flagMask the mask that should be present in the {@code flags} when we need to skip bytes.
439      * @return  {@code true} if the operation is complete and we can move to the next state, {@code false} if we need
440      *          the retry again once we have more readable bytes.
441      */
442     private boolean skipIfNeeded(ByteBuf in, int flagMask) {
443         if ((flags & flagMask) != 0) {
444             for (;;) {
445                 if (!in.isReadable()) {
446                     // We didnt find the end yet, need to retry again once more data is readable
447                     return false;
448                 }
449                 int b = in.readUnsignedByte();
450                 crc.update(b);
451                 if (b == 0x00) {
452                     break;
453                 }
454             }
455         }
456         // Skip is handled, we can move to the next processing state.
457         return true;
458     }
459 
460     /**
461      * Read the GZIP footer.
462      *
463      * @param   in the input.
464      * @return  {@code true} if the footer could be read, {@code false} if the read could not be performed as
465      *          the input {@link ByteBuf} doesn't have enough readable bytes (8 bytes).
466      */
467     private boolean readGZIPFooter(ByteBuf in) {
468         if (in.readableBytes() < 8) {
469             return false;
470         }
471 
472         boolean enoughData = verifyCrc(in);
473         assert enoughData;
474 
475         // read ISIZE and verify
476         int dataLength = 0;
477         for (int i = 0; i < 4; ++i) {
478             dataLength |= in.readUnsignedByte() << i * 8;
479         }
480         int readLength = inflater.getTotalOut();
481         if (dataLength != readLength) {
482             throw new DecompressionException(
483                     "Number of bytes mismatch. Expected: " + dataLength + ", Got: " + readLength);
484         }
485         return true;
486     }
487 
488     /**
489      * Verifies CRC.
490      *
491      * @param   in the input.
492      * @return  {@code true} if verification could be performed, {@code false} if verification could not be performed as
493      *          the input {@link ByteBuf} doesn't have enough readable bytes (4 bytes).
494      */
495     private boolean verifyCrc(ByteBuf in) {
496         if (in.readableBytes() < 4) {
497             return false;
498         }
499         long crcValue = 0;
500         for (int i = 0; i < 4; ++i) {
501             crcValue |= (long) in.readUnsignedByte() << i * 8;
502         }
503         long readCrc = crc.getValue();
504         if (crcValue != readCrc) {
505             throw new DecompressionException(
506                     "CRC value mismatch. Expected: " + crcValue + ", Got: " + readCrc);
507         }
508         return true;
509     }
510 
511     private boolean verifyCrc16(ByteBuf in) {
512         if (in.readableBytes() < 2) {
513             return false;
514         }
515         long readCrc32 = crc.getValue();
516         long crc16Value = 0;
517         long readCrc16 = 0; // the two least significant bytes from the CRC32
518         for (int i = 0; i < 2; ++i) {
519             crc16Value |= (long) in.readUnsignedByte() << (i * 8);
520             readCrc16 |= ((readCrc32 >> (i * 8)) & 0xff) << (i * 8);
521         }
522 
523         if (crc16Value != readCrc16) {
524             throw new DecompressionException(
525                     "CRC16 value mismatch. Expected: " + crc16Value + ", Got: " + readCrc16);
526         }
527         return true;
528     }
529 
530     /*
531      * Returns true if the cmf_flg parameter (think: first two bytes of a zlib stream)
532      * indicates that this is a zlib stream.
533      * <p>
534      * You can lookup the details in the ZLIB RFC:
535      * <a href="https://tools.ietf.org/html/rfc1950#section-2.2">RFC 1950</a>.
536      */
537     private static boolean looksLikeZlib(short cmf_flg) {
538         return (cmf_flg & 0x7800) == 0x7800 &&
539                 cmf_flg % 31 == 0;
540     }
541 
542     @Override
543     public void channelReadComplete(ChannelHandlerContext ctx) throws Exception {
544         // Discard bytes of the cumulation buffer if needed.
545         discardSomeReadBytes();
546 
547         if (needsRead && !ctx.channel().config().isAutoRead()) {
548             ctx.read();
549         }
550         ctx.fireChannelReadComplete();
551     }
552 }