View Javadoc
1   /*
2    * Copyright 2013 The Netty Project
3    *
4    * The Netty Project licenses this file to you under the Apache License,
5    * version 2.0 (the "License"); you may not use this file except in compliance
6    * with the License. You may obtain a copy of the License at:
7    *
8    *   https://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13   * License for the specific language governing permissions and limitations
14   * under the License.
15   */
16  package io.netty.handler.codec.compression;
17  
18  import io.netty.buffer.ByteBuf;
19  import io.netty.buffer.ByteBufAllocator;
20  import io.netty.channel.ChannelHandlerContext;
21  import io.netty.util.internal.ObjectUtil;
22  
23  import java.util.List;
24  import java.util.zip.CRC32;
25  import java.util.zip.DataFormatException;
26  import java.util.zip.Deflater;
27  import java.util.zip.Inflater;
28  
29  /**
30   * Decompress a {@link ByteBuf} using the inflate algorithm.
31   */
32  public class JdkZlibDecoder extends ZlibDecoder {
33      private static final int FHCRC = 0x02;
34      private static final int FEXTRA = 0x04;
35      private static final int FNAME = 0x08;
36      private static final int FCOMMENT = 0x10;
37      private static final int FRESERVED = 0xE0;
38  
39      private Inflater inflater;
40      private final byte[] dictionary;
41  
42      // GZIP related
43      private final ByteBufChecksum crc;
44      private final boolean decompressConcatenated;
45  
46      private enum GzipState {
47          HEADER_START,
48          HEADER_END,
49          FLG_READ,
50          XLEN_READ,
51          SKIP_FNAME,
52          SKIP_COMMENT,
53          PROCESS_FHCRC,
54          FOOTER_START,
55      }
56  
57      private GzipState gzipState = GzipState.HEADER_START;
58      private int flags = -1;
59      private int xlen = -1;
60  
61      private volatile boolean finished;
62  
63      private boolean decideZlibOrNone;
64  
65      /**
66       * Creates a new instance with the default wrapper ({@link ZlibWrapper#ZLIB}).
67       */
68      public JdkZlibDecoder() {
69          this(ZlibWrapper.ZLIB, null, false, 0);
70      }
71  
72      /**
73       * Creates a new instance with the default wrapper ({@link ZlibWrapper#ZLIB})
74       * and the specified maximum buffer allocation.
75       *
76       * @param maxAllocation
77       *          Maximum size of the decompression buffer. Must be >= 0.
78       *          If zero, maximum size is decided by the {@link ByteBufAllocator}.
79       */
80      public JdkZlibDecoder(int maxAllocation) {
81          this(ZlibWrapper.ZLIB, null, false, maxAllocation);
82      }
83  
84      /**
85       * Creates a new instance with the specified preset dictionary. The wrapper
86       * is always {@link ZlibWrapper#ZLIB} because it is the only format that
87       * supports the preset dictionary.
88       */
89      public JdkZlibDecoder(byte[] dictionary) {
90          this(ZlibWrapper.ZLIB, dictionary, false, 0);
91      }
92  
93      /**
94       * Creates a new instance with the specified preset dictionary and maximum buffer allocation.
95       * The wrapper is always {@link ZlibWrapper#ZLIB} because it is the only format that
96       * supports the preset dictionary.
97       *
98       * @param maxAllocation
99       *          Maximum size of the decompression buffer. Must be >= 0.
100      *          If zero, maximum size is decided by the {@link ByteBufAllocator}.
101      */
102     public JdkZlibDecoder(byte[] dictionary, int maxAllocation) {
103         this(ZlibWrapper.ZLIB, dictionary, false, maxAllocation);
104     }
105 
106     /**
107      * Creates a new instance with the specified wrapper.
108      * Be aware that only {@link ZlibWrapper#GZIP}, {@link ZlibWrapper#ZLIB} and {@link ZlibWrapper#NONE} are
109      * supported atm.
110      */
111     public JdkZlibDecoder(ZlibWrapper wrapper) {
112         this(wrapper, null, false, 0);
113     }
114 
115     /**
116      * Creates a new instance with the specified wrapper and maximum buffer allocation.
117      * Be aware that only {@link ZlibWrapper#GZIP}, {@link ZlibWrapper#ZLIB} and {@link ZlibWrapper#NONE} are
118      * supported atm.
119      *
120      * @param maxAllocation
121      *          Maximum size of the decompression buffer. Must be >= 0.
122      *          If zero, maximum size is decided by the {@link ByteBufAllocator}.
123      */
124     public JdkZlibDecoder(ZlibWrapper wrapper, int maxAllocation) {
125         this(wrapper, null, false, maxAllocation);
126     }
127 
128     public JdkZlibDecoder(ZlibWrapper wrapper, boolean decompressConcatenated) {
129         this(wrapper, null, decompressConcatenated, 0);
130     }
131 
132     public JdkZlibDecoder(ZlibWrapper wrapper, boolean decompressConcatenated, int maxAllocation) {
133         this(wrapper, null, decompressConcatenated, maxAllocation);
134     }
135 
136     public JdkZlibDecoder(boolean decompressConcatenated) {
137         this(ZlibWrapper.GZIP, null, decompressConcatenated, 0);
138     }
139 
140     public JdkZlibDecoder(boolean decompressConcatenated, int maxAllocation) {
141         this(ZlibWrapper.GZIP, null, decompressConcatenated, maxAllocation);
142     }
143 
144     private JdkZlibDecoder(ZlibWrapper wrapper, byte[] dictionary, boolean decompressConcatenated, int maxAllocation) {
145         super(maxAllocation);
146 
147         ObjectUtil.checkNotNull(wrapper, "wrapper");
148 
149         this.decompressConcatenated = decompressConcatenated;
150         switch (wrapper) {
151             case GZIP:
152                 inflater = new Inflater(true);
153                 crc = ByteBufChecksum.wrapChecksum(new CRC32());
154                 break;
155             case NONE:
156                 inflater = new Inflater(true);
157                 crc = null;
158                 break;
159             case ZLIB:
160                 inflater = new Inflater();
161                 crc = null;
162                 break;
163             case ZLIB_OR_NONE:
164                 // Postpone the decision until decode(...) is called.
165                 decideZlibOrNone = true;
166                 crc = null;
167                 break;
168             default:
169                 throw new IllegalArgumentException("Only GZIP or ZLIB is supported, but you used " + wrapper);
170         }
171         this.dictionary = dictionary;
172     }
173 
174     @Override
175     public boolean isClosed() {
176         return finished;
177     }
178 
179     @Override
180     protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception {
181         if (finished) {
182             // Skip data received after finished.
183             in.skipBytes(in.readableBytes());
184             return;
185         }
186 
187         int readableBytes = in.readableBytes();
188         if (readableBytes == 0) {
189             return;
190         }
191 
192         if (decideZlibOrNone) {
193             // First two bytes are needed to decide if it's a ZLIB stream.
194             if (readableBytes < 2) {
195                 return;
196             }
197 
198             boolean nowrap = !looksLikeZlib(in.getShort(in.readerIndex()));
199             inflater = new Inflater(nowrap);
200             decideZlibOrNone = false;
201         }
202 
203         if (crc != null) {
204             if (gzipState != GzipState.HEADER_END) {
205                 if (gzipState == GzipState.FOOTER_START) {
206                     if (!handleGzipFooter(in)) {
207                         // Either there was not enough data or the input is finished.
208                         return;
209                     }
210                     // If we consumed the footer we will start with the header again.
211                     assert gzipState == GzipState.HEADER_START;
212                 }
213                 if (!readGZIPHeader(in)) {
214                     // There was not enough data readable to read the GZIP header.
215                     return;
216                 }
217                 // Some bytes may have been consumed, and so we must re-set the number of readable bytes.
218                 readableBytes = in.readableBytes();
219                 if (readableBytes == 0) {
220                     return;
221                 }
222             }
223         }
224 
225         if (inflater.needsInput()) {
226             if (in.hasArray()) {
227                 inflater.setInput(in.array(), in.arrayOffset() + in.readerIndex(), readableBytes);
228             } else {
229                 byte[] array = new byte[readableBytes];
230                 in.getBytes(in.readerIndex(), array);
231                 inflater.setInput(array);
232             }
233         }
234 
235         ByteBuf decompressed = prepareDecompressBuffer(ctx, null, inflater.getRemaining() << 1);
236         try {
237             boolean readFooter = false;
238             while (!inflater.needsInput()) {
239                 byte[] outArray = decompressed.array();
240                 int writerIndex = decompressed.writerIndex();
241                 int outIndex = decompressed.arrayOffset() + writerIndex;
242                 int writable = decompressed.writableBytes();
243                 int outputLength = inflater.inflate(outArray, outIndex, writable);
244                 if (outputLength > 0) {
245                     decompressed.writerIndex(writerIndex + outputLength);
246                     if (crc != null) {
247                         crc.update(outArray, outIndex, outputLength);
248                     }
249                 } else  if (inflater.needsDictionary()) {
250                     if (dictionary == null) {
251                         throw new DecompressionException(
252                                 "decompression failure, unable to set dictionary as non was specified");
253                     }
254                     inflater.setDictionary(dictionary);
255                 }
256 
257                 if (inflater.finished()) {
258                     if (crc == null) {
259                         finished = true; // Do not decode anymore.
260                     } else {
261                         readFooter = true;
262                     }
263                     break;
264                 } else {
265                     decompressed = prepareDecompressBuffer(ctx, decompressed, inflater.getRemaining() << 1);
266                 }
267             }
268 
269             in.skipBytes(readableBytes - inflater.getRemaining());
270 
271             if (readFooter) {
272                 gzipState = GzipState.FOOTER_START;
273                 handleGzipFooter(in);
274             }
275         } catch (DataFormatException e) {
276             throw new DecompressionException("decompression failure", e);
277         } finally {
278             if (decompressed.isReadable()) {
279                 out.add(decompressed);
280             } else {
281                 decompressed.release();
282             }
283         }
284     }
285 
286     private boolean handleGzipFooter(ByteBuf in) {
287         if (readGZIPFooter(in)) {
288             finished = !decompressConcatenated;
289 
290             if (!finished) {
291                 inflater.reset();
292                 crc.reset();
293                 gzipState = GzipState.HEADER_START;
294                 return true;
295             }
296         }
297         return false;
298     }
299 
300     @Override
301     protected void decompressionBufferExhausted(ByteBuf buffer) {
302         finished = true;
303     }
304 
305     @Override
306     protected void handlerRemoved0(ChannelHandlerContext ctx) throws Exception {
307         super.handlerRemoved0(ctx);
308         if (inflater != null) {
309             inflater.end();
310         }
311     }
312 
313     private boolean readGZIPHeader(ByteBuf in) {
314         switch (gzipState) {
315             case HEADER_START:
316                 if (in.readableBytes() < 10) {
317                     return false;
318                 }
319                 // read magic numbers
320                 int magic0 = in.readByte();
321                 int magic1 = in.readByte();
322 
323                 if (magic0 != 31) {
324                     throw new DecompressionException("Input is not in the GZIP format");
325                 }
326                 crc.update(magic0);
327                 crc.update(magic1);
328 
329                 int method = in.readUnsignedByte();
330                 if (method != Deflater.DEFLATED) {
331                     throw new DecompressionException("Unsupported compression method "
332                             + method + " in the GZIP header");
333                 }
334                 crc.update(method);
335 
336                 flags = in.readUnsignedByte();
337                 crc.update(flags);
338 
339                 if ((flags & FRESERVED) != 0) {
340                     throw new DecompressionException(
341                             "Reserved flags are set in the GZIP header");
342                 }
343 
344                 // mtime (int)
345                 crc.update(in, in.readerIndex(), 4);
346                 in.skipBytes(4);
347 
348                 crc.update(in.readUnsignedByte()); // extra flags
349                 crc.update(in.readUnsignedByte()); // operating system
350 
351                 gzipState = GzipState.FLG_READ;
352                 // fall through
353             case FLG_READ:
354                 if ((flags & FEXTRA) != 0) {
355                     if (in.readableBytes() < 2) {
356                         return false;
357                     }
358                     int xlen1 = in.readUnsignedByte();
359                     int xlen2 = in.readUnsignedByte();
360                     crc.update(xlen1);
361                     crc.update(xlen2);
362 
363                     xlen |= xlen1 << 8 | xlen2;
364                 }
365                 gzipState = GzipState.XLEN_READ;
366                 // fall through
367             case XLEN_READ:
368                 if (xlen != -1) {
369                     if (in.readableBytes() < xlen) {
370                         return false;
371                     }
372                     crc.update(in, in.readerIndex(), xlen);
373                     in.skipBytes(xlen);
374                 }
375                 gzipState = GzipState.SKIP_FNAME;
376                 // fall through
377             case SKIP_FNAME:
378                 if (!skipIfNeeded(in, FNAME)) {
379                     return false;
380                 }
381                 gzipState = GzipState.SKIP_COMMENT;
382                 // fall through
383             case SKIP_COMMENT:
384                 if (!skipIfNeeded(in, FCOMMENT)) {
385                     return false;
386                 }
387                 gzipState = GzipState.PROCESS_FHCRC;
388                 // fall through
389             case PROCESS_FHCRC:
390                 if ((flags & FHCRC) != 0) {
391                     if (!verifyCrc16(in)) {
392                         return false;
393                     }
394                 }
395                 crc.reset();
396                 gzipState = GzipState.HEADER_END;
397                 // fall through
398             case HEADER_END:
399                 return true;
400             default:
401                 throw new IllegalStateException();
402         }
403     }
404 
405     /**
406      * Skip bytes in the input if needed until we find the end marker {@code 0x00}.
407      * @param   in the input
408      * @param   flagMask the mask that should be present in the {@code flags} when we need to skip bytes.
409      * @return  {@code true} if the operation is complete and we can move to the next state, {@code false} if we need
410      *          the retry again once we have more readable bytes.
411      */
412     private boolean skipIfNeeded(ByteBuf in, int flagMask) {
413         if ((flags & flagMask) != 0) {
414             for (;;) {
415                 if (!in.isReadable()) {
416                     // We didnt find the end yet, need to retry again once more data is readable
417                     return false;
418                 }
419                 int b = in.readUnsignedByte();
420                 crc.update(b);
421                 if (b == 0x00) {
422                     break;
423                 }
424             }
425         }
426         // Skip is handled, we can move to the next processing state.
427         return true;
428     }
429 
430     /**
431      * Read the GZIP footer.
432      *
433      * @param   in the input.
434      * @return  {@code true} if the footer could be read, {@code false} if the read could not be performed as
435      *          the input {@link ByteBuf} doesn't have enough readable bytes (8 bytes).
436      */
437     private boolean readGZIPFooter(ByteBuf in) {
438         if (in.readableBytes() < 8) {
439             return false;
440         }
441 
442         boolean enoughData = verifyCrc(in);
443         assert enoughData;
444 
445         // read ISIZE and verify
446         int dataLength = 0;
447         for (int i = 0; i < 4; ++i) {
448             dataLength |= in.readUnsignedByte() << i * 8;
449         }
450         int readLength = inflater.getTotalOut();
451         if (dataLength != readLength) {
452             throw new DecompressionException(
453                     "Number of bytes mismatch. Expected: " + dataLength + ", Got: " + readLength);
454         }
455         return true;
456     }
457 
458     /**
459      * Verifies CRC.
460      *
461      * @param   in the input.
462      * @return  {@code true} if verification could be performed, {@code false} if verification could not be performed as
463      *          the input {@link ByteBuf} doesn't have enough readable bytes (4 bytes).
464      */
465     private boolean verifyCrc(ByteBuf in) {
466         if (in.readableBytes() < 4) {
467             return false;
468         }
469         long crcValue = 0;
470         for (int i = 0; i < 4; ++i) {
471             crcValue |= (long) in.readUnsignedByte() << i * 8;
472         }
473         long readCrc = crc.getValue();
474         if (crcValue != readCrc) {
475             throw new DecompressionException(
476                     "CRC value mismatch. Expected: " + crcValue + ", Got: " + readCrc);
477         }
478         return true;
479     }
480 
481     private boolean verifyCrc16(ByteBuf in) {
482         if (in.readableBytes() < 2) {
483             return false;
484         }
485         long readCrc32 = crc.getValue();
486         long crc16Value = 0;
487         long readCrc16 = 0; // the two least significant bytes from the CRC32
488         for (int i = 0; i < 2; ++i) {
489             crc16Value |= (long) in.readUnsignedByte() << (i * 8);
490             readCrc16 |= ((readCrc32 >> (i * 8)) & 0xff) << (i * 8);
491         }
492 
493         if (crc16Value != readCrc16) {
494             throw new DecompressionException(
495                     "CRC16 value mismatch. Expected: " + crc16Value + ", Got: " + readCrc16);
496         }
497         return true;
498     }
499 
500     /*
501      * Returns true if the cmf_flg parameter (think: first two bytes of a zlib stream)
502      * indicates that this is a zlib stream.
503      * <p>
504      * You can lookup the details in the ZLIB RFC:
505      * <a href="https://tools.ietf.org/html/rfc1950#section-2.2">RFC 1950</a>.
506      */
507     private static boolean looksLikeZlib(short cmf_flg) {
508         return (cmf_flg & 0x7800) == 0x7800 &&
509                 cmf_flg % 31 == 0;
510     }
511 }