View Javadoc
1   /*
2    * Copyright 2025 The Netty Project
3    *
4    * The Netty Project licenses this file to you under the Apache License,
5    * version 2.0 (the "License"); you may not use this file except in compliance
6    * with the License. You may obtain a copy of the License at:
7    *
8    *   https://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13   * License for the specific language governing permissions and limitations
14   * under the License.
15   */
16  package io.netty.handler.codec.compression;
17  
18  import io.netty.buffer.ByteBuf;
19  import io.netty.buffer.ByteBufAllocator;
20  import io.netty.buffer.ByteBufUtil;
21  import io.netty.util.internal.ObjectUtil;
22  import io.netty.util.internal.UnstableApi;
23  
24  import java.util.Objects;
25  import java.util.zip.CRC32;
26  import java.util.zip.DataFormatException;
27  import java.util.zip.Deflater;
28  import java.util.zip.Inflater;
29  
30  /**
31   * Decompress a {@link ByteBuf} using the inflate algorithm.
32   */
33  @UnstableApi
34  public final class JdkZlibDecompressor extends InputBufferingDecompressor {
35      private static final int FHCRC = 0x02;
36      private static final int FEXTRA = 0x04;
37      private static final int FNAME = 0x08;
38      private static final int FCOMMENT = 0x10;
39      private static final int FRESERVED = 0xE0;
40  
41      /**
42       * Smallest output buffer we hand to {@link Inflater#inflate(byte[], int, int)}. The number of remaining input
43       * bytes is only a hint for how much output to expect: the inflater may still hold decoded data that did not fit
44       * into the previous output buffer, and by then it may have consumed all input bytes already
45       * ({@link Inflater#getRemaining()} == 0). Inflating into a zero-sized buffer can never make progress.
46       */
47      private static final int MIN_OUTPUT_BUFFER_SIZE = 512;
48  
49      private Inflater inflater;
50      private final int maxAllocation;
51      private final byte[] dictionary;
52  
53      // GZIP related
54      private final ByteBufChecksum crc;
55      private final boolean decompressConcatenated;
56  
57      private enum GzipState {
58          HEADER_START,
59          HEADER_END,
60          FLG_READ,
61          XLEN_READ,
62          SKIP_FNAME,
63          SKIP_COMMENT,
64          PROCESS_FHCRC,
65          FOOTER_START,
66      }
67  
68      private GzipState gzipState = GzipState.HEADER_START;
69      private int flags = -1;
70      private int xlen = -1;
71  
72      private boolean decideZlibOrNone;
73      private boolean finished;
74      private boolean gzipMemberFinished;
75      /**
76       * If this is true, part of the input buffer is still in use by the {@link #inflater}, so we shouldn't touch that
77       * buffer too much (e.g. compact it).
78       */
79      private boolean inputBufferInInflater;
80      /**
81       * If this is true, the last {@link Inflater#inflate(byte[], int, int)} filled the output buffer completely, so
82       * the inflater may still hold decoded data even if {@link Inflater#needsInput()} reports that all input was
83       * consumed.
84       */
85      private boolean inflaterHasPendingOutput;
86  
87      JdkZlibDecompressor(Builder builder, ByteBufAllocator allocator) {
88          super(allocator);
89          maxAllocation = builder.maxAllocation;
90          dictionary = builder.dictionary;
91          this.decompressConcatenated = builder.decompressConcatenated;
92          switch (builder.wrapper) {
93              case GZIP:
94                  inflater = new Inflater(true);
95                  crc = ByteBufChecksum.wrapChecksum(new CRC32());
96                  break;
97              case NONE:
98                  inflater = new Inflater(true);
99                  crc = null;
100                 break;
101             case ZLIB:
102                 inflater = new Inflater();
103                 crc = null;
104                 break;
105             case ZLIB_OR_NONE:
106                 // Postpone the decision until decode(...) is called.
107                 decideZlibOrNone = true;
108                 crc = null;
109                 break;
110             default:
111                 throw new IllegalArgumentException("Only GZIP or ZLIB is supported, but you used " + builder.wrapper);
112         }
113     }
114 
115     @Override
116     public Status status() throws DecompressionException {
117         if (finished) {
118             return Status.COMPLETE;
119         } else if (inflaterHasPendingOutput) {
120             return Status.NEED_OUTPUT;
121         } else if (inflater == null || inflater.needsInput() || gzipState == GzipState.FOOTER_START) {
122             return Status.NEED_INPUT;
123         } else {
124             return Status.NEED_OUTPUT;
125         }
126     }
127 
128     @Override
129     public void endOfInput() throws DecompressionException {
130         if (finished) {
131             return;
132         }
133         if (crc != null && decompressConcatenated && gzipMemberFinished &&
134                 gzipState == GzipState.HEADER_START && available() == 0) {
135             finished = true;
136             return;
137         }
138         throw new DecompressionException("Compressed stream ended before the end-of-stream marker");
139     }
140 
141     @Override
142     void processInput(ByteBuf buf) throws DecompressionException {
143         if (inputBufferInInflater) {
144             throw new IllegalStateException("Not in state NEED_INPUT");
145         }
146 
147         int readableBytes = buf.readableBytes();
148         if (readableBytes == 0) {
149             return;
150         }
151 
152         if (decideZlibOrNone) {
153             // First two bytes are needed to decide if it's a ZLIB stream.
154             if (readableBytes < 2) {
155                 return;
156             }
157 
158             boolean nowrap = !looksLikeZlib(buf.getShort(buf.readerIndex()));
159             inflater = new Inflater(nowrap);
160             decideZlibOrNone = false;
161         }
162 
163         if (crc != null) {
164             if (gzipState != GzipState.HEADER_END) {
165                 if (gzipState == GzipState.FOOTER_START) {
166                     if (!handleGzipFooter(buf)) {
167                         // Either there was not enough data or the input is finished.
168                         return;
169                     }
170                     // If we consumed the footer we will start with the header again.
171                     assert gzipState == GzipState.HEADER_START;
172                 }
173                 if (!readGZIPHeader(buf)) {
174                     // There was not enough data readable to read the GZIP header.
175                     return;
176                 }
177                 // Some bytes may have been consumed, and so we must re-set the number of readable bytes.
178                 readableBytes = buf.readableBytes();
179                 if (readableBytes == 0) {
180                     return;
181                 }
182             }
183         }
184 
185         if (inflater.needsInput()) {
186             if (buf.hasArray()) {
187                 inflater.setInput(buf.array(), buf.arrayOffset() + buf.readerIndex(), readableBytes);
188             } else {
189                 inflater.setInput(ByteBufUtil.getBytes(buf));
190             }
191             inputBufferInInflater = true;
192         }
193     }
194 
195     private void consumeInput(ByteBuf buf) {
196         if (inputBufferInInflater) {
197             buf.readerIndex(buf.writerIndex() - inflater.getRemaining());
198             if (inflater.getRemaining() == 0) {
199                 inputBufferInInflater = false;
200             }
201         }
202     }
203 
204     @Override
205     ByteBuf processOutput(ByteBuf buf) throws DecompressionException {
206         int proposedCapacity = Math.max(inflater.getRemaining() << 1, MIN_OUTPUT_BUFFER_SIZE);
207         int targetCapacity = maxAllocation == 0
208                 ? proposedCapacity : Math.min(maxAllocation, proposedCapacity);
209         ByteBuf decompressed = allocator.heapBuffer(targetCapacity);
210         boolean success = false;
211         try {
212             byte[] outArray = decompressed.array();
213             int writerIndex = decompressed.writerIndex();
214             int outIndex = decompressed.arrayOffset() + writerIndex;
215             int writableBytes = decompressed.writableBytes();
216             int outputLength;
217             try {
218                 outputLength = inflater.inflate(outArray, outIndex, writableBytes);
219             } catch (DataFormatException e) {
220                 throw new DecompressionException("decompression failure", e);
221             }
222             consumeInput(buf);
223             if (outputLength > 0) {
224                 decompressed.writerIndex(writerIndex + outputLength);
225                 if (crc != null) {
226                     crc.update(outArray, outIndex, outputLength);
227                 }
228             }
229             if (inflater.needsDictionary()) {
230                 if (dictionary == null) {
231                     throw new DecompressionException(
232                             "decompression failure, unable to set dictionary as none was specified");
233                 }
234                 inflater.setDictionary(dictionary);
235             }
236             if (inflater.finished()) {
237                 inputBufferInInflater = false;
238                 if (crc == null) {
239                     finished = true; // Do not decode anymore.
240                 } else {
241                     gzipState = GzipState.FOOTER_START;
242                     // potentially consume footer
243                     processInput(buf);
244                 }
245             }
246             // If we filled the whole buffer the inflater may still have decoded data left that it could not write.
247             // In that case we must ask for another output buffer, as needsInput() only tells us that all input
248             // bytes were consumed, not that all output was produced.
249             inflaterHasPendingOutput = outputLength == writableBytes && !inflater.finished();
250             success = true;
251             return decompressed;
252         } finally {
253             if (!success) {
254                 decompressed.release();
255             }
256         }
257     }
258 
259     private boolean handleGzipFooter(ByteBuf in) {
260         if (readGZIPFooter(in)) {
261             gzipMemberFinished = true;
262             finished = !decompressConcatenated;
263 
264             if (!finished) {
265                 inflater.reset();
266                 crc.reset();
267                 xlen = -1;
268                 gzipState = GzipState.HEADER_START;
269                 return true;
270             }
271         }
272         return false;
273     }
274 
275     private boolean readGZIPHeader(ByteBuf in) {
276         switch (gzipState) {
277             case HEADER_START:
278                 if (in.readableBytes() < 10) {
279                     return false;
280                 }
281                 // read magic numbers
282                 int magic0 = in.readUnsignedByte();
283                 int magic1 = in.readUnsignedByte();
284 
285                 if (magic0 != 31 || magic1 != 139) {
286                     throw new DecompressionException("Input is not in the GZIP format");
287                 }
288                 crc.update(magic0);
289                 crc.update(magic1);
290 
291                 int method = in.readUnsignedByte();
292                 if (method != Deflater.DEFLATED) {
293                     throw new DecompressionException("Unsupported compression method "
294                             + method + " in the GZIP header");
295                 }
296                 crc.update(method);
297 
298                 flags = in.readUnsignedByte();
299                 crc.update(flags);
300 
301                 if ((flags & FRESERVED) != 0) {
302                     throw new DecompressionException(
303                             "Reserved flags are set in the GZIP header");
304                 }
305 
306                 // mtime (int)
307                 crc.update(in, in.readerIndex(), 4);
308                 in.skipBytes(4);
309 
310                 crc.update(in.readUnsignedByte()); // extra flags
311                 crc.update(in.readUnsignedByte()); // operating system
312 
313                 gzipState = GzipState.FLG_READ;
314                 // fall through
315             case FLG_READ:
316                 if ((flags & FEXTRA) != 0) {
317                     if (in.readableBytes() < 2) {
318                         return false;
319                     }
320                     int xlen1 = in.readUnsignedByte();
321                     int xlen2 = in.readUnsignedByte();
322                     crc.update(xlen1);
323                     crc.update(xlen2);
324 
325                     xlen = xlen2 << 8 | xlen1;
326                 }
327                 gzipState = GzipState.XLEN_READ;
328                 // fall through
329             case XLEN_READ:
330                 if (xlen != -1) {
331                     if (in.readableBytes() < xlen) {
332                         return false;
333                     }
334                     crc.update(in, in.readerIndex(), xlen);
335                     in.skipBytes(xlen);
336                 }
337                 gzipState = GzipState.SKIP_FNAME;
338                 // fall through
339             case SKIP_FNAME:
340                 if (!skipIfNeeded(in, FNAME)) {
341                     return false;
342                 }
343                 gzipState = GzipState.SKIP_COMMENT;
344                 // fall through
345             case SKIP_COMMENT:
346                 if (!skipIfNeeded(in, FCOMMENT)) {
347                     return false;
348                 }
349                 gzipState = GzipState.PROCESS_FHCRC;
350                 // fall through
351             case PROCESS_FHCRC:
352                 if ((flags & FHCRC) != 0) {
353                     if (!verifyCrc16(in)) {
354                         return false;
355                     }
356                 }
357                 crc.reset();
358                 gzipState = GzipState.HEADER_END;
359                 // fall through
360             case HEADER_END:
361                 return true;
362             default:
363                 throw new IllegalStateException();
364         }
365     }
366 
367     /**
368      * Skip bytes in the input if needed until we find the end marker {@code 0x00}.
369      * @param   in the input
370      * @param   flagMask the mask that should be present in the {@code flags} when we need to skip bytes.
371      * @return  {@code true} if the operation is complete and we can move to the next state, {@code false} if we need
372      *          the retry again once we have more readable bytes.
373      */
374     private boolean skipIfNeeded(ByteBuf in, int flagMask) {
375         if ((flags & flagMask) != 0) {
376             for (;;) {
377                 if (!in.isReadable()) {
378                     // We didnt find the end yet, need to retry again once more data is readable
379                     return false;
380                 }
381                 int b = in.readUnsignedByte();
382                 crc.update(b);
383                 if (b == 0x00) {
384                     break;
385                 }
386             }
387         }
388         // Skip is handled, we can move to the next processing state.
389         return true;
390     }
391 
392     /**
393      * Read the GZIP footer.
394      *
395      * @param   in the input.
396      * @return  {@code true} if the footer could be read, {@code false} if the read could not be performed as
397      *          the input {@link ByteBuf} doesn't have enough readable bytes (8 bytes).
398      */
399     private boolean readGZIPFooter(ByteBuf in) {
400         if (in.readableBytes() < 8) {
401             return false;
402         }
403 
404         boolean enoughData = verifyCrc(in);
405         assert enoughData;
406 
407         // read ISIZE and verify
408         int dataLength = in.readIntLE();
409         int readLength = inflater.getTotalOut();
410         if (dataLength != readLength) {
411             throw new DecompressionException(
412                     "Number of bytes mismatch. Expected: " + dataLength + ", Got: " + readLength);
413         }
414         return true;
415     }
416 
417     /**
418      * Verifies CRC.
419      *
420      * @param   in the input.
421      * @return  {@code true} if verification could be performed, {@code false} if verification could not be performed as
422      *          the input {@link ByteBuf} doesn't have enough readable bytes (4 bytes).
423      */
424     private boolean verifyCrc(ByteBuf in) {
425         if (in.readableBytes() < 4) {
426             return false;
427         }
428         long crcValue = in.readUnsignedIntLE();
429 
430         long readCrc = crc.getValue();
431         if (crcValue != readCrc) {
432             throw new DecompressionException(
433                     "CRC value mismatch. Expected: " + crcValue + ", Got: " + readCrc);
434         }
435         return true;
436     }
437 
438     private boolean verifyCrc16(ByteBuf in) {
439         if (in.readableBytes() < 2) {
440             return false;
441         }
442 
443         int crc16Value = in.readUnsignedShortLE();
444         // the two least significant bytes from the CRC32
445         int readCrc16 = (int) (crc.getValue() & 0xFFFF);
446 
447         if (crc16Value != readCrc16) {
448             throw new DecompressionException(
449                     "CRC16 value mismatch. Expected: " + crc16Value + ", Got: " + readCrc16);
450         }
451         return true;
452     }
453 
454     /*
455      * Returns true if the cmf_flg parameter (think: first two bytes of a zlib stream)
456      * indicates that this is a zlib stream.
457      * <p>
458      * You can lookup the details in the ZLIB RFC:
459      * <a href="https://tools.ietf.org/html/rfc1950#section-2.2">RFC 1950</a>.
460      */
461     private static boolean looksLikeZlib(short cmf_flg) {
462         return (cmf_flg & 0x7800) == 0x7800 &&
463                 cmf_flg % 31 == 0;
464     }
465 
466     @Override
467     public void close() {
468         try {
469             super.close();
470         } finally {
471             if (inflater != null) {
472                 inflater.end();
473                 inflater = null;
474             }
475         }
476     }
477 
478     @UnstableApi
479     public static Builder builder() {
480         return new Builder();
481     }
482 
483     @UnstableApi
484     public static final class Builder extends Decompressor.AbstractDecompressorBuilder {
485         private ZlibWrapper wrapper = ZlibWrapper.ZLIB;
486         private byte[] dictionary;
487         private int maxAllocation = 1024 * 1024;
488         boolean decompressConcatenated;
489 
490         Builder() {
491         }
492 
493         /**
494          * Set the wrapper format for the deflated data. Defaults to {@link ZlibWrapper#ZLIB}.
495          *
496          * @param wrapper The wrapper format
497          * @return This builder
498          */
499         public Builder wrapper(ZlibWrapper wrapper) {
500             this.wrapper = Objects.requireNonNull(wrapper, "wrapper");
501             return this;
502         }
503 
504         /**
505          * Set the preset dictionary to use. Defaults to no dictionary.
506          *
507          * @param dictionary The dictionary
508          * @return This builder
509          */
510         public Builder dictionary(byte[] dictionary) {
511             this.dictionary = dictionary;
512             return this;
513         }
514 
515         /**
516          * Set the maximum output buffer size. Defaults to 1M.
517          *
518          * @param maxAllocation The maximum output buffer size.
519          * @return This builder
520          */
521         public Builder maxAllocation(int maxAllocation) {
522             this.maxAllocation = ObjectUtil.checkPositiveOrZero(maxAllocation, "maxAllocation");
523             return this;
524         }
525 
526         public Builder decompressConcatenated(boolean decompressConcatenated) {
527             this.decompressConcatenated = decompressConcatenated;
528             return this;
529         }
530 
531         @Override
532         public Decompressor build(ByteBufAllocator allocator) throws DecompressionException {
533             return new DefensiveDecompressor(new JdkZlibDecompressor(this, allocator));
534         }
535     }
536 }