View Javadoc
1   /*
2    * Copyright 2025 The Netty Project
3    *
4    * The Netty Project licenses this file to you under the Apache License,
5    * version 2.0 (the "License"); you may not use this file except in compliance
6    * with the License. You may obtain a copy of the License at:
7    *
8    *   https://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13   * License for the specific language governing permissions and limitations
14   * under the License.
15   */
16  package io.netty.handler.codec.compression;
17  
18  import io.netty.buffer.ByteBuf;
19  import io.netty.buffer.ByteBufAllocator;
20  import io.netty.buffer.ByteBufUtil;
21  import io.netty.util.internal.ObjectUtil;
22  import io.netty.util.internal.UnstableApi;
23  
24  import java.util.Objects;
25  import java.util.zip.CRC32;
26  import java.util.zip.DataFormatException;
27  import java.util.zip.Deflater;
28  import java.util.zip.Inflater;
29  
30  /**
31   * Decompress a {@link ByteBuf} using the inflate algorithm.
32   */
33  @UnstableApi
34  public final class JdkZlibDecompressor extends InputBufferingDecompressor {
35      private static final int FHCRC = 0x02;
36      private static final int FEXTRA = 0x04;
37      private static final int FNAME = 0x08;
38      private static final int FCOMMENT = 0x10;
39      private static final int FRESERVED = 0xE0;
40  
41      /**
42       * Smallest output buffer we hand to {@link Inflater#inflate(byte[], int, int)}. The number of remaining input
43       * bytes is only a hint for how much output to expect: the inflater may still hold decoded data that did not fit
44       * into the previous output buffer, and by then it may have consumed all input bytes already
45       * ({@link Inflater#getRemaining()} == 0). Inflating into a zero-sized buffer can never make progress.
46       */
47      private static final int MIN_OUTPUT_BUFFER_SIZE = 512;
48  
49      private Inflater inflater;
50      private final int maxAllocation;
51      private final byte[] dictionary;
52  
53      // GZIP related
54      private final ByteBufChecksum crc;
55      private final boolean decompressConcatenated;
56  
57      private enum GzipState {
58          HEADER_START,
59          HEADER_END,
60          FLG_READ,
61          XLEN_READ,
62          SKIP_FNAME,
63          SKIP_COMMENT,
64          PROCESS_FHCRC,
65          FOOTER_START,
66      }
67  
68      private GzipState gzipState = GzipState.HEADER_START;
69      private int flags = -1;
70      private int xlen = -1;
71  
72      private boolean decideZlibOrNone;
73      private boolean finished;
74      private boolean gzipMemberFinished;
75      /**
76       * If this is true, part of the input buffer is still in use by the {@link #inflater}, so we shouldn't touch that
77       * buffer too much (e.g. compact it).
78       */
79      private boolean inputBufferInInflater;
80      /**
81       * If this is true, the last {@link Inflater#inflate(byte[], int, int)} filled the output buffer completely, so
82       * the inflater may still hold decoded data even if {@link Inflater#needsInput()} reports that all input was
83       * consumed.
84       */
85      private boolean inflaterHasPendingOutput;
86  
87      JdkZlibDecompressor(Builder builder, ByteBufAllocator allocator) {
88          super(allocator);
89          maxAllocation = builder.maxAllocation;
90          dictionary = builder.dictionary;
91          this.decompressConcatenated = builder.decompressConcatenated;
92          switch (builder.wrapper) {
93              case GZIP:
94                  inflater = new Inflater(true);
95                  crc = ByteBufChecksum.wrapChecksum(new CRC32());
96                  break;
97              case NONE:
98                  inflater = new Inflater(true);
99                  crc = null;
100                 break;
101             case ZLIB:
102                 inflater = new Inflater();
103                 crc = null;
104                 break;
105             case ZLIB_OR_NONE:
106                 // Postpone the decision until decode(...) is called.
107                 decideZlibOrNone = true;
108                 crc = null;
109                 break;
110             default:
111                 throw new IllegalArgumentException("Only GZIP or ZLIB is supported, but you used " + builder.wrapper);
112         }
113     }
114 
115     @Override
116     public Status status() throws DecompressionException {
117         if (finished) {
118             return Status.COMPLETE;
119         } else if (inflaterHasPendingOutput) {
120             return Status.NEED_OUTPUT;
121         } else if (inflater == null || inflater.needsInput() || gzipState == GzipState.FOOTER_START) {
122             return Status.NEED_INPUT;
123         } else {
124             return Status.NEED_OUTPUT;
125         }
126     }
127 
128     @Override
129     public void endOfInput() throws DecompressionException {
130         if (finished) {
131             return;
132         }
133         if (crc != null && decompressConcatenated && gzipMemberFinished &&
134                 gzipState == GzipState.HEADER_START && available() == 0) {
135             finished = true;
136             return;
137         }
138         throw new DecompressionException("Compressed stream ended before the end-of-stream marker");
139     }
140 
141     @Override
142     void processInput(ByteBuf buf) throws DecompressionException {
143         if (inputBufferInInflater) {
144             throw new IllegalStateException("Not in state NEED_INPUT");
145         }
146 
147         int readableBytes = buf.readableBytes();
148         if (readableBytes == 0) {
149             return;
150         }
151 
152         if (decideZlibOrNone) {
153             // First two bytes are needed to decide if it's a ZLIB stream.
154             if (readableBytes < 2) {
155                 return;
156             }
157 
158             boolean nowrap = !looksLikeZlib(buf.getShort(buf.readerIndex()));
159             inflater = new Inflater(nowrap);
160             decideZlibOrNone = false;
161         }
162 
163         if (crc != null) {
164             if (gzipState != GzipState.HEADER_END) {
165                 if (gzipState == GzipState.FOOTER_START) {
166                     if (!handleGzipFooter(buf)) {
167                         // Either there was not enough data or the input is finished.
168                         return;
169                     }
170                     // If we consumed the footer we will start with the header again.
171                     assert gzipState == GzipState.HEADER_START;
172                 }
173                 if (!readGZIPHeader(buf)) {
174                     // There was not enough data readable to read the GZIP header.
175                     return;
176                 }
177                 // Some bytes may have been consumed, and so we must re-set the number of readable bytes.
178                 readableBytes = buf.readableBytes();
179                 if (readableBytes == 0) {
180                     return;
181                 }
182             }
183         }
184 
185         if (inflater.needsInput()) {
186             if (buf.hasArray()) {
187                 inflater.setInput(buf.array(), buf.arrayOffset() + buf.readerIndex(), readableBytes);
188             } else {
189                 inflater.setInput(ByteBufUtil.getBytes(buf));
190             }
191             inputBufferInInflater = true;
192         }
193     }
194 
195     private void consumeInput(ByteBuf buf) {
196         if (inputBufferInInflater) {
197             buf.readerIndex(buf.writerIndex() - inflater.getRemaining());
198             if (inflater.getRemaining() == 0) {
199                 inputBufferInInflater = false;
200             }
201         }
202     }
203 
204     @Override
205     ByteBuf processOutput(ByteBuf buf) throws DecompressionException {
206         int proposedCapacity = Math.max(inflater.getRemaining() << 1, MIN_OUTPUT_BUFFER_SIZE);
207         int targetCapacity = maxAllocation == 0
208                 ? proposedCapacity : Math.min(maxAllocation, proposedCapacity);
209         ByteBuf decompressed = allocator.heapBuffer(targetCapacity);
210         boolean success = false;
211         try {
212             byte[] outArray = decompressed.array();
213             int writerIndex = decompressed.writerIndex();
214             int outIndex = decompressed.arrayOffset() + writerIndex;
215             int writableBytes = decompressed.writableBytes();
216             int outputLength;
217             try {
218                 outputLength = inflater.inflate(outArray, outIndex, writableBytes);
219             } catch (DataFormatException e) {
220                 throw new DecompressionException("decompression failure", e);
221             }
222             consumeInput(buf);
223             if (outputLength > 0) {
224                 decompressed.writerIndex(writerIndex + outputLength);
225                 if (crc != null) {
226                     crc.update(outArray, outIndex, outputLength);
227                 }
228             }
229             if (inflater.needsDictionary()) {
230                 if (dictionary == null) {
231                     throw new DecompressionException(
232                             "decompression failure, unable to set dictionary as none was specified");
233                 }
234                 try {
235                     inflater.setDictionary(dictionary);
236                 } catch (IllegalArgumentException e) {
237                     // The dictionary ID in the stream header does not match the configured dictionary.
238                     throw new DecompressionException("decompression failure, dictionary mismatch", e);
239                 }
240             }
241             if (inflater.finished()) {
242                 inputBufferInInflater = false;
243                 if (crc == null) {
244                     finished = true; // Do not decode anymore.
245                 } else {
246                     gzipState = GzipState.FOOTER_START;
247                     // potentially consume footer
248                     processInput(buf);
249                 }
250             }
251             // If we filled the whole buffer the inflater may still have decoded data left that it could not write.
252             // In that case we must ask for another output buffer, as needsInput() only tells us that all input
253             // bytes were consumed, not that all output was produced.
254             inflaterHasPendingOutput = outputLength == writableBytes && !inflater.finished();
255             success = true;
256             return decompressed;
257         } finally {
258             if (!success) {
259                 decompressed.release();
260             }
261         }
262     }
263 
264     private boolean handleGzipFooter(ByteBuf in) {
265         if (readGZIPFooter(in)) {
266             gzipMemberFinished = true;
267             finished = !decompressConcatenated;
268 
269             if (!finished) {
270                 inflater.reset();
271                 crc.reset();
272                 xlen = -1;
273                 gzipState = GzipState.HEADER_START;
274                 return true;
275             }
276         }
277         return false;
278     }
279 
280     private boolean readGZIPHeader(ByteBuf in) {
281         switch (gzipState) {
282             case HEADER_START:
283                 if (in.readableBytes() < 10) {
284                     return false;
285                 }
286                 // read magic numbers
287                 int magic0 = in.readUnsignedByte();
288                 int magic1 = in.readUnsignedByte();
289 
290                 if (magic0 != 31 || magic1 != 139) {
291                     throw new DecompressionException("Input is not in the GZIP format");
292                 }
293                 crc.update(magic0);
294                 crc.update(magic1);
295 
296                 int method = in.readUnsignedByte();
297                 if (method != Deflater.DEFLATED) {
298                     throw new DecompressionException("Unsupported compression method "
299                             + method + " in the GZIP header");
300                 }
301                 crc.update(method);
302 
303                 flags = in.readUnsignedByte();
304                 crc.update(flags);
305 
306                 if ((flags & FRESERVED) != 0) {
307                     throw new DecompressionException(
308                             "Reserved flags are set in the GZIP header");
309                 }
310 
311                 // mtime (int)
312                 crc.update(in, in.readerIndex(), 4);
313                 in.skipBytes(4);
314 
315                 crc.update(in.readUnsignedByte()); // extra flags
316                 crc.update(in.readUnsignedByte()); // operating system
317 
318                 gzipState = GzipState.FLG_READ;
319                 // fall through
320             case FLG_READ:
321                 if ((flags & FEXTRA) != 0) {
322                     if (in.readableBytes() < 2) {
323                         return false;
324                     }
325                     int xlen1 = in.readUnsignedByte();
326                     int xlen2 = in.readUnsignedByte();
327                     crc.update(xlen1);
328                     crc.update(xlen2);
329 
330                     xlen = xlen2 << 8 | xlen1;
331                 }
332                 gzipState = GzipState.XLEN_READ;
333                 // fall through
334             case XLEN_READ:
335                 if (xlen != -1) {
336                     if (in.readableBytes() < xlen) {
337                         return false;
338                     }
339                     crc.update(in, in.readerIndex(), xlen);
340                     in.skipBytes(xlen);
341                 }
342                 gzipState = GzipState.SKIP_FNAME;
343                 // fall through
344             case SKIP_FNAME:
345                 if (!skipIfNeeded(in, FNAME)) {
346                     return false;
347                 }
348                 gzipState = GzipState.SKIP_COMMENT;
349                 // fall through
350             case SKIP_COMMENT:
351                 if (!skipIfNeeded(in, FCOMMENT)) {
352                     return false;
353                 }
354                 gzipState = GzipState.PROCESS_FHCRC;
355                 // fall through
356             case PROCESS_FHCRC:
357                 if ((flags & FHCRC) != 0) {
358                     if (!verifyCrc16(in)) {
359                         return false;
360                     }
361                 }
362                 crc.reset();
363                 gzipState = GzipState.HEADER_END;
364                 // fall through
365             case HEADER_END:
366                 return true;
367             default:
368                 throw new IllegalStateException();
369         }
370     }
371 
372     /**
373      * Skip bytes in the input if needed until we find the end marker {@code 0x00}.
374      * @param   in the input
375      * @param   flagMask the mask that should be present in the {@code flags} when we need to skip bytes.
376      * @return  {@code true} if the operation is complete and we can move to the next state, {@code false} if we need
377      *          the retry again once we have more readable bytes.
378      */
379     private boolean skipIfNeeded(ByteBuf in, int flagMask) {
380         if ((flags & flagMask) != 0) {
381             for (;;) {
382                 if (!in.isReadable()) {
383                     // We didnt find the end yet, need to retry again once more data is readable
384                     return false;
385                 }
386                 int b = in.readUnsignedByte();
387                 crc.update(b);
388                 if (b == 0x00) {
389                     break;
390                 }
391             }
392         }
393         // Skip is handled, we can move to the next processing state.
394         return true;
395     }
396 
397     /**
398      * Read the GZIP footer.
399      *
400      * @param   in the input.
401      * @return  {@code true} if the footer could be read, {@code false} if the read could not be performed as
402      *          the input {@link ByteBuf} doesn't have enough readable bytes (8 bytes).
403      */
404     private boolean readGZIPFooter(ByteBuf in) {
405         if (in.readableBytes() < 8) {
406             return false;
407         }
408 
409         boolean enoughData = verifyCrc(in);
410         assert enoughData;
411 
412         // read ISIZE and verify
413         int dataLength = in.readIntLE();
414         int readLength = inflater.getTotalOut();
415         if (dataLength != readLength) {
416             throw new DecompressionException(
417                     "Number of bytes mismatch. Expected: " + dataLength + ", Got: " + readLength);
418         }
419         return true;
420     }
421 
422     /**
423      * Verifies CRC.
424      *
425      * @param   in the input.
426      * @return  {@code true} if verification could be performed, {@code false} if verification could not be performed as
427      *          the input {@link ByteBuf} doesn't have enough readable bytes (4 bytes).
428      */
429     private boolean verifyCrc(ByteBuf in) {
430         if (in.readableBytes() < 4) {
431             return false;
432         }
433         long crcValue = in.readUnsignedIntLE();
434 
435         long readCrc = crc.getValue();
436         if (crcValue != readCrc) {
437             throw new DecompressionException(
438                     "CRC value mismatch. Expected: " + crcValue + ", Got: " + readCrc);
439         }
440         return true;
441     }
442 
443     private boolean verifyCrc16(ByteBuf in) {
444         if (in.readableBytes() < 2) {
445             return false;
446         }
447 
448         int crc16Value = in.readUnsignedShortLE();
449         // the two least significant bytes from the CRC32
450         int readCrc16 = (int) (crc.getValue() & 0xFFFF);
451 
452         if (crc16Value != readCrc16) {
453             throw new DecompressionException(
454                     "CRC16 value mismatch. Expected: " + crc16Value + ", Got: " + readCrc16);
455         }
456         return true;
457     }
458 
459     /*
460      * Returns true if the cmf_flg parameter (think: first two bytes of a zlib stream)
461      * indicates that this is a zlib stream.
462      * <p>
463      * You can lookup the details in the ZLIB RFC:
464      * <a href="https://tools.ietf.org/html/rfc1950#section-2.2">RFC 1950</a>.
465      */
466     private static boolean looksLikeZlib(short cmf_flg) {
467         return (cmf_flg & 0x7800) == 0x7800 &&
468                 cmf_flg % 31 == 0;
469     }
470 
471     @Override
472     public void close() {
473         try {
474             super.close();
475         } finally {
476             if (inflater != null) {
477                 inflater.end();
478                 inflater = null;
479             }
480         }
481     }
482 
483     @UnstableApi
484     public static Builder builder() {
485         return new Builder();
486     }
487 
488     @UnstableApi
489     public static final class Builder extends Decompressor.AbstractDecompressorBuilder {
490         private ZlibWrapper wrapper = ZlibWrapper.ZLIB;
491         private byte[] dictionary;
492         private int maxAllocation = 1024 * 1024;
493         boolean decompressConcatenated;
494 
495         Builder() {
496         }
497 
498         /**
499          * Set the wrapper format for the deflated data. Defaults to {@link ZlibWrapper#ZLIB}.
500          *
501          * @param wrapper The wrapper format
502          * @return This builder
503          */
504         public Builder wrapper(ZlibWrapper wrapper) {
505             this.wrapper = Objects.requireNonNull(wrapper, "wrapper");
506             return this;
507         }
508 
509         /**
510          * Set the preset dictionary to use. Defaults to no dictionary.
511          *
512          * @param dictionary The dictionary
513          * @return This builder
514          */
515         public Builder dictionary(byte[] dictionary) {
516             this.dictionary = dictionary;
517             return this;
518         }
519 
520         /**
521          * Set the maximum output buffer size. Defaults to 1M.
522          *
523          * @param maxAllocation The maximum output buffer size.
524          * @return This builder
525          */
526         public Builder maxAllocation(int maxAllocation) {
527             this.maxAllocation = ObjectUtil.checkPositiveOrZero(maxAllocation, "maxAllocation");
528             return this;
529         }
530 
531         public Builder decompressConcatenated(boolean decompressConcatenated) {
532             this.decompressConcatenated = decompressConcatenated;
533             return this;
534         }
535 
536         @Override
537         public Decompressor build(ByteBufAllocator allocator) throws DecompressionException {
538             return new DefensiveDecompressor(new JdkZlibDecompressor(this, allocator));
539         }
540     }
541 }