View Javadoc
1   /*
2    * Copyright 2025 The Netty Project
3    *
4    * The Netty Project licenses this file to you under the Apache License,
5    * version 2.0 (the "License"); you may not use this file except in compliance
6    * with the License. You may obtain a copy of the License at:
7    *
8    *   https://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13   * License for the specific language governing permissions and limitations
14   * under the License.
15   */
16  package io.netty.handler.codec.compression;
17  
18  import io.netty.buffer.ByteBuf;
19  import io.netty.buffer.ByteBufAllocator;
20  import io.netty.util.internal.ObjectUtil;
21  import io.netty.util.internal.UnstableApi;
22  import net.jpountz.lz4.LZ4Exception;
23  import net.jpountz.lz4.LZ4Factory;
24  import net.jpountz.lz4.LZ4SafeDecompressor;
25  
26  import java.nio.ByteBuffer;
27  import java.util.zip.Checksum;
28  
29  import static io.netty.handler.codec.compression.Lz4Constants.BLOCK_TYPE_COMPRESSED;
30  import static io.netty.handler.codec.compression.Lz4Constants.BLOCK_TYPE_NON_COMPRESSED;
31  import static io.netty.handler.codec.compression.Lz4Constants.COMPRESSION_LEVEL_BASE;
32  import static io.netty.handler.codec.compression.Lz4Constants.DEFAULT_SEED;
33  import static io.netty.handler.codec.compression.Lz4Constants.HEADER_LENGTH;
34  import static io.netty.handler.codec.compression.Lz4Constants.MAGIC_NUMBER;
35  import static io.netty.handler.codec.compression.Lz4Constants.MAX_BLOCK_SIZE;
36  
37  /**
38   * Uncompresses a {@link ByteBuf} encoded with the LZ4 format.
39   *
40   * See original <a href="https://github.com/Cyan4973/lz4">LZ4 Github project</a>
41   * and <a href="https://fastcompression.blogspot.ru/2011/05/lz4-explained.html">LZ4 block format</a>
42   * for full description.
43   *
44   * Since the original LZ4 block format does not contains size of compressed block and size of original data
45   * this encoder uses format like <a href="https://github.com/idelpivnitskiy/lz4-java">LZ4 Java</a> library
46   * written by Adrien Grand and approved by Yann Collet (author of original LZ4 library).
47   *
48   *  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *     * * * * * * * * * *
49   *  * Magic * Token *  Compressed *  Decompressed *  Checksum *  +  *  LZ4 compressed *
50   *  *       *       *    length   *     length    *           *     *      block      *
51   *  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *     * * * * * * * * * *
52   */
53  @UnstableApi
54  public final class Lz4FrameDecompressor extends InputBufferingDecompressor {
55      private static final int DEFAULT_MAX_DECOMPRESSED_LENGTH = 256 * 1024;
56  
57      /**
58       * Current state of stream.
59       */
60      private enum State {
61          INIT_BLOCK,
62          DECOMPRESS_DATA,
63          FINISHED,
64      }
65  
66      private State currentState = State.INIT_BLOCK;
67  
68      /**
69       * Underlying decompressor in use.
70       */
71      private LZ4SafeDecompressor decompressor;
72  
73      /**
74       * Underlying checksum calculator in use.
75       */
76      private ByteBufChecksum checksum;
77  
78      private final int maxDecompressedLength;
79  
80      /**
81       * Type of current block.
82       */
83      private int blockType;
84  
85      /**
86       * Compressed length of current incoming block.
87       */
88      private int compressedLength;
89  
90      /**
91       * Decompressed length of current incoming block.
92       */
93      private int decompressedLength;
94  
95      /**
96       * Checksum value of current incoming block.
97       */
98      private int currentChecksum;
99  
100     Lz4FrameDecompressor(Builder builder, ByteBufAllocator allocator) {
101         super(allocator);
102         this.decompressor = builder.factory.safeDecompressor();
103         this.checksum = builder.checksum == null ? null : ByteBufChecksum.wrapChecksum(builder.checksum);
104         this.maxDecompressedLength = builder.maxDecompressedLength;
105     }
106 
107     @Override
108     void processInput(ByteBuf buf) throws DecompressionException {
109         if (currentState != State.INIT_BLOCK) {
110             return;
111         }
112 
113         if (buf.readableBytes() < HEADER_LENGTH) {
114             return;
115         }
116         final long magic = buf.readLong();
117         if (magic != MAGIC_NUMBER) {
118             throw new DecompressionException("unexpected block identifier");
119         }
120 
121         final int token = buf.readByte();
122         final int compressionLevel = (token & 0x0F) + COMPRESSION_LEVEL_BASE;
123         int blockType = token & 0xF0;
124 
125         int compressedLength = Integer.reverseBytes(buf.readInt());
126         if (compressedLength < 0 || compressedLength > MAX_BLOCK_SIZE) {
127             throw new DecompressionException(String.format(
128                     "invalid compressedLength: %d (expected: 0-%d)",
129                     compressedLength, MAX_BLOCK_SIZE));
130         }
131 
132         int decompressedLength = Integer.reverseBytes(buf.readInt());
133         if (decompressedLength > maxDecompressedLength) {
134             throw new DecompressionException(String.format(
135                     "decompressedLength too large: %d (expected: 0-%d)",
136                     decompressedLength, maxDecompressedLength));
137         }
138 
139         final int maxLocalDecompressedLength = 1 << compressionLevel;
140         if (decompressedLength < 0 || decompressedLength > maxLocalDecompressedLength) {
141             throw new DecompressionException(String.format(
142                     "invalid decompressedLength: %d (expected: 0-%d)",
143                     decompressedLength, maxLocalDecompressedLength));
144         }
145         if (decompressedLength == 0 && compressedLength != 0
146                 || decompressedLength != 0 && compressedLength == 0
147                 || blockType == BLOCK_TYPE_NON_COMPRESSED && decompressedLength != compressedLength) {
148             throw new DecompressionException(String.format(
149                     "stream corrupted: compressedLength(%d) and decompressedLength(%d) mismatch",
150                     compressedLength, decompressedLength));
151         }
152 
153         int currentChecksum = Integer.reverseBytes(buf.readInt());
154         if (decompressedLength == 0 && compressedLength == 0) {
155             if (currentChecksum != 0) {
156                 throw new DecompressionException("stream corrupted: checksum error");
157             }
158             currentState = State.FINISHED;
159             decompressor = null;
160             checksum = null;
161             return;
162         }
163 
164         this.blockType = blockType;
165         this.compressedLength = compressedLength;
166         this.decompressedLength = decompressedLength;
167         this.currentChecksum = currentChecksum;
168 
169         currentState = State.DECOMPRESS_DATA;
170     }
171 
172     @Override
173     public Status status() throws DecompressionException {
174         switch (currentState) {
175             case INIT_BLOCK:
176                 return Status.NEED_INPUT;
177             case DECOMPRESS_DATA:
178                 return available() < compressedLength ? Status.NEED_INPUT : Status.NEED_OUTPUT;
179             case FINISHED:
180                 return Status.COMPLETE;
181             default:
182                 throw new AssertionError("Unexpected state: " + currentState);
183         }
184     }
185 
186     @Override
187     public void endOfInput() throws DecompressionException {
188         throw new DecompressionException("Unexpected end of input");
189     }
190 
191     @Override
192     ByteBuf processOutput(ByteBuf in) throws DecompressionException {
193         ByteBuf uncompressed = null;
194         try {
195             switch (blockType) {
196                 case BLOCK_TYPE_NON_COMPRESSED:
197                     // Just pass through, we not update the readerIndex yet as we do this outside of the
198                     // switch statement.
199                     uncompressed = in.retainedSlice(in.readerIndex(), decompressedLength);
200                     break;
201                 case BLOCK_TYPE_COMPRESSED:
202                     uncompressed = allocator.buffer(decompressedLength, decompressedLength);
203 
204                     ByteBuffer inBuffer = CompressionUtil.safeNioBuffer(
205                             in, in.readerIndex(), compressedLength);
206                     ByteBuffer outBuffer = uncompressed.internalNioBuffer(
207                             uncompressed.writerIndex(), decompressedLength);
208                     if (inBuffer.remaining() < compressedLength || outBuffer.remaining() < decompressedLength) {
209                         throw new DecompressionException(String.format(
210                                 "buffer lengths too small: compressed(%d/%d), decompressed(%d/%d)",
211                                 inBuffer.remaining(), compressedLength,
212                                 outBuffer.remaining(), decompressedLength));
213                     }
214                     final int actualDecompressedLength;
215                     try {
216                         actualDecompressedLength = decompressor.decompress(
217                                 inBuffer, inBuffer.position(), compressedLength,
218                                 outBuffer, outBuffer.position(), decompressedLength);
219                     } catch (LZ4Exception e) {
220                         throw new DecompressionException(e);
221                     }
222                     if (actualDecompressedLength != decompressedLength) {
223                         throw new DecompressionException(String.format(
224                                 "stream corrupted: decompressedLength(%d) and actual length(%d) mismatch",
225                                 decompressedLength, actualDecompressedLength));
226                     }
227                     // Update the writerIndex now to reflect what we decompressed.
228                     uncompressed.writerIndex(uncompressed.writerIndex() + actualDecompressedLength);
229                     break;
230                 default:
231                     throw new DecompressionException(String.format(
232                             "unexpected blockType: %d (expected: %d or %d)",
233                             blockType, BLOCK_TYPE_NON_COMPRESSED, BLOCK_TYPE_COMPRESSED));
234             }
235             // Skip inbound bytes after we processed them.
236             in.skipBytes(compressedLength);
237             if (checksum != null) {
238                 CompressionUtil.checkChecksum(checksum, uncompressed, currentChecksum);
239             }
240             currentState = State.INIT_BLOCK;
241             return uncompressed;
242         } catch (Throwable t) {
243             if (uncompressed != null) {
244                 uncompressed.release();
245             }
246             throw t;
247         }
248     }
249 
250     @UnstableApi
251     public static Builder builder() {
252         return new Builder();
253     }
254 
255     @UnstableApi
256     public static final class Builder extends AbstractDecompressorBuilder {
257         private LZ4Factory factory = LZ4Factory.fastestInstance();
258         private Checksum checksum;
259         private int maxDecompressedLength = DEFAULT_MAX_DECOMPRESSED_LENGTH;
260 
261         Builder() {
262         }
263 
264         /**
265          * User customizable {@link LZ4Factory} instance
266          * which may be JNI bindings to the original C implementation, a pure Java implementation
267          * or a Java implementation that uses the {@link sun.misc.Unsafe}.
268          *
269          * @param factory The factory to use
270          * @return This builder
271          */
272         public Builder factory(LZ4Factory factory) {
273             this.factory = ObjectUtil.checkNotNull(factory, "factory");
274             return this;
275         }
276 
277         /**
278          * The {@link Checksum} instance to use to check data for integrity. By default, no checksum validation is
279          * performed.
280          *
281          * @param checksum The checksum to use
282          * @return This builder
283          */
284         public Builder checksum(Checksum checksum) {
285             this.checksum = checksum;
286             return this;
287         }
288 
289         /**
290          * Enable checksum validation using the default checksum, xxhash.
291          *
292          * @return This builder
293          */
294         public Builder defaultChecksum() {
295             return checksum(new Lz4XXHash32(DEFAULT_SEED));
296         }
297 
298         /**
299          * Set the maximum allowed decompressed block length. The default is 256 KiB. Use {@code 0} to allow the
300          * LZ4 format maximum of 32 MiB.
301          *
302          * @param maxDecompressedLength maximum decompressed block length
303          * @return This builder
304          */
305         public Builder maxDecompressedLength(int maxDecompressedLength) {
306             this.maxDecompressedLength = maxDecompressedLength == 0 ? MAX_BLOCK_SIZE :
307                     ObjectUtil.checkInRange(maxDecompressedLength, 0, MAX_BLOCK_SIZE, "maxDecompressedLength");
308             return this;
309         }
310 
311         @Override
312         public Decompressor build(ByteBufAllocator allocator) throws DecompressionException {
313             return new DefensiveDecompressor(new Lz4FrameDecompressor(this, allocator));
314         }
315     }
316 }