View Javadoc
1   /*
2    * Copyright 2025 The Netty Project
3    *
4    * The Netty Project licenses this file to you under the Apache License,
5    * version 2.0 (the "License"); you may not use this file except in compliance
6    * with the License. You may obtain a copy of the License at:
7    *
8    *   https://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13   * License for the specific language governing permissions and limitations
14   * under the License.
15   */
16  package io.netty.handler.codec.compression;
17  
18  import io.netty.buffer.ByteBuf;
19  import io.netty.buffer.ByteBufAllocator;
20  import io.netty.util.internal.ObjectUtil;
21  import io.netty.util.internal.UnstableApi;
22  
23  import static io.netty.handler.codec.compression.Bzip2Constants.BASE_BLOCK_SIZE;
24  import static io.netty.handler.codec.compression.Bzip2Constants.BLOCK_HEADER_MAGIC_1;
25  import static io.netty.handler.codec.compression.Bzip2Constants.BLOCK_HEADER_MAGIC_2;
26  import static io.netty.handler.codec.compression.Bzip2Constants.END_OF_STREAM_MAGIC_1;
27  import static io.netty.handler.codec.compression.Bzip2Constants.END_OF_STREAM_MAGIC_2;
28  import static io.netty.handler.codec.compression.Bzip2Constants.HUFFMAN_MAXIMUM_TABLES;
29  import static io.netty.handler.codec.compression.Bzip2Constants.HUFFMAN_MAX_ALPHABET_SIZE;
30  import static io.netty.handler.codec.compression.Bzip2Constants.HUFFMAN_MINIMUM_TABLES;
31  import static io.netty.handler.codec.compression.Bzip2Constants.HUFFMAN_SELECTOR_LIST_MAX_LENGTH;
32  import static io.netty.handler.codec.compression.Bzip2Constants.HUFFMAN_SYMBOL_RANGE_SIZE;
33  import static io.netty.handler.codec.compression.Bzip2Constants.MAGIC_NUMBER;
34  import static io.netty.handler.codec.compression.Bzip2Constants.MAX_BLOCK_SIZE;
35  import static io.netty.handler.codec.compression.Bzip2Constants.MAX_SELECTORS;
36  import static io.netty.handler.codec.compression.Bzip2Constants.MIN_BLOCK_SIZE;
37  
38  /**
39   * Uncompresses a {@link ByteBuf} encoded with the Bzip2 format.
40   *
41   * See <a href="https://en.wikipedia.org/wiki/Bzip2">Bzip2</a>.
42   */
43  @UnstableApi
44  public final class Bzip2Decompressor extends InputBufferingDecompressor {
45      private final int outputBufferSize;
46  
47      /**
48       * Current state of stream.
49       */
50      private enum State {
51          INIT,
52          INIT_BLOCK,
53          INIT_BLOCK_PARAMS,
54          RECEIVE_HUFFMAN_USED_MAP,
55          RECEIVE_HUFFMAN_USED_BITMAPS,
56          RECEIVE_SELECTORS_NUMBER,
57          RECEIVE_SELECTORS,
58          RECEIVE_HUFFMAN_LENGTH,
59          DECODE_HUFFMAN_DATA,
60          DECODED_HUFFMAN_DATA,
61          EOF
62      }
63      private State currentState = State.INIT;
64  
65      /**
66       * A reader that provides bit-level reads.
67       */
68      private final Bzip2BitReader reader = new Bzip2BitReader();
69  
70      /**
71       * The decompressor for the current block.
72       */
73      private Bzip2BlockDecompressor blockDecompressor;
74  
75      /**
76       * Bzip2 Huffman coding stage.
77       */
78      private Bzip2HuffmanStageDecoder huffmanStageDecoder;
79  
80      /**
81       * Always: in the range 0 .. 9. The current block size is 100000 * this number.
82       */
83      private int blockSize;
84  
85      /**
86       * The CRC of the current block as read from the block header.
87       */
88      private int blockCRC;
89  
90      /**
91       * The merged CRC of all blocks decompressed so far.
92       */
93      private int streamCRC;
94  
95      Bzip2Decompressor(Builder builder, ByteBufAllocator allocator) {
96          super(allocator);
97          this.outputBufferSize = builder.outputBufferSize;
98      }
99  
100     @Override
101     void processInput(ByteBuf buf) throws DecompressionException {
102         reader.setByteBuf(buf);
103         switch (currentState) {
104             case INIT:
105                 if (buf.readableBytes() < 4) {
106                     break;
107                 }
108                 int magicNumber = buf.readUnsignedMedium();
109                 if (magicNumber != MAGIC_NUMBER) {
110                     throw new DecompressionException("Unexpected stream identifier contents. Mismatched bzip2 " +
111                             "protocol version?");
112                 }
113                 int blockSize = buf.readByte() - '0';
114                 if (blockSize < MIN_BLOCK_SIZE || blockSize > MAX_BLOCK_SIZE) {
115                     throw new DecompressionException("block size is invalid");
116                 }
117                 this.blockSize = blockSize * BASE_BLOCK_SIZE;
118 
119                 streamCRC = 0;
120                 currentState = State.INIT_BLOCK;
121                 // fall through
122             case INIT_BLOCK:
123                 if (!reader.hasReadableBytes(10)) {
124                     break;
125                 }
126                 // Get the block magic bytes.
127                 final int magic1 = reader.readBits(24);
128                 final int magic2 = reader.readBits(24);
129                 if (magic1 == END_OF_STREAM_MAGIC_1 && magic2 == END_OF_STREAM_MAGIC_2) {
130                     // End of stream was reached. Check the combined CRC.
131                     final int storedCombinedCRC = reader.readInt();
132                     if (storedCombinedCRC != streamCRC) {
133                         throw new DecompressionException("stream CRC error");
134                     }
135                     currentState = State.EOF;
136                     break;
137                 }
138                 if (magic1 != BLOCK_HEADER_MAGIC_1 || magic2 != BLOCK_HEADER_MAGIC_2) {
139                     throw new DecompressionException("bad block header");
140                 }
141                 blockCRC = reader.readInt();
142                 currentState = State.INIT_BLOCK_PARAMS;
143                 // fall through
144             case INIT_BLOCK_PARAMS:
145                 if (!reader.hasReadableBits(25)) {
146                     break;
147                 }
148                 final boolean blockRandomised = reader.readBoolean();
149                 final int bwtStartPointer = reader.readBits(24);
150 
151                 blockDecompressor = new Bzip2BlockDecompressor(this.blockSize, blockCRC,
152                         blockRandomised, bwtStartPointer, reader);
153                 currentState = State.RECEIVE_HUFFMAN_USED_MAP;
154                 // fall through
155             case RECEIVE_HUFFMAN_USED_MAP:
156                 if (!reader.hasReadableBits(16)) {
157                     break;
158                 }
159                 blockDecompressor.huffmanInUse16 = reader.readBits(16);
160                 currentState = State.RECEIVE_HUFFMAN_USED_BITMAPS;
161                 // fall through
162             case RECEIVE_HUFFMAN_USED_BITMAPS:
163                 Bzip2BlockDecompressor blockDecompressor = this.blockDecompressor;
164                 final int inUse16 = blockDecompressor.huffmanInUse16;
165                 final int bitNumber = Integer.bitCount(inUse16);
166                 final byte[] huffmanSymbolMap = blockDecompressor.huffmanSymbolMap;
167 
168                 if (!reader.hasReadableBits(bitNumber * HUFFMAN_SYMBOL_RANGE_SIZE + 3)) {
169                     break;
170                 }
171 
172                 int huffmanSymbolCount = 0;
173                 if (bitNumber > 0) {
174                     for (int i = 0; i < 16; i++) {
175                         if ((inUse16 & 1 << 15 >>> i) != 0) {
176                             for (int j = 0, k = i << 4; j < HUFFMAN_SYMBOL_RANGE_SIZE; j++, k++) {
177                                 if (reader.readBoolean()) {
178                                     huffmanSymbolMap[huffmanSymbolCount++] = (byte) k;
179                                 }
180                             }
181                         }
182                     }
183                 }
184                 blockDecompressor.huffmanEndOfBlockSymbol = huffmanSymbolCount + 1;
185 
186                 int totalTables = reader.readBits(3);
187                 if (totalTables < HUFFMAN_MINIMUM_TABLES || totalTables > HUFFMAN_MAXIMUM_TABLES) {
188                     throw new DecompressionException("incorrect huffman groups number");
189                 }
190                 int alphaSize = huffmanSymbolCount + 2;
191                 if (alphaSize > HUFFMAN_MAX_ALPHABET_SIZE) {
192                     throw new DecompressionException("incorrect alphabet size");
193                 }
194                 huffmanStageDecoder = new Bzip2HuffmanStageDecoder(reader, totalTables, alphaSize);
195                 currentState = State.RECEIVE_SELECTORS_NUMBER;
196                 // fall through
197             case RECEIVE_SELECTORS_NUMBER:
198                 if (!reader.hasReadableBits(15)) {
199                     break;
200                 }
201                 int totalSelectors = reader.readBits(15);
202                 if (totalSelectors < 1 || totalSelectors > MAX_SELECTORS) {
203                     throw new DecompressionException("incorrect selectors number");
204                 }
205                 huffmanStageDecoder.selectors = new byte[totalSelectors];
206 
207                 currentState = State.RECEIVE_SELECTORS;
208                 // fall through
209             case RECEIVE_SELECTORS:
210                 Bzip2HuffmanStageDecoder huffmanStageDecoder = this.huffmanStageDecoder;
211                 byte[] selectors = huffmanStageDecoder.selectors;
212                 totalSelectors = selectors.length;
213                 final Bzip2MoveToFrontTable tableMtf = huffmanStageDecoder.tableMTF;
214 
215                 int currSelector;
216                 // Get zero-terminated bit runs (0..62) of MTF'ed Huffman table. length = 1..6
217                 for (currSelector = huffmanStageDecoder.currentSelector;
218                      currSelector < totalSelectors; currSelector++) {
219                     if (!reader.hasReadableBits(HUFFMAN_SELECTOR_LIST_MAX_LENGTH)) {
220                         // Save state if end of current ByteBuf was reached
221                         huffmanStageDecoder.currentSelector = currSelector;
222                         return;
223                     }
224                     int index = 0;
225                     while (reader.readBoolean()) {
226                         index++;
227                         if (index >= huffmanStageDecoder.totalTables) {
228                             throw new DecompressionException("incorrect selector index " + index);
229                         }
230                     }
231                     selectors[currSelector] = tableMtf.indexToFront(index);
232                 }
233 
234                 currentState = State.RECEIVE_HUFFMAN_LENGTH;
235                 // fall through
236             case RECEIVE_HUFFMAN_LENGTH:
237                 huffmanStageDecoder = this.huffmanStageDecoder;
238                 totalTables = huffmanStageDecoder.totalTables;
239                 final byte[][] codeLength = huffmanStageDecoder.tableCodeLengths;
240                 alphaSize = huffmanStageDecoder.alphabetSize;
241 
242                 /* Now the coding tables */
243                 int currGroup;
244                 int currLength = huffmanStageDecoder.currentLength;
245                 int currAlpha = 0;
246                 boolean modifyLength = huffmanStageDecoder.modifyLength;
247                 boolean saveStateAndReturn = false;
248                 loop: for (currGroup = huffmanStageDecoder.currentGroup; currGroup < totalTables; currGroup++) {
249                     // start_huffman_length
250                     if (!reader.hasReadableBits(5)) {
251                         saveStateAndReturn = true;
252                         break;
253                     }
254                     if (currLength < 0) {
255                         currLength = reader.readBits(5);
256                     }
257                     for (currAlpha = huffmanStageDecoder.currentAlpha; currAlpha < alphaSize; currAlpha++) {
258                         // delta_bit_length: 1..40
259                         if (!reader.isReadable()) {
260                             saveStateAndReturn = true;
261                             break loop;
262                         }
263                         while (modifyLength || reader.readBoolean()) {  // 0=>next symbol; 1=>alter length
264                             if (!reader.isReadable()) {
265                                 modifyLength = true;
266                                 saveStateAndReturn = true;
267                                 break loop;
268                             }
269                             // 1=>decrement length;  0=>increment length
270                             currLength += reader.readBoolean() ? -1 : 1;
271                             modifyLength = false;
272                             if (!reader.isReadable()) {
273                                 saveStateAndReturn = true;
274                                 break loop;
275                             }
276                         }
277                         codeLength[currGroup][currAlpha] = (byte) currLength;
278                     }
279                     currLength = -1;
280                     currAlpha = huffmanStageDecoder.currentAlpha = 0;
281                     modifyLength = false;
282                 }
283                 if (saveStateAndReturn) {
284                     // Save state if end of current ByteBuf was reached
285                     huffmanStageDecoder.currentGroup = currGroup;
286                     huffmanStageDecoder.currentLength = currLength;
287                     huffmanStageDecoder.currentAlpha = currAlpha;
288                     huffmanStageDecoder.modifyLength = modifyLength;
289                     break;
290                 }
291 
292                 // Finally create the Huffman tables
293                 huffmanStageDecoder.createHuffmanDecodingTables();
294                 currentState = State.DECODE_HUFFMAN_DATA;
295                 // fall through
296             case DECODE_HUFFMAN_DATA:
297                 blockDecompressor = this.blockDecompressor;
298                 final boolean decoded = blockDecompressor.decodeHuffmanData(this.huffmanStageDecoder);
299                 if (!decoded) {
300                     return;
301                 }
302                 currentState = State.DECODED_HUFFMAN_DATA;
303                 break;
304         }
305     }
306 
307     @Override
308     ByteBuf processOutput(ByteBuf buf) throws DecompressionException {
309         if (currentState != State.DECODED_HUFFMAN_DATA) {
310             throw new IllegalStateException("Not in state NEED_OUTPUT");
311         }
312         ByteBuf uncompressed = allocator.buffer(outputBufferSize);
313         try {
314             int uncByte;
315             while ((uncByte = blockDecompressor.read()) >= 0) {
316                 uncompressed.writeByte(uncByte);
317                 if (uncompressed.readableBytes() >= outputBufferSize) {
318                     break;
319                 }
320             }
321             if (uncByte < 0) {
322                 // all data read
323                 currentState = State.INIT_BLOCK;
324                 int currentBlockCRC = blockDecompressor.checkCRC();
325                 streamCRC = (streamCRC << 1 | streamCRC >>> 31) ^ currentBlockCRC;
326             }
327             return uncompressed;
328         } catch (Throwable t) {
329             uncompressed.release();
330             throw t;
331         }
332     }
333 
334     @Override
335     public void endOfInput() throws DecompressionException {
336         if (currentState != State.EOF) {
337             throw new DecompressionException("Unexpected end of input");
338         }
339     }
340 
341     @Override
342     public Status status() throws DecompressionException {
343         switch (currentState) {
344             case DECODED_HUFFMAN_DATA:
345                 return Status.NEED_OUTPUT;
346             case EOF:
347                 return Status.COMPLETE;
348             default:
349                 return Status.NEED_INPUT;
350         }
351     }
352 
353     @UnstableApi
354     public static Builder builder() {
355         return new Builder();
356     }
357 
358     @UnstableApi
359     public static final class Builder extends AbstractDecompressorBuilder {
360         private int outputBufferSize = 65536;
361 
362         Builder() {
363         }
364 
365         /**
366          * Size of the output buffer to return from {@link #takeOutput()}. Default 64K.
367          *
368          * @param outputBufferSize Output buffer size
369          * @return This builder
370          */
371         @UnstableApi
372         public Builder outputBufferSize(int outputBufferSize) {
373             this.outputBufferSize = ObjectUtil.checkPositive(outputBufferSize, "outputBufferSize");
374             return this;
375         }
376 
377         @Override
378         @UnstableApi
379         public Decompressor build(ByteBufAllocator allocator) throws DecompressionException {
380             return new DefensiveDecompressor(new Bzip2Decompressor(this, allocator));
381         }
382     }
383 }