View Javadoc
1   /*
2    * Copyright 2025 The Netty Project
3    *
4    * The Netty Project licenses this file to you under the Apache License,
5    * version 2.0 (the "License"); you may not use this file except in compliance
6    * with the License. You may obtain a copy of the License at:
7    *
8    *   https://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13   * License for the specific language governing permissions and limitations
14   * under the License.
15   */
16  package io.netty.handler.codec.compression;
17  
18  import io.netty.buffer.ByteBuf;
19  import io.netty.buffer.ByteBufAllocator;
20  import io.netty.util.internal.ObjectUtil;
21  import io.netty.util.internal.UnstableApi;
22  
23  import static io.netty.handler.codec.compression.Bzip2Constants.BASE_BLOCK_SIZE;
24  import static io.netty.handler.codec.compression.Bzip2Constants.BLOCK_HEADER_MAGIC_1;
25  import static io.netty.handler.codec.compression.Bzip2Constants.BLOCK_HEADER_MAGIC_2;
26  import static io.netty.handler.codec.compression.Bzip2Constants.END_OF_STREAM_MAGIC_1;
27  import static io.netty.handler.codec.compression.Bzip2Constants.END_OF_STREAM_MAGIC_2;
28  import static io.netty.handler.codec.compression.Bzip2Constants.HUFFMAN_DECODE_MAX_CODE_LENGTH;
29  import static io.netty.handler.codec.compression.Bzip2Constants.HUFFMAN_MAXIMUM_TABLES;
30  import static io.netty.handler.codec.compression.Bzip2Constants.HUFFMAN_MAX_ALPHABET_SIZE;
31  import static io.netty.handler.codec.compression.Bzip2Constants.HUFFMAN_MINIMUM_TABLES;
32  import static io.netty.handler.codec.compression.Bzip2Constants.HUFFMAN_SELECTOR_LIST_MAX_LENGTH;
33  import static io.netty.handler.codec.compression.Bzip2Constants.HUFFMAN_SYMBOL_RANGE_SIZE;
34  import static io.netty.handler.codec.compression.Bzip2Constants.MAGIC_NUMBER;
35  import static io.netty.handler.codec.compression.Bzip2Constants.MAX_BLOCK_SIZE;
36  import static io.netty.handler.codec.compression.Bzip2Constants.MAX_SELECTORS;
37  import static io.netty.handler.codec.compression.Bzip2Constants.MIN_BLOCK_SIZE;
38  
39  /**
40   * Uncompresses a {@link ByteBuf} encoded with the Bzip2 format.
41   *
42   * See <a href="https://en.wikipedia.org/wiki/Bzip2">Bzip2</a>.
43   */
44  @UnstableApi
45  public final class Bzip2Decompressor extends InputBufferingDecompressor {
46      private final int outputBufferSize;
47  
48      /**
49       * Current state of stream.
50       */
51      private enum State {
52          INIT,
53          INIT_BLOCK,
54          INIT_BLOCK_PARAMS,
55          RECEIVE_HUFFMAN_USED_MAP,
56          RECEIVE_HUFFMAN_USED_BITMAPS,
57          RECEIVE_SELECTORS_NUMBER,
58          RECEIVE_SELECTORS,
59          RECEIVE_HUFFMAN_LENGTH,
60          DECODE_HUFFMAN_DATA,
61          DECODED_HUFFMAN_DATA,
62          EOF
63      }
64      private State currentState = State.INIT;
65  
66      /**
67       * A reader that provides bit-level reads.
68       */
69      private final Bzip2BitReader reader = new Bzip2BitReader();
70  
71      /**
72       * The decompressor for the current block.
73       */
74      private Bzip2BlockDecompressor blockDecompressor;
75  
76      /**
77       * Bzip2 Huffman coding stage.
78       */
79      private Bzip2HuffmanStageDecoder huffmanStageDecoder;
80  
81      /**
82       * Always: in the range 0 .. 9. The current block size is 100000 * this number.
83       */
84      private int blockSize;
85  
86      /**
87       * The CRC of the current block as read from the block header.
88       */
89      private int blockCRC;
90  
91      /**
92       * The merged CRC of all blocks decompressed so far.
93       */
94      private int streamCRC;
95  
96      Bzip2Decompressor(Builder builder, ByteBufAllocator allocator) {
97          super(allocator);
98          this.outputBufferSize = builder.outputBufferSize;
99      }
100 
101     @Override
102     void processInput(ByteBuf buf) throws DecompressionException {
103         reader.setByteBuf(buf);
104         switch (currentState) {
105             case INIT:
106                 if (buf.readableBytes() < 4) {
107                     break;
108                 }
109                 int magicNumber = buf.readUnsignedMedium();
110                 if (magicNumber != MAGIC_NUMBER) {
111                     throw new DecompressionException("Unexpected stream identifier contents. Mismatched bzip2 " +
112                             "protocol version?");
113                 }
114                 int blockSize = buf.readByte() - '0';
115                 if (blockSize < MIN_BLOCK_SIZE || blockSize > MAX_BLOCK_SIZE) {
116                     throw new DecompressionException("block size is invalid");
117                 }
118                 this.blockSize = blockSize * BASE_BLOCK_SIZE;
119 
120                 streamCRC = 0;
121                 currentState = State.INIT_BLOCK;
122                 // fall through
123             case INIT_BLOCK:
124                 if (!reader.hasReadableBytes(10)) {
125                     break;
126                 }
127                 // Get the block magic bytes.
128                 final int magic1 = reader.readBits(24);
129                 final int magic2 = reader.readBits(24);
130                 if (magic1 == END_OF_STREAM_MAGIC_1 && magic2 == END_OF_STREAM_MAGIC_2) {
131                     // End of stream was reached. Check the combined CRC.
132                     final int storedCombinedCRC = reader.readInt();
133                     if (storedCombinedCRC != streamCRC) {
134                         throw new DecompressionException("stream CRC error");
135                     }
136                     currentState = State.EOF;
137                     break;
138                 }
139                 if (magic1 != BLOCK_HEADER_MAGIC_1 || magic2 != BLOCK_HEADER_MAGIC_2) {
140                     throw new DecompressionException("bad block header");
141                 }
142                 blockCRC = reader.readInt();
143                 currentState = State.INIT_BLOCK_PARAMS;
144                 // fall through
145             case INIT_BLOCK_PARAMS:
146                 if (!reader.hasReadableBits(25)) {
147                     break;
148                 }
149                 final boolean blockRandomised = reader.readBoolean();
150                 final int bwtStartPointer = reader.readBits(24);
151 
152                 blockDecompressor = new Bzip2BlockDecompressor(this.blockSize, blockCRC,
153                         blockRandomised, bwtStartPointer, reader);
154                 currentState = State.RECEIVE_HUFFMAN_USED_MAP;
155                 // fall through
156             case RECEIVE_HUFFMAN_USED_MAP:
157                 if (!reader.hasReadableBits(16)) {
158                     break;
159                 }
160                 blockDecompressor.huffmanInUse16 = reader.readBits(16);
161                 currentState = State.RECEIVE_HUFFMAN_USED_BITMAPS;
162                 // fall through
163             case RECEIVE_HUFFMAN_USED_BITMAPS:
164                 Bzip2BlockDecompressor blockDecompressor = this.blockDecompressor;
165                 final int inUse16 = blockDecompressor.huffmanInUse16;
166                 final int bitNumber = Integer.bitCount(inUse16);
167                 final byte[] huffmanSymbolMap = blockDecompressor.huffmanSymbolMap;
168 
169                 if (!reader.hasReadableBits(bitNumber * HUFFMAN_SYMBOL_RANGE_SIZE + 3)) {
170                     break;
171                 }
172 
173                 int huffmanSymbolCount = 0;
174                 if (bitNumber > 0) {
175                     for (int i = 0; i < 16; i++) {
176                         if ((inUse16 & 1 << 15 >>> i) != 0) {
177                             for (int j = 0, k = i << 4; j < HUFFMAN_SYMBOL_RANGE_SIZE; j++, k++) {
178                                 if (reader.readBoolean()) {
179                                     huffmanSymbolMap[huffmanSymbolCount++] = (byte) k;
180                                 }
181                             }
182                         }
183                     }
184                 }
185                 blockDecompressor.huffmanEndOfBlockSymbol = huffmanSymbolCount + 1;
186 
187                 int totalTables = reader.readBits(3);
188                 if (totalTables < HUFFMAN_MINIMUM_TABLES || totalTables > HUFFMAN_MAXIMUM_TABLES) {
189                     throw new DecompressionException("incorrect huffman groups number");
190                 }
191                 int alphaSize = huffmanSymbolCount + 2;
192                 if (alphaSize > HUFFMAN_MAX_ALPHABET_SIZE) {
193                     throw new DecompressionException("incorrect alphabet size");
194                 }
195                 huffmanStageDecoder = new Bzip2HuffmanStageDecoder(reader, totalTables, alphaSize);
196                 currentState = State.RECEIVE_SELECTORS_NUMBER;
197                 // fall through
198             case RECEIVE_SELECTORS_NUMBER:
199                 if (!reader.hasReadableBits(15)) {
200                     break;
201                 }
202                 int totalSelectors = reader.readBits(15);
203                 if (totalSelectors < 1 || totalSelectors > MAX_SELECTORS) {
204                     throw new DecompressionException("incorrect selectors number");
205                 }
206                 huffmanStageDecoder.selectors = new byte[totalSelectors];
207 
208                 currentState = State.RECEIVE_SELECTORS;
209                 // fall through
210             case RECEIVE_SELECTORS:
211                 Bzip2HuffmanStageDecoder huffmanStageDecoder = this.huffmanStageDecoder;
212                 byte[] selectors = huffmanStageDecoder.selectors;
213                 totalSelectors = selectors.length;
214                 final Bzip2MoveToFrontTable tableMtf = huffmanStageDecoder.tableMTF;
215 
216                 int currSelector;
217                 // Get zero-terminated bit runs (0..62) of MTF'ed Huffman table. length = 1..6
218                 for (currSelector = huffmanStageDecoder.currentSelector;
219                      currSelector < totalSelectors; currSelector++) {
220                     if (!reader.hasReadableBits(HUFFMAN_SELECTOR_LIST_MAX_LENGTH)) {
221                         // Save state if end of current ByteBuf was reached
222                         huffmanStageDecoder.currentSelector = currSelector;
223                         return;
224                     }
225                     int index = 0;
226                     while (reader.readBoolean()) {
227                         index++;
228                         if (index >= huffmanStageDecoder.totalTables) {
229                             throw new DecompressionException("incorrect selector index " + index);
230                         }
231                     }
232                     selectors[currSelector] = tableMtf.indexToFront(index);
233                 }
234 
235                 currentState = State.RECEIVE_HUFFMAN_LENGTH;
236                 // fall through
237             case RECEIVE_HUFFMAN_LENGTH:
238                 huffmanStageDecoder = this.huffmanStageDecoder;
239                 totalTables = huffmanStageDecoder.totalTables;
240                 final byte[][] codeLength = huffmanStageDecoder.tableCodeLengths;
241                 alphaSize = huffmanStageDecoder.alphabetSize;
242 
243                 /* Now the coding tables */
244                 int currGroup;
245                 int currLength = huffmanStageDecoder.currentLength;
246                 int currAlpha = 0;
247                 boolean modifyLength = huffmanStageDecoder.modifyLength;
248                 boolean saveStateAndReturn = false;
249                 loop: for (currGroup = huffmanStageDecoder.currentGroup; currGroup < totalTables; currGroup++) {
250                     // start_huffman_length
251                     if (!reader.hasReadableBits(5)) {
252                         saveStateAndReturn = true;
253                         break;
254                     }
255                     if (currLength < 0) {
256                         currLength = reader.readBits(5);
257                     }
258                     for (currAlpha = huffmanStageDecoder.currentAlpha; currAlpha < alphaSize; currAlpha++) {
259                         // delta_bit_length: 1..40
260                         if (!reader.isReadable()) {
261                             saveStateAndReturn = true;
262                             break loop;
263                         }
264                         while (modifyLength || reader.readBoolean()) {  // 0=>next symbol; 1=>alter length
265                             if (!reader.isReadable()) {
266                                 modifyLength = true;
267                                 saveStateAndReturn = true;
268                                 break loop;
269                             }
270                             // 1=>decrement length;  0=>increment length
271                             currLength += reader.readBoolean() ? -1 : 1;
272                             modifyLength = false;
273                             if (!reader.isReadable()) {
274                                 saveStateAndReturn = true;
275                                 break loop;
276                             }
277                         }
278                         if (currLength < 1 || currLength > HUFFMAN_DECODE_MAX_CODE_LENGTH) {
279                             throw new DecompressionException("Bzip2 block Huffman code length not in range 1.."
280                                     + HUFFMAN_DECODE_MAX_CODE_LENGTH);
281                         }
282                         codeLength[currGroup][currAlpha] = (byte) currLength;
283                     }
284                     currLength = -1;
285                     currAlpha = huffmanStageDecoder.currentAlpha = 0;
286                     modifyLength = false;
287                 }
288                 if (saveStateAndReturn) {
289                     // Save state if end of current ByteBuf was reached
290                     huffmanStageDecoder.currentGroup = currGroup;
291                     huffmanStageDecoder.currentLength = currLength;
292                     huffmanStageDecoder.currentAlpha = currAlpha;
293                     huffmanStageDecoder.modifyLength = modifyLength;
294                     break;
295                 }
296 
297                 // Finally create the Huffman tables
298                 huffmanStageDecoder.createHuffmanDecodingTables();
299                 currentState = State.DECODE_HUFFMAN_DATA;
300                 // fall through
301             case DECODE_HUFFMAN_DATA:
302                 blockDecompressor = this.blockDecompressor;
303                 final boolean decoded = blockDecompressor.decodeHuffmanData(this.huffmanStageDecoder);
304                 if (!decoded) {
305                     return;
306                 }
307                 currentState = State.DECODED_HUFFMAN_DATA;
308                 break;
309         }
310     }
311 
312     @Override
313     ByteBuf processOutput(ByteBuf buf) throws DecompressionException {
314         if (currentState != State.DECODED_HUFFMAN_DATA) {
315             throw new IllegalStateException("Not in state NEED_OUTPUT");
316         }
317         ByteBuf uncompressed = allocator.buffer(outputBufferSize);
318         try {
319             int uncByte;
320             while ((uncByte = blockDecompressor.read()) >= 0) {
321                 uncompressed.writeByte(uncByte);
322                 if (uncompressed.readableBytes() >= outputBufferSize) {
323                     break;
324                 }
325             }
326             if (uncByte < 0) {
327                 // all data read
328                 currentState = State.INIT_BLOCK;
329                 int currentBlockCRC = blockDecompressor.checkCRC();
330                 streamCRC = (streamCRC << 1 | streamCRC >>> 31) ^ currentBlockCRC;
331             }
332             return uncompressed;
333         } catch (Throwable t) {
334             uncompressed.release();
335             throw t;
336         }
337     }
338 
339     @Override
340     public void endOfInput() throws DecompressionException {
341         if (currentState != State.EOF) {
342             throw new DecompressionException("Unexpected end of input");
343         }
344     }
345 
346     @Override
347     public Status status() throws DecompressionException {
348         switch (currentState) {
349             case DECODED_HUFFMAN_DATA:
350                 return Status.NEED_OUTPUT;
351             case EOF:
352                 return Status.COMPLETE;
353             default:
354                 return Status.NEED_INPUT;
355         }
356     }
357 
358     @UnstableApi
359     public static Builder builder() {
360         return new Builder();
361     }
362 
363     @UnstableApi
364     public static final class Builder extends AbstractDecompressorBuilder {
365         private int outputBufferSize = 65536;
366 
367         Builder() {
368         }
369 
370         /**
371          * Size of the output buffer to return from {@link #takeOutput()}. Default 64K.
372          *
373          * @param outputBufferSize Output buffer size
374          * @return This builder
375          */
376         @UnstableApi
377         public Builder outputBufferSize(int outputBufferSize) {
378             this.outputBufferSize = ObjectUtil.checkPositive(outputBufferSize, "outputBufferSize");
379             return this;
380         }
381 
382         @Override
383         @UnstableApi
384         public Decompressor build(ByteBufAllocator allocator) throws DecompressionException {
385             return new DefensiveDecompressor(new Bzip2Decompressor(this, allocator));
386         }
387     }
388 }