View Javadoc
1   /*
2    * Copyright 2025 The Netty Project
3    *
4    * The Netty Project licenses this file to you under the Apache License,
5    * version 2.0 (the "License"); you may not use this file except in compliance
6    * with the License. You may obtain a copy of the License at:
7    *
8    *   https://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13   * License for the specific language governing permissions and limitations
14   * under the License.
15   */
16  package io.netty.handler.codec.compression;
17  
18  import io.netty.buffer.ByteBuf;
19  import io.netty.buffer.ByteBufAllocator;
20  import io.netty.util.internal.ObjectUtil;
21  import io.netty.util.internal.UnstableApi;
22  
23  import static io.netty.handler.codec.compression.Bzip2Constants.BASE_BLOCK_SIZE;
24  import static io.netty.handler.codec.compression.Bzip2Constants.BLOCK_HEADER_MAGIC_1;
25  import static io.netty.handler.codec.compression.Bzip2Constants.BLOCK_HEADER_MAGIC_2;
26  import static io.netty.handler.codec.compression.Bzip2Constants.END_OF_STREAM_MAGIC_1;
27  import static io.netty.handler.codec.compression.Bzip2Constants.END_OF_STREAM_MAGIC_2;
28  import static io.netty.handler.codec.compression.Bzip2Constants.HUFFMAN_MAXIMUM_TABLES;
29  import static io.netty.handler.codec.compression.Bzip2Constants.HUFFMAN_MAX_ALPHABET_SIZE;
30  import static io.netty.handler.codec.compression.Bzip2Constants.HUFFMAN_MINIMUM_TABLES;
31  import static io.netty.handler.codec.compression.Bzip2Constants.HUFFMAN_SELECTOR_LIST_MAX_LENGTH;
32  import static io.netty.handler.codec.compression.Bzip2Constants.HUFFMAN_SYMBOL_RANGE_SIZE;
33  import static io.netty.handler.codec.compression.Bzip2Constants.MAGIC_NUMBER;
34  import static io.netty.handler.codec.compression.Bzip2Constants.MAX_BLOCK_SIZE;
35  import static io.netty.handler.codec.compression.Bzip2Constants.MAX_SELECTORS;
36  import static io.netty.handler.codec.compression.Bzip2Constants.MIN_BLOCK_SIZE;
37  
38  /**
39   * Uncompresses a {@link ByteBuf} encoded with the Bzip2 format.
40   *
41   * See <a href="https://en.wikipedia.org/wiki/Bzip2">Bzip2</a>.
42   */
43  @UnstableApi
44  public final class Bzip2Decompressor extends InputBufferingDecompressor {
45      private final int outputBufferSize;
46  
47      /**
48       * Current state of stream.
49       */
50      private enum State {
51          INIT,
52          INIT_BLOCK,
53          INIT_BLOCK_PARAMS,
54          RECEIVE_HUFFMAN_USED_MAP,
55          RECEIVE_HUFFMAN_USED_BITMAPS,
56          RECEIVE_SELECTORS_NUMBER,
57          RECEIVE_SELECTORS,
58          RECEIVE_HUFFMAN_LENGTH,
59          DECODE_HUFFMAN_DATA,
60          DECODED_HUFFMAN_DATA,
61          EOF
62      }
63      private State currentState = State.INIT;
64  
65      /**
66       * A reader that provides bit-level reads.
67       */
68      private final Bzip2BitReader reader = new Bzip2BitReader();
69  
70      /**
71       * The decompressor for the current block.
72       */
73      private Bzip2BlockDecompressor blockDecompressor;
74  
75      /**
76       * Bzip2 Huffman coding stage.
77       */
78      private Bzip2HuffmanStageDecoder huffmanStageDecoder;
79  
80      /**
81       * Always: in the range 0 .. 9. The current block size is 100000 * this number.
82       */
83      private int blockSize;
84  
85      /**
86       * The CRC of the current block as read from the block header.
87       */
88      private int blockCRC;
89  
90      /**
91       * The merged CRC of all blocks decompressed so far.
92       */
93      private int streamCRC;
94  
95      Bzip2Decompressor(Builder builder, ByteBufAllocator allocator) {
96          super(allocator);
97          this.outputBufferSize = builder.outputBufferSize;
98      }
99  
100     @Override
101     void processInput(ByteBuf buf) throws DecompressionException {
102         reader.setByteBuf(buf);
103         switch (currentState) {
104             case INIT:
105                 if (buf.readableBytes() < 4) {
106                     break;
107                 }
108                 int magicNumber = buf.readUnsignedMedium();
109                 if (magicNumber != MAGIC_NUMBER) {
110                     throw new DecompressionException("Unexpected stream identifier contents. Mismatched bzip2 " +
111                             "protocol version?");
112                 }
113                 int blockSize = buf.readByte() - '0';
114                 if (blockSize < MIN_BLOCK_SIZE || blockSize > MAX_BLOCK_SIZE) {
115                     throw new DecompressionException("block size is invalid");
116                 }
117                 this.blockSize = blockSize * BASE_BLOCK_SIZE;
118 
119                 streamCRC = 0;
120                 currentState = State.INIT_BLOCK;
121                 // fall through
122             case INIT_BLOCK:
123                 if (!reader.hasReadableBytes(10)) {
124                     break;
125                 }
126                 // Get the block magic bytes.
127                 final int magic1 = reader.readBits(24);
128                 final int magic2 = reader.readBits(24);
129                 if (magic1 == END_OF_STREAM_MAGIC_1 && magic2 == END_OF_STREAM_MAGIC_2) {
130                     // End of stream was reached. Check the combined CRC.
131                     final int storedCombinedCRC = reader.readInt();
132                     if (storedCombinedCRC != streamCRC) {
133                         throw new DecompressionException("stream CRC error");
134                     }
135                     currentState = State.EOF;
136                     break;
137                 }
138                 if (magic1 != BLOCK_HEADER_MAGIC_1 || magic2 != BLOCK_HEADER_MAGIC_2) {
139                     throw new DecompressionException("bad block header");
140                 }
141                 blockCRC = reader.readInt();
142                 currentState = State.INIT_BLOCK_PARAMS;
143                 // fall through
144             case INIT_BLOCK_PARAMS:
145                 if (!reader.hasReadableBits(25)) {
146                     break;
147                 }
148                 final boolean blockRandomised = reader.readBoolean();
149                 final int bwtStartPointer = reader.readBits(24);
150 
151                 blockDecompressor = new Bzip2BlockDecompressor(this.blockSize, blockCRC,
152                         blockRandomised, bwtStartPointer, reader);
153                 currentState = State.RECEIVE_HUFFMAN_USED_MAP;
154                 // fall through
155             case RECEIVE_HUFFMAN_USED_MAP:
156                 if (!reader.hasReadableBits(16)) {
157                     break;
158                 }
159                 blockDecompressor.huffmanInUse16 = reader.readBits(16);
160                 currentState = State.RECEIVE_HUFFMAN_USED_BITMAPS;
161                 // fall through
162             case RECEIVE_HUFFMAN_USED_BITMAPS:
163                 Bzip2BlockDecompressor blockDecompressor = this.blockDecompressor;
164                 final int inUse16 = blockDecompressor.huffmanInUse16;
165                 final int bitNumber = Integer.bitCount(inUse16);
166                 final byte[] huffmanSymbolMap = blockDecompressor.huffmanSymbolMap;
167 
168                 if (!reader.hasReadableBits(bitNumber * HUFFMAN_SYMBOL_RANGE_SIZE + 3)) {
169                     break;
170                 }
171 
172                 int huffmanSymbolCount = 0;
173                 if (bitNumber > 0) {
174                     for (int i = 0; i < 16; i++) {
175                         if ((inUse16 & 1 << 15 >>> i) != 0) {
176                             for (int j = 0, k = i << 4; j < HUFFMAN_SYMBOL_RANGE_SIZE; j++, k++) {
177                                 if (reader.readBoolean()) {
178                                     huffmanSymbolMap[huffmanSymbolCount++] = (byte) k;
179                                 }
180                             }
181                         }
182                     }
183                 }
184                 blockDecompressor.huffmanEndOfBlockSymbol = huffmanSymbolCount + 1;
185 
186                 int totalTables = reader.readBits(3);
187                 if (totalTables < HUFFMAN_MINIMUM_TABLES || totalTables > HUFFMAN_MAXIMUM_TABLES) {
188                     throw new DecompressionException("incorrect huffman groups number");
189                 }
190                 int alphaSize = huffmanSymbolCount + 2;
191                 if (alphaSize > HUFFMAN_MAX_ALPHABET_SIZE) {
192                     throw new DecompressionException("incorrect alphabet size");
193                 }
194                 huffmanStageDecoder = new Bzip2HuffmanStageDecoder(reader, totalTables, alphaSize);
195                 currentState = State.RECEIVE_SELECTORS_NUMBER;
196                 // fall through
197             case RECEIVE_SELECTORS_NUMBER:
198                 if (!reader.hasReadableBits(15)) {
199                     break;
200                 }
201                 int totalSelectors = reader.readBits(15);
202                 if (totalSelectors < 1 || totalSelectors > MAX_SELECTORS) {
203                     throw new DecompressionException("incorrect selectors number");
204                 }
205                 huffmanStageDecoder.selectors = new byte[totalSelectors];
206 
207                 currentState = State.RECEIVE_SELECTORS;
208                 // fall through
209             case RECEIVE_SELECTORS:
210                 Bzip2HuffmanStageDecoder huffmanStageDecoder = this.huffmanStageDecoder;
211                 byte[] selectors = huffmanStageDecoder.selectors;
212                 totalSelectors = selectors.length;
213                 final Bzip2MoveToFrontTable tableMtf = huffmanStageDecoder.tableMTF;
214 
215                 int currSelector;
216                 // Get zero-terminated bit runs (0..62) of MTF'ed Huffman table. length = 1..6
217                 for (currSelector = huffmanStageDecoder.currentSelector;
218                      currSelector < totalSelectors; currSelector++) {
219                     if (!reader.hasReadableBits(HUFFMAN_SELECTOR_LIST_MAX_LENGTH)) {
220                         // Save state if end of current ByteBuf was reached
221                         huffmanStageDecoder.currentSelector = currSelector;
222                         return;
223                     }
224                     int index = 0;
225                     while (reader.readBoolean()) {
226                         index++;
227                     }
228                     selectors[currSelector] = tableMtf.indexToFront(index);
229                 }
230 
231                 currentState = State.RECEIVE_HUFFMAN_LENGTH;
232                 // fall through
233             case RECEIVE_HUFFMAN_LENGTH:
234                 huffmanStageDecoder = this.huffmanStageDecoder;
235                 totalTables = huffmanStageDecoder.totalTables;
236                 final byte[][] codeLength = huffmanStageDecoder.tableCodeLengths;
237                 alphaSize = huffmanStageDecoder.alphabetSize;
238 
239                 /* Now the coding tables */
240                 int currGroup;
241                 int currLength = huffmanStageDecoder.currentLength;
242                 int currAlpha = 0;
243                 boolean modifyLength = huffmanStageDecoder.modifyLength;
244                 boolean saveStateAndReturn = false;
245                 loop: for (currGroup = huffmanStageDecoder.currentGroup; currGroup < totalTables; currGroup++) {
246                     // start_huffman_length
247                     if (!reader.hasReadableBits(5)) {
248                         saveStateAndReturn = true;
249                         break;
250                     }
251                     if (currLength < 0) {
252                         currLength = reader.readBits(5);
253                     }
254                     for (currAlpha = huffmanStageDecoder.currentAlpha; currAlpha < alphaSize; currAlpha++) {
255                         // delta_bit_length: 1..40
256                         if (!reader.isReadable()) {
257                             saveStateAndReturn = true;
258                             break loop;
259                         }
260                         while (modifyLength || reader.readBoolean()) {  // 0=>next symbol; 1=>alter length
261                             if (!reader.isReadable()) {
262                                 modifyLength = true;
263                                 saveStateAndReturn = true;
264                                 break loop;
265                             }
266                             // 1=>decrement length;  0=>increment length
267                             currLength += reader.readBoolean() ? -1 : 1;
268                             modifyLength = false;
269                             if (!reader.isReadable()) {
270                                 saveStateAndReturn = true;
271                                 break loop;
272                             }
273                         }
274                         codeLength[currGroup][currAlpha] = (byte) currLength;
275                     }
276                     currLength = -1;
277                     currAlpha = huffmanStageDecoder.currentAlpha = 0;
278                     modifyLength = false;
279                 }
280                 if (saveStateAndReturn) {
281                     // Save state if end of current ByteBuf was reached
282                     huffmanStageDecoder.currentGroup = currGroup;
283                     huffmanStageDecoder.currentLength = currLength;
284                     huffmanStageDecoder.currentAlpha = currAlpha;
285                     huffmanStageDecoder.modifyLength = modifyLength;
286                     break;
287                 }
288 
289                 // Finally create the Huffman tables
290                 huffmanStageDecoder.createHuffmanDecodingTables();
291                 currentState = State.DECODE_HUFFMAN_DATA;
292                 // fall through
293             case DECODE_HUFFMAN_DATA:
294                 blockDecompressor = this.blockDecompressor;
295                 final boolean decoded = blockDecompressor.decodeHuffmanData(this.huffmanStageDecoder);
296                 if (!decoded) {
297                     return;
298                 }
299                 currentState = State.DECODED_HUFFMAN_DATA;
300                 break;
301         }
302     }
303 
304     @Override
305     ByteBuf processOutput(ByteBuf buf) throws DecompressionException {
306         if (currentState != State.DECODED_HUFFMAN_DATA) {
307             throw new IllegalStateException("Not in state NEED_OUTPUT");
308         }
309         ByteBuf uncompressed = allocator.buffer(outputBufferSize);
310         try {
311             int uncByte;
312             while ((uncByte = blockDecompressor.read()) >= 0) {
313                 uncompressed.writeByte(uncByte);
314                 if (uncompressed.readableBytes() >= outputBufferSize) {
315                     break;
316                 }
317             }
318             if (uncByte < 0) {
319                 // all data read
320                 currentState = State.INIT_BLOCK;
321                 int currentBlockCRC = blockDecompressor.checkCRC();
322                 streamCRC = (streamCRC << 1 | streamCRC >>> 31) ^ currentBlockCRC;
323             }
324             return uncompressed;
325         } catch (Throwable t) {
326             uncompressed.release();
327             throw t;
328         }
329     }
330 
331     @Override
332     public void endOfInput() throws DecompressionException {
333         if (currentState != State.EOF) {
334             throw new DecompressionException("Unexpected end of input");
335         }
336     }
337 
338     @Override
339     public Status status() throws DecompressionException {
340         switch (currentState) {
341             case DECODED_HUFFMAN_DATA:
342                 return Status.NEED_OUTPUT;
343             case EOF:
344                 return Status.COMPLETE;
345             default:
346                 return Status.NEED_INPUT;
347         }
348     }
349 
350     @UnstableApi
351     public static Builder builder() {
352         return new Builder();
353     }
354 
355     @UnstableApi
356     public static final class Builder extends AbstractDecompressorBuilder {
357         private int outputBufferSize = 65536;
358 
359         Builder() {
360         }
361 
362         /**
363          * Size of the output buffer to return from {@link #takeOutput()}. Default 64K.
364          *
365          * @param outputBufferSize Output buffer size
366          * @return This builder
367          */
368         @UnstableApi
369         public Builder outputBufferSize(int outputBufferSize) {
370             this.outputBufferSize = ObjectUtil.checkPositive(outputBufferSize, "outputBufferSize");
371             return this;
372         }
373 
374         @Override
375         @UnstableApi
376         public Decompressor build(ByteBufAllocator allocator) throws DecompressionException {
377             return new DefensiveDecompressor(new Bzip2Decompressor(this, allocator));
378         }
379     }
380 }