1 /*
2 * Copyright 2014 The Netty Project
3 *
4 * The Netty Project licenses this file to you under the Apache License,
5 * version 2.0 (the "License"); you may not use this file except in compliance
6 * with the License. You may obtain a copy of the License at:
7 *
8 * https://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13 * License for the specific language governing permissions and limitations
14 * under the License.
15 */
16 package io.netty.handler.codec.compression;
17
18 import com.ning.compress.BufferRecycler;
19 import com.ning.compress.lzf.ChunkEncoder;
20 import com.ning.compress.lzf.LZFChunk;
21 import com.ning.compress.lzf.LZFEncoder;
22 import com.ning.compress.lzf.util.ChunkEncoderFactory;
23 import io.netty.buffer.ByteBuf;
24 import io.netty.channel.ChannelHandlerContext;
25 import io.netty.handler.codec.MessageToByteEncoder;
26 import io.netty.util.internal.PlatformDependent;
27
28 import static com.ning.compress.lzf.LZFChunk.MAX_CHUNK_LEN;
29
30 /**
31 * Compresses a {@link ByteBuf} using the LZF format.
32 * <p>
33 * See original <a href="http://oldhome.schmorp.de/marc/liblzf.html">LZF package</a>
34 * and <a href="https://github.com/ning/compress/wiki/LZFFormat">LZF format</a> for full description.
35 */
36 public class LzfEncoder extends MessageToByteEncoder<ByteBuf> {
37
38 /**
39 * Minimum block size ready for compression. Blocks with length
40 * less than {@link #MIN_BLOCK_TO_COMPRESS} will write as uncompressed.
41 */
42 private static final int MIN_BLOCK_TO_COMPRESS = 16;
43 private static final boolean DEFAULT_SAFE = !PlatformDependent.hasUnsafe();
44
45 /**
46 * Compress threshold for LZF format. When the amount of input data is less than compressThreshold,
47 * we will construct an uncompressed output according to the LZF format.
48 * <p>
49 * When the value is less than {@see ChunkEncoder#MIN_BLOCK_TO_COMPRESS}, since LZF will not compress data
50 * that is less than {@see ChunkEncoder#MIN_BLOCK_TO_COMPRESS}, compressThreshold will not work.
51 */
52 private final int compressThreshold;
53
54 /**
55 * Underlying decoder in use.
56 */
57 private final ChunkEncoder encoder;
58
59 /**
60 * Object that handles details of buffer recycling.
61 */
62 private final BufferRecycler recycler;
63
64 /**
65 * Creates a new LZF encoder with the most optimal available methods for underlying data access.
66 * It will "unsafe" instance if one can be used on current JVM.
67 * It should be safe to call this constructor as implementations are dynamically loaded; however, on some
68 * non-standard platforms it may be necessary to use {@link #LzfEncoder(boolean)} with {@code true} param.
69 */
70 public LzfEncoder() {
71 this(DEFAULT_SAFE);
72 }
73
74 /**
75 * Creates a new LZF encoder with specified encoding instance.
76 *
77 * @param safeInstance If {@code true} encoder will use {@link ChunkEncoder} that only uses
78 * standard JDK access methods, and should work on all Java platforms and JVMs.
79 * Otherwise encoder will try to use highly optimized {@link ChunkEncoder}
80 * implementation that uses Sun JDK's {@link sun.misc.Unsafe}
81 * class (which may be included by other JDK's as well).
82 * @deprecated Use the constructor without the {@code safeInstance} parameter.
83 */
84 @Deprecated
85 public LzfEncoder(boolean safeInstance) {
86 this(safeInstance, MAX_CHUNK_LEN);
87 }
88
89 /**
90 * Creates a new LZF encoder with specified encoding instance and compressThreshold.
91 *
92 * @param safeInstance If {@code true} encoder will use {@link ChunkEncoder} that only uses standard
93 * JDK access methods, and should work on all Java platforms and JVMs.
94 * Otherwise encoder will try to use highly optimized {@link ChunkEncoder}
95 * implementation that uses Sun JDK's {@link sun.misc.Unsafe}
96 * class (which may be included by other JDK's as well).
97 * @param totalLength Expected total length of content to compress; only matters for outgoing messages
98 * that is smaller than maximum chunk size (64k), to optimize encoding hash tables.
99 * @deprecated Use the constructor without the {@code safeInstance} parameter.
100 */
101 @Deprecated
102 public LzfEncoder(boolean safeInstance, int totalLength) {
103 this(safeInstance, totalLength, MIN_BLOCK_TO_COMPRESS);
104 }
105
106 /**
107 * Creates a new LZF encoder with specified total length of encoded chunk. You can configure it to encode
108 * your data flow more efficient if you know the average size of messages that you send.
109 *
110 * @param totalLength Expected total length of content to compress;
111 * only matters for outgoing messages that is smaller than maximum chunk size (64k),
112 * to optimize encoding hash tables.
113 */
114 public LzfEncoder(int totalLength) {
115 this(DEFAULT_SAFE, totalLength);
116 }
117
118 /**
119 * Creates a new LZF encoder with specified settings.
120 *
121 * @param totalLength Expected total length of content to compress; only matters for outgoing messages
122 * that is smaller than maximum chunk size (64k), to optimize encoding hash tables.
123 * @param compressThreshold Compress threshold for LZF format. When the amount of input data is less than
124 * compressThreshold, we will construct an uncompressed output according
125 * to the LZF format.
126 */
127 public LzfEncoder(int totalLength, int compressThreshold) {
128 this(DEFAULT_SAFE, totalLength, compressThreshold);
129 }
130
131 /**
132 * Creates a new LZF encoder with specified settings.
133 *
134 * @param safeInstance If {@code true} encoder will use {@link ChunkEncoder} that only uses standard JDK
135 * access methods, and should work on all Java platforms and JVMs.
136 * Otherwise encoder will try to use highly optimized {@link ChunkEncoder}
137 * implementation that uses Sun JDK's {@link sun.misc.Unsafe}
138 * class (which may be included by other JDK's as well).
139 * @param totalLength Expected total length of content to compress; only matters for outgoing messages
140 * that is smaller than maximum chunk size (64k), to optimize encoding hash tables.
141 * @param compressThreshold Compress threshold for LZF format. When the amount of input data is less than
142 * compressThreshold, we will construct an uncompressed output according
143 * to the LZF format.
144 * @deprecated Use the constructor without the {@code safeInstance} parameter.
145 */
146 @Deprecated
147 public LzfEncoder(boolean safeInstance, int totalLength, int compressThreshold) {
148 super(false);
149 if (totalLength < MIN_BLOCK_TO_COMPRESS || totalLength > MAX_CHUNK_LEN) {
150 throw new IllegalArgumentException("totalLength: " + totalLength +
151 " (expected: " + MIN_BLOCK_TO_COMPRESS + '-' + MAX_CHUNK_LEN + ')');
152 }
153
154 if (compressThreshold < MIN_BLOCK_TO_COMPRESS) {
155 // not a suitable value.
156 throw new IllegalArgumentException("compressThreshold:" + compressThreshold +
157 " expected >=" + MIN_BLOCK_TO_COMPRESS);
158 }
159 this.compressThreshold = compressThreshold;
160
161 this.encoder = safeInstance ?
162 ChunkEncoderFactory.safeNonAllocatingInstance(totalLength)
163 : ChunkEncoderFactory.optimalNonAllocatingInstance(totalLength);
164
165 this.recycler = BufferRecycler.instance();
166 }
167
168 @Override
169 protected void encode(ChannelHandlerContext ctx, ByteBuf in, ByteBuf out) throws Exception {
170 final int length = in.readableBytes();
171 final int idx = in.readerIndex();
172 final byte[] input;
173 final int inputPtr;
174 if (in.hasArray()) {
175 input = in.array();
176 inputPtr = in.arrayOffset() + idx;
177 } else {
178 input = recycler.allocInputBuffer(length);
179 in.getBytes(idx, input, 0, length);
180 inputPtr = 0;
181 }
182
183 // Estimate may apparently under-count by one in some cases.
184 final int maxOutputLength = LZFEncoder.estimateMaxWorkspaceSize(length) + 1;
185 out.ensureWritable(maxOutputLength);
186 final byte[] output;
187 final int outputPtr;
188 if (out.hasArray()) {
189 output = out.array();
190 outputPtr = out.arrayOffset() + out.writerIndex();
191 } else {
192 output = new byte[maxOutputLength];
193 outputPtr = 0;
194 }
195
196 final int outputLength;
197 if (length >= compressThreshold) {
198 // compress.
199 outputLength = encodeCompress(input, inputPtr, length, output, outputPtr);
200 } else {
201 // not compress.
202 outputLength = encodeNonCompress(input, inputPtr, length, output, outputPtr);
203 }
204
205 if (out.hasArray()) {
206 out.writerIndex(out.writerIndex() + outputLength);
207 } else {
208 out.writeBytes(output, 0, outputLength);
209 }
210
211 in.skipBytes(length);
212
213 if (!in.hasArray()) {
214 recycler.releaseInputBuffer(input);
215 }
216 }
217
218 private int encodeCompress(byte[] input, int inputPtr, int length, byte[] output, int outputPtr) {
219 return LZFEncoder.appendEncoded(encoder,
220 input, inputPtr, length, output, outputPtr) - outputPtr;
221 }
222
223 private static int lzfEncodeNonCompress(byte[] input, int inputPtr, int length, byte[] output, int outputPtr) {
224 int left = length;
225 int chunkLen = Math.min(LZFChunk.MAX_CHUNK_LEN, left);
226 outputPtr = LZFChunk.appendNonCompressed(input, inputPtr, chunkLen, output, outputPtr);
227 left -= chunkLen;
228 if (left < 1) {
229 return outputPtr;
230 }
231 inputPtr += chunkLen;
232 do {
233 chunkLen = Math.min(left, LZFChunk.MAX_CHUNK_LEN);
234 outputPtr = LZFChunk.appendNonCompressed(input, inputPtr, chunkLen, output, outputPtr);
235 inputPtr += chunkLen;
236 left -= chunkLen;
237 } while (left > 0);
238 return outputPtr;
239 }
240
241 /**
242 * Use lzf uncompressed format to encode a piece of input.
243 */
244 private static int encodeNonCompress(byte[] input, int inputPtr, int length, byte[] output, int outputPtr) {
245 return lzfEncodeNonCompress(input, inputPtr, length, output, outputPtr) - outputPtr;
246 }
247
248 @Override
249 public void handlerRemoved(ChannelHandlerContext ctx) throws Exception {
250 encoder.close();
251 super.handlerRemoved(ctx);
252 }
253 }