View Javadoc
1   /*
2    * Copyright 2012 The Netty Project
3    *
4    * The Netty Project licenses this file to you under the Apache License,
5    * version 2.0 (the "License"); you may not use this file except in compliance
6    * with the License. You may obtain a copy of the License at:
7    *
8    *   http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13   * License for the specific language governing permissions and limitations
14   * under the License.
15   */
16  package io.netty.handler.codec;
17  
18  import java.nio.ByteOrder;
19  import java.util.List;
20  
21  import io.netty.buffer.ByteBuf;
22  import io.netty.channel.ChannelHandlerContext;
23  import io.netty.handler.codec.serialization.ObjectDecoder;
24  
25  /**
26   * A decoder that splits the received {@link ByteBuf}s dynamically by the
27   * value of the length field in the message.  It is particularly useful when you
28   * decode a binary message which has an integer header field that represents the
29   * length of the message body or the whole message.
30   * <p>
31   * {@link LengthFieldBasedFrameDecoder} has many configuration parameters so
32   * that it can decode any message with a length field, which is often seen in
33   * proprietary client-server protocols. Here are some example that will give
34   * you the basic idea on which option does what.
35   *
36   * <h3>2 bytes length field at offset 0, do not strip header</h3>
37   *
38   * The value of the length field in this example is <tt>12 (0x0C)</tt> which
39   * represents the length of "HELLO, WORLD".  By default, the decoder assumes
40   * that the length field represents the number of the bytes that follows the
41   * length field.  Therefore, it can be decoded with the simplistic parameter
42   * combination.
43   * <pre>
44   * <b>lengthFieldOffset</b>   = <b>0</b>
45   * <b>lengthFieldLength</b>   = <b>2</b>
46   * lengthAdjustment    = 0
47   * initialBytesToStrip = 0 (= do not strip header)
48   *
49   * BEFORE DECODE (14 bytes)         AFTER DECODE (14 bytes)
50   * +--------+----------------+      +--------+----------------+
51   * | Length | Actual Content |----->| Length | Actual Content |
52   * | 0x000C | "HELLO, WORLD" |      | 0x000C | "HELLO, WORLD" |
53   * +--------+----------------+      +--------+----------------+
54   * </pre>
55   *
56   * <h3>2 bytes length field at offset 0, strip header</h3>
57   *
58   * Because we can get the length of the content by calling
59   * {@link ByteBuf#readableBytes()}, you might want to strip the length
60   * field by specifying <tt>initialBytesToStrip</tt>.  In this example, we
61   * specified <tt>2</tt>, that is same with the length of the length field, to
62   * strip the first two bytes.
63   * <pre>
64   * lengthFieldOffset   = 0
65   * lengthFieldLength   = 2
66   * lengthAdjustment    = 0
67   * <b>initialBytesToStrip</b> = <b>2</b> (= the length of the Length field)
68   *
69   * BEFORE DECODE (14 bytes)         AFTER DECODE (12 bytes)
70   * +--------+----------------+      +----------------+
71   * | Length | Actual Content |----->| Actual Content |
72   * | 0x000C | "HELLO, WORLD" |      | "HELLO, WORLD" |
73   * +--------+----------------+      +----------------+
74   * </pre>
75   *
76   * <h3>2 bytes length field at offset 0, do not strip header, the length field
77   *     represents the length of the whole message</h3>
78   *
79   * In most cases, the length field represents the length of the message body
80   * only, as shown in the previous examples.  However, in some protocols, the
81   * length field represents the length of the whole message, including the
82   * message header.  In such a case, we specify a non-zero
83   * <tt>lengthAdjustment</tt>.  Because the length value in this example message
84   * is always greater than the body length by <tt>2</tt>, we specify <tt>-2</tt>
85   * as <tt>lengthAdjustment</tt> for compensation.
86   * <pre>
87   * lengthFieldOffset   =  0
88   * lengthFieldLength   =  2
89   * <b>lengthAdjustment</b>    = <b>-2</b> (= the length of the Length field)
90   * initialBytesToStrip =  0
91   *
92   * BEFORE DECODE (14 bytes)         AFTER DECODE (14 bytes)
93   * +--------+----------------+      +--------+----------------+
94   * | Length | Actual Content |----->| Length | Actual Content |
95   * | 0x000E | "HELLO, WORLD" |      | 0x000E | "HELLO, WORLD" |
96   * +--------+----------------+      +--------+----------------+
97   * </pre>
98   *
99   * <h3>3 bytes length field at the end of 5 bytes header, do not strip header</h3>
100  *
101  * The following message is a simple variation of the first example.  An extra
102  * header value is prepended to the message.  <tt>lengthAdjustment</tt> is zero
103  * again because the decoder always takes the length of the prepended data into
104  * account during frame length calculation.
105  * <pre>
106  * <b>lengthFieldOffset</b>   = <b>2</b> (= the length of Header 1)
107  * <b>lengthFieldLength</b>   = <b>3</b>
108  * lengthAdjustment    = 0
109  * initialBytesToStrip = 0
110  *
111  * BEFORE DECODE (17 bytes)                      AFTER DECODE (17 bytes)
112  * +----------+----------+----------------+      +----------+----------+----------------+
113  * | Header 1 |  Length  | Actual Content |----->| Header 1 |  Length  | Actual Content |
114  * |  0xCAFE  | 0x00000C | "HELLO, WORLD" |      |  0xCAFE  | 0x00000C | "HELLO, WORLD" |
115  * +----------+----------+----------------+      +----------+----------+----------------+
116  * </pre>
117  *
118  * <h3>3 bytes length field at the beginning of 5 bytes header, do not strip header</h3>
119  *
120  * This is an advanced example that shows the case where there is an extra
121  * header between the length field and the message body.  You have to specify a
122  * positive <tt>lengthAdjustment</tt> so that the decoder counts the extra
123  * header into the frame length calculation.
124  * <pre>
125  * lengthFieldOffset   = 0
126  * lengthFieldLength   = 3
127  * <b>lengthAdjustment</b>    = <b>2</b> (= the length of Header 1)
128  * initialBytesToStrip = 0
129  *
130  * BEFORE DECODE (17 bytes)                      AFTER DECODE (17 bytes)
131  * +----------+----------+----------------+      +----------+----------+----------------+
132  * |  Length  | Header 1 | Actual Content |----->|  Length  | Header 1 | Actual Content |
133  * | 0x00000C |  0xCAFE  | "HELLO, WORLD" |      | 0x00000C |  0xCAFE  | "HELLO, WORLD" |
134  * +----------+----------+----------------+      +----------+----------+----------------+
135  * </pre>
136  *
137  * <h3>2 bytes length field at offset 1 in the middle of 4 bytes header,
138  *     strip the first header field and the length field</h3>
139  *
140  * This is a combination of all the examples above.  There are the prepended
141  * header before the length field and the extra header after the length field.
142  * The prepended header affects the <tt>lengthFieldOffset</tt> and the extra
143  * header affects the <tt>lengthAdjustment</tt>.  We also specified a non-zero
144  * <tt>initialBytesToStrip</tt> to strip the length field and the prepended
145  * header from the frame.  If you don't want to strip the prepended header, you
146  * could specify <tt>0</tt> for <tt>initialBytesToSkip</tt>.
147  * <pre>
148  * lengthFieldOffset   = 1 (= the length of HDR1)
149  * lengthFieldLength   = 2
150  * <b>lengthAdjustment</b>    = <b>1</b> (= the length of HDR2)
151  * <b>initialBytesToStrip</b> = <b>3</b> (= the length of HDR1 + LEN)
152  *
153  * BEFORE DECODE (16 bytes)                       AFTER DECODE (13 bytes)
154  * +------+--------+------+----------------+      +------+----------------+
155  * | HDR1 | Length | HDR2 | Actual Content |----->| HDR2 | Actual Content |
156  * | 0xCA | 0x000C | 0xFE | "HELLO, WORLD" |      | 0xFE | "HELLO, WORLD" |
157  * +------+--------+------+----------------+      +------+----------------+
158  * </pre>
159  *
160  * <h3>2 bytes length field at offset 1 in the middle of 4 bytes header,
161  *     strip the first header field and the length field, the length field
162  *     represents the length of the whole message</h3>
163  *
164  * Let's give another twist to the previous example.  The only difference from
165  * the previous example is that the length field represents the length of the
166  * whole message instead of the message body, just like the third example.
167  * We have to count the length of HDR1 and Length into <tt>lengthAdjustment</tt>.
168  * Please note that we don't need to take the length of HDR2 into account
169  * because the length field already includes the whole header length.
170  * <pre>
171  * lengthFieldOffset   =  1
172  * lengthFieldLength   =  2
173  * <b>lengthAdjustment</b>    = <b>-3</b> (= the length of HDR1 + LEN, negative)
174  * <b>initialBytesToStrip</b> = <b> 3</b>
175  *
176  * BEFORE DECODE (16 bytes)                       AFTER DECODE (13 bytes)
177  * +------+--------+------+----------------+      +------+----------------+
178  * | HDR1 | Length | HDR2 | Actual Content |----->| HDR2 | Actual Content |
179  * | 0xCA | 0x0010 | 0xFE | "HELLO, WORLD" |      | 0xFE | "HELLO, WORLD" |
180  * +------+--------+------+----------------+      +------+----------------+
181  * </pre>
182  * @see LengthFieldPrepender
183  */
184 public class LengthFieldBasedFrameDecoder extends ByteToMessageDecoder {
185 
186     private final ByteOrder byteOrder;
187     private final int maxFrameLength;
188     private final int lengthFieldOffset;
189     private final int lengthFieldLength;
190     private final int lengthFieldEndOffset;
191     private final int lengthAdjustment;
192     private final int initialBytesToStrip;
193     private final boolean failFast;
194     private boolean discardingTooLongFrame;
195     private long tooLongFrameLength;
196     private long bytesToDiscard;
197 
198     /**
199      * Creates a new instance.
200      *
201      * @param maxFrameLength
202      *        the maximum length of the frame.  If the length of the frame is
203      *        greater than this value, {@link TooLongFrameException} will be
204      *        thrown.
205      * @param lengthFieldOffset
206      *        the offset of the length field
207      * @param lengthFieldLength
208      *        the length of the length field
209      */
210     public LengthFieldBasedFrameDecoder(
211             int maxFrameLength,
212             int lengthFieldOffset, int lengthFieldLength) {
213         this(maxFrameLength, lengthFieldOffset, lengthFieldLength, 0, 0);
214     }
215 
216     /**
217      * Creates a new instance.
218      *
219      * @param maxFrameLength
220      *        the maximum length of the frame.  If the length of the frame is
221      *        greater than this value, {@link TooLongFrameException} will be
222      *        thrown.
223      * @param lengthFieldOffset
224      *        the offset of the length field
225      * @param lengthFieldLength
226      *        the length of the length field
227      * @param lengthAdjustment
228      *        the compensation value to add to the value of the length field
229      * @param initialBytesToStrip
230      *        the number of first bytes to strip out from the decoded frame
231      */
232     public LengthFieldBasedFrameDecoder(
233             int maxFrameLength,
234             int lengthFieldOffset, int lengthFieldLength,
235             int lengthAdjustment, int initialBytesToStrip) {
236         this(
237                 maxFrameLength,
238                 lengthFieldOffset, lengthFieldLength, lengthAdjustment,
239                 initialBytesToStrip, true);
240     }
241 
242     /**
243      * Creates a new instance.
244      *
245      * @param maxFrameLength
246      *        the maximum length of the frame.  If the length of the frame is
247      *        greater than this value, {@link TooLongFrameException} will be
248      *        thrown.
249      * @param lengthFieldOffset
250      *        the offset of the length field
251      * @param lengthFieldLength
252      *        the length of the length field
253      * @param lengthAdjustment
254      *        the compensation value to add to the value of the length field
255      * @param initialBytesToStrip
256      *        the number of first bytes to strip out from the decoded frame
257      * @param failFast
258      *        If <tt>true</tt>, a {@link TooLongFrameException} is thrown as
259      *        soon as the decoder notices the length of the frame will exceed
260      *        <tt>maxFrameLength</tt> regardless of whether the entire frame
261      *        has been read.  If <tt>false</tt>, a {@link TooLongFrameException}
262      *        is thrown after the entire frame that exceeds <tt>maxFrameLength</tt>
263      *        has been read.
264      */
265     public LengthFieldBasedFrameDecoder(
266             int maxFrameLength, int lengthFieldOffset, int lengthFieldLength,
267             int lengthAdjustment, int initialBytesToStrip, boolean failFast) {
268         this(
269                 ByteOrder.BIG_ENDIAN, maxFrameLength, lengthFieldOffset, lengthFieldLength,
270                 lengthAdjustment, initialBytesToStrip, failFast);
271     }
272 
273     /**
274      * Creates a new instance.
275      *
276      * @param byteOrder
277      *        the {@link ByteOrder} of the length field
278      * @param maxFrameLength
279      *        the maximum length of the frame.  If the length of the frame is
280      *        greater than this value, {@link TooLongFrameException} will be
281      *        thrown.
282      * @param lengthFieldOffset
283      *        the offset of the length field
284      * @param lengthFieldLength
285      *        the length of the length field
286      * @param lengthAdjustment
287      *        the compensation value to add to the value of the length field
288      * @param initialBytesToStrip
289      *        the number of first bytes to strip out from the decoded frame
290      * @param failFast
291      *        If <tt>true</tt>, a {@link TooLongFrameException} is thrown as
292      *        soon as the decoder notices the length of the frame will exceed
293      *        <tt>maxFrameLength</tt> regardless of whether the entire frame
294      *        has been read.  If <tt>false</tt>, a {@link TooLongFrameException}
295      *        is thrown after the entire frame that exceeds <tt>maxFrameLength</tt>
296      *        has been read.
297      */
298     public LengthFieldBasedFrameDecoder(
299             ByteOrder byteOrder, int maxFrameLength, int lengthFieldOffset, int lengthFieldLength,
300             int lengthAdjustment, int initialBytesToStrip, boolean failFast) {
301         if (byteOrder == null) {
302             throw new NullPointerException("byteOrder");
303         }
304 
305         if (maxFrameLength <= 0) {
306             throw new IllegalArgumentException(
307                     "maxFrameLength must be a positive integer: " +
308                     maxFrameLength);
309         }
310 
311         if (lengthFieldOffset < 0) {
312             throw new IllegalArgumentException(
313                     "lengthFieldOffset must be a non-negative integer: " +
314                     lengthFieldOffset);
315         }
316 
317         if (initialBytesToStrip < 0) {
318             throw new IllegalArgumentException(
319                     "initialBytesToStrip must be a non-negative integer: " +
320                     initialBytesToStrip);
321         }
322 
323         if (lengthFieldOffset > maxFrameLength - lengthFieldLength) {
324             throw new IllegalArgumentException(
325                     "maxFrameLength (" + maxFrameLength + ") " +
326                     "must be equal to or greater than " +
327                     "lengthFieldOffset (" + lengthFieldOffset + ") + " +
328                     "lengthFieldLength (" + lengthFieldLength + ").");
329         }
330 
331         this.byteOrder = byteOrder;
332         this.maxFrameLength = maxFrameLength;
333         this.lengthFieldOffset = lengthFieldOffset;
334         this.lengthFieldLength = lengthFieldLength;
335         this.lengthAdjustment = lengthAdjustment;
336         lengthFieldEndOffset = lengthFieldOffset + lengthFieldLength;
337         this.initialBytesToStrip = initialBytesToStrip;
338         this.failFast = failFast;
339     }
340 
341     @Override
342     protected final void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception {
343         Object decoded = decode(ctx, in);
344         if (decoded != null) {
345             out.add(decoded);
346         }
347     }
348 
349     private void discardingTooLongFrame(ByteBuf in) {
350         long bytesToDiscard = this.bytesToDiscard;
351         int localBytesToDiscard = (int) Math.min(bytesToDiscard, in.readableBytes());
352         in.skipBytes(localBytesToDiscard);
353         bytesToDiscard -= localBytesToDiscard;
354         this.bytesToDiscard = bytesToDiscard;
355 
356         failIfNecessary(false);
357     }
358 
359     private static void failOnNegativeLengthField(ByteBuf in, long frameLength, int lengthFieldEndOffset) {
360         in.skipBytes(lengthFieldEndOffset);
361         throw new CorruptedFrameException(
362            "negative pre-adjustment length field: " + frameLength);
363     }
364 
365     private static void failOnFrameLengthLessThanLengthFieldEndOffset(ByteBuf in,
366                                                                       long frameLength,
367                                                                       int lengthFieldEndOffset) {
368         in.skipBytes(lengthFieldEndOffset);
369         throw new CorruptedFrameException(
370            "Adjusted frame length (" + frameLength + ") is less " +
371               "than lengthFieldEndOffset: " + lengthFieldEndOffset);
372     }
373 
374     private void exceededFrameLength(ByteBuf in, long frameLength) {
375         long discard = frameLength - in.readableBytes();
376         tooLongFrameLength = frameLength;
377 
378         if (discard < 0) {
379             // buffer contains more bytes then the frameLength so we can discard all now
380             in.skipBytes((int) frameLength);
381         } else {
382             // Enter the discard mode and discard everything received so far.
383             discardingTooLongFrame = true;
384             bytesToDiscard = discard;
385             in.skipBytes(in.readableBytes());
386         }
387         failIfNecessary(true);
388     }
389 
390     private static void failOnFrameLengthLessThanInitialBytesToStrip(ByteBuf in,
391                                                                      long frameLength,
392                                                                      int initialBytesToStrip) {
393         in.skipBytes((int) frameLength);
394         throw new CorruptedFrameException(
395            "Adjusted frame length (" + frameLength + ") is less " +
396               "than initialBytesToStrip: " + initialBytesToStrip);
397     }
398 
399     /**
400      * Create a frame out of the {@link ByteBuf} and return it.
401      *
402      * @param   ctx             the {@link ChannelHandlerContext} which this {@link ByteToMessageDecoder} belongs to
403      * @param   in              the {@link ByteBuf} from which to read data
404      * @return  frame           the {@link ByteBuf} which represent the frame or {@code null} if no frame could
405      *                          be created.
406      */
407     protected Object decode(ChannelHandlerContext ctx, ByteBuf in) throws Exception {
408         if (discardingTooLongFrame) {
409             discardingTooLongFrame(in);
410         }
411 
412         if (in.readableBytes() < lengthFieldEndOffset) {
413             return null;
414         }
415 
416         int actualLengthFieldOffset = in.readerIndex() + lengthFieldOffset;
417         long frameLength = getUnadjustedFrameLength(in, actualLengthFieldOffset, lengthFieldLength, byteOrder);
418 
419         if (frameLength < 0) {
420             failOnNegativeLengthField(in, frameLength, lengthFieldEndOffset);
421         }
422 
423         frameLength += lengthAdjustment + lengthFieldEndOffset;
424 
425         if (frameLength < lengthFieldEndOffset) {
426             failOnFrameLengthLessThanLengthFieldEndOffset(in, frameLength, lengthFieldEndOffset);
427         }
428 
429         if (frameLength > maxFrameLength) {
430             exceededFrameLength(in, frameLength);
431             return null;
432         }
433 
434         // never overflows because it's less than maxFrameLength
435         int frameLengthInt = (int) frameLength;
436         if (in.readableBytes() < frameLengthInt) {
437             return null;
438         }
439 
440         if (initialBytesToStrip > frameLengthInt) {
441             failOnFrameLengthLessThanInitialBytesToStrip(in, frameLength, initialBytesToStrip);
442         }
443         in.skipBytes(initialBytesToStrip);
444 
445         // extract frame
446         int readerIndex = in.readerIndex();
447         int actualFrameLength = frameLengthInt - initialBytesToStrip;
448         ByteBuf frame = extractFrame(ctx, in, readerIndex, actualFrameLength);
449         in.readerIndex(readerIndex + actualFrameLength);
450         return frame;
451     }
452 
453     /**
454      * Decodes the specified region of the buffer into an unadjusted frame length.  The default implementation is
455      * capable of decoding the specified region into an unsigned 8/16/24/32/64 bit integer.  Override this method to
456      * decode the length field encoded differently.  Note that this method must not modify the state of the specified
457      * buffer (e.g. {@code readerIndex}, {@code writerIndex}, and the content of the buffer.)
458      *
459      * @throws DecoderException if failed to decode the specified region
460      */
461     protected long getUnadjustedFrameLength(ByteBuf buf, int offset, int length, ByteOrder order) {
462         buf = buf.order(order);
463         long frameLength;
464         switch (length) {
465         case 1:
466             frameLength = buf.getUnsignedByte(offset);
467             break;
468         case 2:
469             frameLength = buf.getUnsignedShort(offset);
470             break;
471         case 3:
472             frameLength = buf.getUnsignedMedium(offset);
473             break;
474         case 4:
475             frameLength = buf.getUnsignedInt(offset);
476             break;
477         case 8:
478             frameLength = buf.getLong(offset);
479             break;
480         default:
481             throw new DecoderException(
482                     "unsupported lengthFieldLength: " + lengthFieldLength + " (expected: 1, 2, 3, 4, or 8)");
483         }
484         return frameLength;
485     }
486 
487     private void failIfNecessary(boolean firstDetectionOfTooLongFrame) {
488         if (bytesToDiscard == 0) {
489             // Reset to the initial state and tell the handlers that
490             // the frame was too large.
491             long tooLongFrameLength = this.tooLongFrameLength;
492             this.tooLongFrameLength = 0;
493             discardingTooLongFrame = false;
494             if (!failFast || firstDetectionOfTooLongFrame) {
495                 fail(tooLongFrameLength);
496             }
497         } else {
498             // Keep discarding and notify handlers if necessary.
499             if (failFast && firstDetectionOfTooLongFrame) {
500                 fail(tooLongFrameLength);
501             }
502         }
503     }
504 
505     /**
506      * Extract the sub-region of the specified buffer.
507      * <p>
508      * If you are sure that the frame and its content are not accessed after
509      * the current {@link #decode(ChannelHandlerContext, ByteBuf)}
510      * call returns, you can even avoid memory copy by returning the sliced
511      * sub-region (i.e. <tt>return buffer.slice(index, length)</tt>).
512      * It's often useful when you convert the extracted frame into an object.
513      * Refer to the source code of {@link ObjectDecoder} to see how this method
514      * is overridden to avoid memory copy.
515      */
516     protected ByteBuf extractFrame(ChannelHandlerContext ctx, ByteBuf buffer, int index, int length) {
517         return buffer.slice(index, length).retain();
518     }
519 
520     private void fail(long frameLength) {
521         if (frameLength > 0) {
522             throw new TooLongFrameException(
523                             "Adjusted frame length exceeds " + maxFrameLength +
524                             ": " + frameLength + " - discarded");
525         } else {
526             throw new TooLongFrameException(
527                             "Adjusted frame length exceeds " + maxFrameLength +
528                             " - discarding");
529         }
530     }
531 }