1 /*
2 * Copyright 2012 The Netty Project
3 *
4 * The Netty Project licenses this file to you under the Apache License,
5 * version 2.0 (the "License"); you may not use this file except in compliance
6 * with the License. You may obtain a copy of the License at:
7 *
8 * https://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13 * License for the specific language governing permissions and limitations
14 * under the License.
15 */
16 package io.netty.handler.codec;
17
18 import static io.netty.util.internal.ObjectUtil.checkPositive;
19
20 import io.netty.buffer.ByteBuf;
21 import io.netty.buffer.ByteBufUtil;
22 import io.netty.channel.ChannelHandlerContext;
23 import io.netty.util.internal.ObjectUtil;
24
25 import java.util.List;
26
27 /**
28 * A decoder that splits the received {@link ByteBuf}s by one or more
29 * delimiters. It is particularly useful for decoding the frames which ends
30 * with a delimiter such as {@link Delimiters#nulDelimiter() NUL} or
31 * {@linkplain Delimiters#lineDelimiter() newline characters}.
32 *
33 * <h3>Predefined delimiters</h3>
34 * <p>
35 * {@link Delimiters} defines frequently used delimiters for convenience' sake.
36 *
37 * <h3>Specifying more than one delimiter</h3>
38 * <p>
39 * {@link DelimiterBasedFrameDecoder} allows you to specify more than one
40 * delimiter. If more than one delimiter is found in the buffer, it chooses
41 * the delimiter which produces the shortest frame. For example, if you have
42 * the following data in the buffer:
43 * <pre>
44 * +--------------+
45 * | ABC\nDEF\r\n |
46 * +--------------+
47 * </pre>
48 * a {@link DelimiterBasedFrameDecoder}({@link Delimiters#lineDelimiter() Delimiters.lineDelimiter()})
49 * will choose {@code '\n'} as the first delimiter and produce two frames:
50 * <pre>
51 * +-----+-----+
52 * | ABC | DEF |
53 * +-----+-----+
54 * </pre>
55 * rather than incorrectly choosing {@code '\r\n'} as the first delimiter:
56 * <pre>
57 * +----------+
58 * | ABC\nDEF |
59 * +----------+
60 * </pre>
61 */
62 public class DelimiterBasedFrameDecoder extends ByteToMessageDecoder {
63
64 /** Size of the first window searched when more than one delimiter is configured. */
65 private static final int INITIAL_SCAN_WINDOW = 64;
66
67 private final ByteBuf[] delimiters;
68 private final int maxDelimiterLength;
69 private final int maxFrameLength;
70 private final boolean stripDelimiter;
71 private final boolean failFast;
72 private boolean discardingTooLongFrame;
73 private int tooLongFrameLength;
74 /** Set only when decoding with "\n" and "\r\n" as the delimiter. */
75 private final LineBasedFrameDecoder lineBasedDecoder;
76 /**
77 * Last scan position, relative to the reader index, at which no delimiter could be found.
78 * Resuming from this offset avoids rescanning the whole buffer from the reader index on
79 * every {@link #decode(ChannelHandlerContext, ByteBuf)} call.
80 */
81 private int scanOffset;
82
83 /**
84 * Creates a new instance.
85 *
86 * @param maxFrameLength the maximum length of the decoded frame.
87 * A {@link TooLongFrameException} is thrown if
88 * the length of the frame exceeds this value.
89 * @param delimiter the delimiter
90 */
91 public DelimiterBasedFrameDecoder(int maxFrameLength, ByteBuf delimiter) {
92 this(maxFrameLength, true, delimiter);
93 }
94
95 /**
96 * Creates a new instance.
97 *
98 * @param maxFrameLength the maximum length of the decoded frame.
99 * A {@link TooLongFrameException} is thrown if
100 * the length of the frame exceeds this value.
101 * @param stripDelimiter whether the decoded frame should strip out the
102 * delimiter or not
103 * @param delimiter the delimiter
104 */
105 public DelimiterBasedFrameDecoder(
106 int maxFrameLength, boolean stripDelimiter, ByteBuf delimiter) {
107 this(maxFrameLength, stripDelimiter, true, delimiter);
108 }
109
110 /**
111 * Creates a new instance.
112 *
113 * @param maxFrameLength the maximum length of the decoded frame.
114 * A {@link TooLongFrameException} is thrown if
115 * the length of the frame exceeds this value.
116 * @param stripDelimiter whether the decoded frame should strip out the
117 * delimiter or not
118 * @param failFast If <tt>true</tt>, a {@link TooLongFrameException} is
119 * thrown as soon as the decoder notices the length of the
120 * frame will exceed <tt>maxFrameLength</tt> regardless of
121 * whether the entire frame has been read.
122 * If <tt>false</tt>, a {@link TooLongFrameException} is
123 * thrown after the entire frame that exceeds
124 * <tt>maxFrameLength</tt> has been read.
125 * @param delimiter the delimiter
126 */
127 public DelimiterBasedFrameDecoder(
128 int maxFrameLength, boolean stripDelimiter, boolean failFast,
129 ByteBuf delimiter) {
130 this(maxFrameLength, stripDelimiter, failFast, new ByteBuf[] {
131 delimiter.slice(delimiter.readerIndex(), delimiter.readableBytes())});
132 }
133
134 /**
135 * Creates a new instance.
136 *
137 * @param maxFrameLength the maximum length of the decoded frame.
138 * A {@link TooLongFrameException} is thrown if
139 * the length of the frame exceeds this value.
140 * @param delimiters the delimiters
141 */
142 public DelimiterBasedFrameDecoder(int maxFrameLength, ByteBuf... delimiters) {
143 this(maxFrameLength, true, delimiters);
144 }
145
146 /**
147 * Creates a new instance.
148 *
149 * @param maxFrameLength the maximum length of the decoded frame.
150 * A {@link TooLongFrameException} is thrown if
151 * the length of the frame exceeds this value.
152 * @param stripDelimiter whether the decoded frame should strip out the
153 * delimiter or not
154 * @param delimiters the delimiters
155 */
156 public DelimiterBasedFrameDecoder(
157 int maxFrameLength, boolean stripDelimiter, ByteBuf... delimiters) {
158 this(maxFrameLength, stripDelimiter, true, delimiters);
159 }
160
161 /**
162 * Creates a new instance.
163 *
164 * @param maxFrameLength the maximum length of the decoded frame.
165 * A {@link TooLongFrameException} is thrown if
166 * the length of the frame exceeds this value.
167 * @param stripDelimiter whether the decoded frame should strip out the
168 * delimiter or not
169 * @param failFast If <tt>true</tt>, a {@link TooLongFrameException} is
170 * thrown as soon as the decoder notices the length of the
171 * frame will exceed <tt>maxFrameLength</tt> regardless of
172 * whether the entire frame has been read.
173 * If <tt>false</tt>, a {@link TooLongFrameException} is
174 * thrown after the entire frame that exceeds
175 * <tt>maxFrameLength</tt> has been read.
176 * @param delimiters the delimiters
177 */
178 public DelimiterBasedFrameDecoder(
179 int maxFrameLength, boolean stripDelimiter, boolean failFast, ByteBuf... delimiters) {
180 validateMaxFrameLength(maxFrameLength);
181 ObjectUtil.checkNonEmpty(delimiters, "delimiters");
182
183 if (isLineBased(delimiters) && !isSubclass()) {
184 lineBasedDecoder = new LineBasedFrameDecoder(maxFrameLength, stripDelimiter, failFast);
185 this.delimiters = null;
186 this.maxDelimiterLength = 0;
187 } else {
188 this.delimiters = new ByteBuf[delimiters.length];
189 int maxDelimiterLength = 0;
190 for (int i = 0; i < delimiters.length; i ++) {
191 ByteBuf d = delimiters[i];
192 validateDelimiter(d);
193 this.delimiters[i] = d.slice(d.readerIndex(), d.readableBytes());
194 maxDelimiterLength = Math.max(maxDelimiterLength, this.delimiters[i].readableBytes());
195 }
196 this.maxDelimiterLength = maxDelimiterLength;
197 lineBasedDecoder = null;
198 }
199 this.maxFrameLength = maxFrameLength;
200 this.stripDelimiter = stripDelimiter;
201 this.failFast = failFast;
202 }
203
204 /** Returns true if the delimiters are "\n" and "\r\n". */
205 private static boolean isLineBased(final ByteBuf[] delimiters) {
206 if (delimiters.length != 2) {
207 return false;
208 }
209 ByteBuf a = delimiters[0];
210 ByteBuf b = delimiters[1];
211 if (a.capacity() < b.capacity()) {
212 a = delimiters[1];
213 b = delimiters[0];
214 }
215 return a.capacity() == 2 && b.capacity() == 1
216 && a.getByte(0) == '\r' && a.getByte(1) == '\n'
217 && b.getByte(0) == '\n';
218 }
219
220 /**
221 * Return {@code true} if the current instance is a subclass of DelimiterBasedFrameDecoder
222 */
223 private boolean isSubclass() {
224 return getClass() != DelimiterBasedFrameDecoder.class;
225 }
226
227 @Override
228 protected final void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception {
229 Object decoded = decode(ctx, in);
230 if (decoded != null) {
231 out.add(decoded);
232 }
233 }
234
235 /**
236 * Create a frame out of the {@link ByteBuf} and return it.
237 *
238 * @param ctx the {@link ChannelHandlerContext} which this {@link ByteToMessageDecoder} belongs to
239 * @param buffer the {@link ByteBuf} from which to read data
240 * @return frame the {@link ByteBuf} which represent the frame or {@code null} if no frame could
241 * be created.
242 */
243 protected Object decode(ChannelHandlerContext ctx, ByteBuf buffer) throws Exception {
244 if (lineBasedDecoder != null) {
245 return lineBasedDecoder.decode(ctx, buffer);
246 }
247 // Try all delimiters and choose the delimiter which yields the shortest frame.
248 int minFrameLength = Integer.MAX_VALUE;
249 ByteBuf minDelim = null;
250 final int readableBytes = buffer.readableBytes();
251 // Searching every delimiter through the whole remaining buffer would cost O(readableBytes) per delimiter
252 // even if another delimiter matches right at the start, which is quadratic when many small frames are
253 // contained in one large buffer. With more than one delimiter we therefore search in windows that double in
254 // size, so the work done is bounded by the length of the frame we find rather than by the buffer size.
255 long window = delimiters.length == 1 ? readableBytes : Math.max(INITIAL_SCAN_WINDOW, maxDelimiterLength);
256 int from = scanOffset;
257 while (from < readableBytes) {
258 // Extend the region by maxDelimiterLength - 1 so a delimiter that starts inside the window but ends
259 // after it is still found.
260 int end = (int) Math.min(readableBytes, from + window + maxDelimiterLength - 1);
261 ByteBuf region = buffer.slice(buffer.readerIndex() + from, end - from);
262 for (ByteBuf delim: delimiters) {
263 int index = ByteBufUtil.indexOf(delim, region);
264 if (index >= 0 && from + index < minFrameLength) {
265 minFrameLength = from + index;
266 minDelim = delim;
267 }
268 }
269 if (minDelim != null && (end == readableBytes || minFrameLength < from + window)) {
270 // All matches that start before the current minimum are guaranteed to be inside the region.
271 break;
272 }
273 // Either nothing was found, or the only matches start in the overlap and another delimiter might
274 // match earlier. Continue with the next window, which will find those matches again.
275 minFrameLength = Integer.MAX_VALUE;
276 minDelim = null;
277 from = (int) Math.min(readableBytes, from + window);
278 window = Math.min(window << 1, Integer.MAX_VALUE);
279 }
280
281 if (minDelim != null) {
282 // A delimiter was found; the next decode() call starts scanning fresh data only.
283 scanOffset = 0;
284 int minDelimLength = minDelim.capacity();
285 ByteBuf frame;
286
287 if (discardingTooLongFrame) {
288 // We've just finished discarding a very large frame.
289 // Go back to the initial state.
290 discardingTooLongFrame = false;
291 buffer.skipBytes(minFrameLength + minDelimLength);
292
293 int tooLongFrameLength = this.tooLongFrameLength;
294 this.tooLongFrameLength = 0;
295 if (!failFast) {
296 fail(tooLongFrameLength);
297 }
298 return null;
299 }
300
301 if (minFrameLength > maxFrameLength) {
302 // Discard read frame.
303 buffer.skipBytes(minFrameLength + minDelimLength);
304 fail(minFrameLength);
305 return null;
306 }
307
308 if (stripDelimiter) {
309 frame = buffer.readRetainedSlice(minFrameLength);
310 buffer.skipBytes(minDelimLength);
311 } else {
312 frame = buffer.readRetainedSlice(minFrameLength + minDelimLength);
313 }
314
315 return frame;
316 } else {
317 if (!discardingTooLongFrame) {
318 if (readableBytes > maxFrameLength) {
319 // Discard the content of the buffer until a delimiter is found.
320 tooLongFrameLength = readableBytes;
321 buffer.skipBytes(readableBytes);
322 discardingTooLongFrame = true;
323 scanOffset = 0;
324 if (failFast) {
325 fail(tooLongFrameLength);
326 }
327 } else {
328 // No delimiter found yet. Remember how far we've already scanned so the next
329 // call doesn't rescan bytes that cannot possibly contain the start of a match.
330 scanOffset = Math.max(0, readableBytes - maxDelimiterLength + 1);
331 }
332 } else {
333 // Still discarding the buffer since a delimiter is not found.
334 tooLongFrameLength += buffer.readableBytes();
335 buffer.skipBytes(buffer.readableBytes());
336 scanOffset = 0;
337 }
338 return null;
339 }
340 }
341
342 private void fail(long frameLength) {
343 if (frameLength > 0) {
344 throw new TooLongFrameException(
345 "frame length exceeds " + maxFrameLength +
346 ": " + frameLength + " - discarded");
347 } else {
348 throw new TooLongFrameException(
349 "frame length exceeds " + maxFrameLength +
350 " - discarding");
351 }
352 }
353
354 private static void validateDelimiter(ByteBuf delimiter) {
355 ObjectUtil.checkNotNull(delimiter, "delimiter");
356 if (!delimiter.isReadable()) {
357 throw new IllegalArgumentException("empty delimiter");
358 }
359 }
360
361 private static void validateMaxFrameLength(int maxFrameLength) {
362 checkPositive(maxFrameLength, "maxFrameLength");
363 }
364 }