View Javadoc
1   /*
2    * Copyright 2012 The Netty Project
3    *
4    * The Netty Project licenses this file to you under the Apache License,
5    * version 2.0 (the "License"); you may not use this file except in compliance
6    * with the License. You may obtain a copy of the License at:
7    *
8    *   https://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13   * License for the specific language governing permissions and limitations
14   * under the License.
15   */
16  package io.netty.handler.codec.http;
17  
18  import static io.netty.util.internal.ObjectUtil.checkPositive;
19  
20  import io.netty.buffer.ByteBuf;
21  import io.netty.buffer.Unpooled;
22  import io.netty.channel.ChannelHandlerContext;
23  import io.netty.channel.ChannelPipeline;
24  import io.netty.handler.codec.ByteToMessageDecoder;
25  import io.netty.handler.codec.DecoderResult;
26  import io.netty.handler.codec.PrematureChannelClosureException;
27  import io.netty.handler.codec.TooLongFrameException;
28  import io.netty.util.AsciiString;
29  import io.netty.util.ByteProcessor;
30  import io.netty.util.internal.StringUtil;
31  
32  import java.util.List;
33  import java.util.concurrent.atomic.AtomicBoolean;
34  
35  /**
36   * Decodes {@link ByteBuf}s into {@link HttpMessage}s and
37   * {@link HttpContent}s.
38   *
39   * <h3>Parameters that prevents excessive memory consumption</h3>
40   * <table border="1">
41   * <tr>
42   * <th>Name</th><th>Default value</th><th>Meaning</th>
43   * </tr>
44   * <tr>
45   * <td>{@code maxInitialLineLength}</td>
46   * <td>{@value #DEFAULT_MAX_INITIAL_LINE_LENGTH}</td>
47   * <td>The maximum length of the initial line
48   *     (e.g. {@code "GET / HTTP/1.0"} or {@code "HTTP/1.0 200 OK"})
49   *     If the length of the initial line exceeds this value, a
50   *     {@link TooLongHttpLineException} will be raised.</td>
51   * </tr>
52   * <tr>
53   * <td>{@code maxHeaderSize}</td>
54   * <td>{@value #DEFAULT_MAX_HEADER_SIZE}</td>
55   * <td>The maximum length of all headers.  If the sum of the length of each
56   *     header exceeds this value, a {@link TooLongHttpHeaderException} will be raised.</td>
57   * </tr>
58   * <tr>
59   * <td>{@code maxChunkSize}</td>
60   * <td>{@value #DEFAULT_MAX_CHUNK_SIZE}</td>
61   * <td>The maximum length of the content or each chunk.  If the content length
62   *     (or the length of each chunk) exceeds this value, the content or chunk
63   *     will be split into multiple {@link HttpContent}s whose length is
64   *     {@code maxChunkSize} at maximum.</td>
65   * </tr>
66   * </table>
67   *
68   * <h3>Parameters that control parsing behavior</h3>
69   * <table border="1">
70   * <tr>
71   * <th>Name</th><th>Default value</th><th>Meaning</th>
72   * </tr>
73   * <tr>
74   * <td>{@code allowDuplicateContentLengths}</td>
75   * <td>{@value #DEFAULT_ALLOW_DUPLICATE_CONTENT_LENGTHS}</td>
76   * <td>When set to {@code false}, will reject any messages that contain multiple Content-Length header fields.
77   *     When set to {@code true}, will allow multiple Content-Length headers only if they are all the same decimal value.
78   *     The duplicated field-values will be replaced with a single valid Content-Length field.
79   *     See <a href="https://tools.ietf.org/html/rfc7230#section-3.3.2">RFC 7230, Section 3.3.2</a>.</td>
80   * </tr>
81   * <tr>
82   * <td>{@code allowPartialChunks}</td>
83   * <td>{@value #DEFAULT_ALLOW_PARTIAL_CHUNKS}</td>
84   * <td>If the length of a chunk exceeds the {@link ByteBuf}s readable bytes and {@code allowPartialChunks}
85   *     is set to {@code true}, the chunk will be split into multiple {@link HttpContent}s.
86   *     Otherwise, if the chunk size does not exceed {@code maxChunkSize} and {@code allowPartialChunks}
87   *     is set to {@code false}, the {@link ByteBuf} is not decoded into an {@link HttpContent} until
88   *     the readable bytes are greater or equal to the chunk size.</td>
89   * </tr>
90   * </table>
91   *
92   * <h3>Chunked Content</h3>
93   *
94   * If the content of an HTTP message is greater than {@code maxChunkSize} or
95   * the transfer encoding of the HTTP message is 'chunked', this decoder
96   * generates one {@link HttpMessage} instance and its following
97   * {@link HttpContent}s per single HTTP message to avoid excessive memory
98   * consumption. For example, the following HTTP message:
99   * <pre>
100  * GET / HTTP/1.1
101  * Transfer-Encoding: chunked
102  *
103  * 1a
104  * abcdefghijklmnopqrstuvwxyz
105  * 10
106  * 1234567890abcdef
107  * 0
108  * Content-MD5: ...
109  * <i>[blank line]</i>
110  * </pre>
111  * triggers {@link HttpRequestDecoder} to generate 3 objects:
112  * <ol>
113  * <li>An {@link HttpRequest},</li>
114  * <li>The first {@link HttpContent} whose content is {@code 'abcdefghijklmnopqrstuvwxyz'},</li>
115  * <li>The second {@link LastHttpContent} whose content is {@code '1234567890abcdef'}, which marks
116  * the end of the content.</li>
117  * </ol>
118  *
119  * If you prefer not to handle {@link HttpContent}s by yourself for your
120  * convenience, insert {@link HttpObjectAggregator} after this decoder in the
121  * {@link ChannelPipeline}.  However, please note that your server might not
122  * be as memory efficient as without the aggregator.
123  *
124  * <h3>Extensibility</h3>
125  *
126  * Please note that this decoder is designed to be extended to implement
127  * a protocol derived from HTTP, such as
128  * <a href="https://en.wikipedia.org/wiki/Real_Time_Streaming_Protocol">RTSP</a> and
129  * <a href="https://en.wikipedia.org/wiki/Internet_Content_Adaptation_Protocol">ICAP</a>.
130  * To implement the decoder of such a derived protocol, extend this class and
131  * implement all abstract methods properly.
132  */
133 public abstract class HttpObjectDecoder extends ByteToMessageDecoder {
134     public static final int DEFAULT_MAX_INITIAL_LINE_LENGTH = 4096;
135     public static final int DEFAULT_MAX_HEADER_SIZE = 8192;
136     public static final boolean DEFAULT_CHUNKED_SUPPORTED = true;
137     public static final boolean DEFAULT_ALLOW_PARTIAL_CHUNKS = true;
138     public static final int DEFAULT_MAX_CHUNK_SIZE = 8192;
139     public static final boolean DEFAULT_VALIDATE_HEADERS = true;
140     public static final int DEFAULT_INITIAL_BUFFER_SIZE = 128;
141     public static final boolean DEFAULT_ALLOW_DUPLICATE_CONTENT_LENGTHS = false;
142     private final int maxChunkSize;
143     private final boolean chunkedSupported;
144     private final boolean allowPartialChunks;
145     protected final boolean validateHeaders;
146     private final boolean allowDuplicateContentLengths;
147     private final ByteBuf parserScratchBuffer;
148     private final HeaderParser headerParser;
149     private final LineParser lineParser;
150 
151     private HttpMessage message;
152     private long chunkSize;
153     private long contentLength = Long.MIN_VALUE;
154     private final AtomicBoolean resetRequested = new AtomicBoolean();
155 
156     // These will be updated by splitHeader(...)
157     private AsciiString name;
158     private String value;
159     private LastHttpContent trailer;
160 
161     @Override
162     protected void handlerRemoved0(ChannelHandlerContext ctx) throws Exception {
163         try {
164             parserScratchBuffer.release();
165         } finally {
166             super.handlerRemoved0(ctx);
167         }
168     }
169 
170     /**
171      * The internal state of {@link HttpObjectDecoder}.
172      * <em>Internal use only</em>.
173      */
174     private enum State {
175         SKIP_CONTROL_CHARS,
176         READ_INITIAL,
177         READ_HEADER,
178         READ_VARIABLE_LENGTH_CONTENT,
179         READ_FIXED_LENGTH_CONTENT,
180         READ_CHUNK_SIZE,
181         READ_CHUNKED_CONTENT,
182         READ_CHUNK_DELIMITER,
183         READ_CHUNK_FOOTER,
184         BAD_MESSAGE,
185         UPGRADED
186     }
187 
188     private State currentState = State.SKIP_CONTROL_CHARS;
189 
190     /**
191      * Creates a new instance with the default
192      * {@code maxInitialLineLength (4096)}, {@code maxHeaderSize (8192)}, and
193      * {@code maxChunkSize (8192)}.
194      */
195     protected HttpObjectDecoder() {
196         this(DEFAULT_MAX_INITIAL_LINE_LENGTH, DEFAULT_MAX_HEADER_SIZE, DEFAULT_MAX_CHUNK_SIZE,
197              DEFAULT_CHUNKED_SUPPORTED);
198     }
199 
200     /**
201      * Creates a new instance with the specified parameters.
202      */
203     protected HttpObjectDecoder(
204             int maxInitialLineLength, int maxHeaderSize, int maxChunkSize, boolean chunkedSupported) {
205         this(maxInitialLineLength, maxHeaderSize, maxChunkSize, chunkedSupported, DEFAULT_VALIDATE_HEADERS);
206     }
207 
208     /**
209      * Creates a new instance with the specified parameters.
210      */
211     protected HttpObjectDecoder(
212             int maxInitialLineLength, int maxHeaderSize, int maxChunkSize,
213             boolean chunkedSupported, boolean validateHeaders) {
214         this(maxInitialLineLength, maxHeaderSize, maxChunkSize, chunkedSupported, validateHeaders,
215              DEFAULT_INITIAL_BUFFER_SIZE);
216     }
217 
218     /**
219      * Creates a new instance with the specified parameters.
220      */
221     protected HttpObjectDecoder(
222             int maxInitialLineLength, int maxHeaderSize, int maxChunkSize,
223             boolean chunkedSupported, boolean validateHeaders, int initialBufferSize) {
224         this(maxInitialLineLength, maxHeaderSize, maxChunkSize, chunkedSupported, validateHeaders, initialBufferSize,
225              DEFAULT_ALLOW_DUPLICATE_CONTENT_LENGTHS);
226     }
227 
228     /**
229      * Creates a new instance with the specified parameters.
230      */
231     protected HttpObjectDecoder(
232             int maxInitialLineLength, int maxHeaderSize, int maxChunkSize,
233             boolean chunkedSupported, boolean validateHeaders, int initialBufferSize,
234             boolean allowDuplicateContentLengths) {
235         this(maxInitialLineLength, maxHeaderSize, maxChunkSize, chunkedSupported, validateHeaders, initialBufferSize,
236             allowDuplicateContentLengths, DEFAULT_ALLOW_PARTIAL_CHUNKS);
237     }
238 
239     /**
240      * Creates a new instance with the specified parameters.
241      */
242     protected HttpObjectDecoder(
243             int maxInitialLineLength, int maxHeaderSize, int maxChunkSize,
244             boolean chunkedSupported, boolean validateHeaders, int initialBufferSize,
245             boolean allowDuplicateContentLengths, boolean allowPartialChunks) {
246         checkPositive(maxInitialLineLength, "maxInitialLineLength");
247         checkPositive(maxHeaderSize, "maxHeaderSize");
248         checkPositive(maxChunkSize, "maxChunkSize");
249 
250         parserScratchBuffer = Unpooled.buffer(initialBufferSize);
251         lineParser = new LineParser(parserScratchBuffer, maxInitialLineLength);
252         headerParser = new HeaderParser(parserScratchBuffer, maxHeaderSize);
253         this.maxChunkSize = maxChunkSize;
254         this.chunkedSupported = chunkedSupported;
255         this.validateHeaders = validateHeaders;
256         this.allowDuplicateContentLengths = allowDuplicateContentLengths;
257         this.allowPartialChunks = allowPartialChunks;
258     }
259 
260     @Override
261     protected void decode(ChannelHandlerContext ctx, ByteBuf buffer, List<Object> out) throws Exception {
262         if (resetRequested.get()) {
263             resetNow();
264         }
265 
266         switch (currentState) {
267         case SKIP_CONTROL_CHARS:
268             // Fall-through
269         case READ_INITIAL: try {
270             ByteBuf line = lineParser.parse(buffer);
271             if (line == null) {
272                 return;
273             }
274             final String[] initialLine = splitInitialLine(line);
275             assert initialLine.length == 3 : "initialLine::length must be 3";
276 
277             message = createMessage(initialLine);
278             currentState = State.READ_HEADER;
279             // fall-through
280         } catch (Exception e) {
281             out.add(invalidMessage(buffer, e));
282             return;
283         }
284         case READ_HEADER: try {
285             State nextState = readHeaders(buffer);
286             if (nextState == null) {
287                 return;
288             }
289             currentState = nextState;
290             switch (nextState) {
291             case SKIP_CONTROL_CHARS:
292                 // fast-path
293                 // No content is expected.
294                 out.add(message);
295                 out.add(LastHttpContent.EMPTY_LAST_CONTENT);
296                 resetNow();
297                 return;
298             case READ_CHUNK_SIZE:
299                 if (!chunkedSupported) {
300                     throw new IllegalArgumentException("Chunked messages not supported");
301                 }
302                 // Chunked encoding - generate HttpMessage first.  HttpChunks will follow.
303                 out.add(message);
304                 return;
305             default:
306                 /**
307                  * <a href="https://tools.ietf.org/html/rfc7230#section-3.3.3">RFC 7230, 3.3.3</a> states that if a
308                  * request does not have either a transfer-encoding or a content-length header then the message body
309                  * length is 0. However for a response the body length is the number of octets received prior to the
310                  * server closing the connection. So we treat this as variable length chunked encoding.
311                  */
312                 long contentLength = contentLength();
313                 if (contentLength == 0 || contentLength == -1 && isDecodingRequest()) {
314                     out.add(message);
315                     out.add(LastHttpContent.EMPTY_LAST_CONTENT);
316                     resetNow();
317                     return;
318                 }
319 
320                 assert nextState == State.READ_FIXED_LENGTH_CONTENT ||
321                         nextState == State.READ_VARIABLE_LENGTH_CONTENT;
322 
323                 out.add(message);
324 
325                 if (nextState == State.READ_FIXED_LENGTH_CONTENT) {
326                     // chunkSize will be decreased as the READ_FIXED_LENGTH_CONTENT state reads data chunk by chunk.
327                     chunkSize = contentLength;
328                 }
329 
330                 // We return here, this forces decode to be called again where we will decode the content
331                 return;
332             }
333         } catch (Exception e) {
334             out.add(invalidMessage(buffer, e));
335             return;
336         }
337         case READ_VARIABLE_LENGTH_CONTENT: {
338             // Keep reading data as a chunk until the end of connection is reached.
339             int toRead = Math.min(buffer.readableBytes(), maxChunkSize);
340             if (toRead > 0) {
341                 ByteBuf content = buffer.readRetainedSlice(toRead);
342                 out.add(new DefaultHttpContent(content));
343             }
344             return;
345         }
346         case READ_FIXED_LENGTH_CONTENT: {
347             int readLimit = buffer.readableBytes();
348 
349             // Check if the buffer is readable first as we use the readable byte count
350             // to create the HttpChunk. This is needed as otherwise we may end up with
351             // create an HttpChunk instance that contains an empty buffer and so is
352             // handled like it is the last HttpChunk.
353             //
354             // See https://github.com/netty/netty/issues/433
355             if (readLimit == 0) {
356                 return;
357             }
358 
359             int toRead = Math.min(readLimit, maxChunkSize);
360             if (toRead > chunkSize) {
361                 toRead = (int) chunkSize;
362             }
363             ByteBuf content = buffer.readRetainedSlice(toRead);
364             chunkSize -= toRead;
365 
366             if (chunkSize == 0) {
367                 // Read all content.
368                 out.add(new DefaultLastHttpContent(content, validateHeaders));
369                 resetNow();
370             } else {
371                 out.add(new DefaultHttpContent(content));
372             }
373             return;
374         }
375         /**
376          * everything else after this point takes care of reading chunked content. basically, read chunk size,
377          * read chunk, read and ignore the CRLF and repeat until 0
378          */
379         case READ_CHUNK_SIZE: try {
380             ByteBuf line = lineParser.parse(buffer);
381             if (line == null) {
382                 return;
383             }
384             int chunkSize = getChunkSize(line.array(), line.arrayOffset() + line.readerIndex(), line.readableBytes());
385             this.chunkSize = chunkSize;
386             if (chunkSize == 0) {
387                 currentState = State.READ_CHUNK_FOOTER;
388                 return;
389             }
390             currentState = State.READ_CHUNKED_CONTENT;
391             // fall-through
392         } catch (Exception e) {
393             out.add(invalidChunk(buffer, e));
394             return;
395         }
396         case READ_CHUNKED_CONTENT: {
397             assert chunkSize <= Integer.MAX_VALUE;
398             int toRead = Math.min((int) chunkSize, maxChunkSize);
399             if (!allowPartialChunks && buffer.readableBytes() < toRead) {
400                 return;
401             }
402             toRead = Math.min(toRead, buffer.readableBytes());
403             if (toRead == 0) {
404                 return;
405             }
406             HttpContent chunk = new DefaultHttpContent(buffer.readRetainedSlice(toRead));
407             chunkSize -= toRead;
408 
409             out.add(chunk);
410 
411             if (chunkSize != 0) {
412                 return;
413             }
414             currentState = State.READ_CHUNK_DELIMITER;
415             // fall-through
416         }
417         case READ_CHUNK_DELIMITER: {
418             final int wIdx = buffer.writerIndex();
419             int rIdx = buffer.readerIndex();
420             while (wIdx > rIdx) {
421                 byte next = buffer.getByte(rIdx++);
422                 if (next == HttpConstants.LF) {
423                     currentState = State.READ_CHUNK_SIZE;
424                     break;
425                 }
426             }
427             buffer.readerIndex(rIdx);
428             return;
429         }
430         case READ_CHUNK_FOOTER: try {
431             LastHttpContent trailer = readTrailingHeaders(buffer);
432             if (trailer == null) {
433                 return;
434             }
435             out.add(trailer);
436             resetNow();
437             return;
438         } catch (Exception e) {
439             out.add(invalidChunk(buffer, e));
440             return;
441         }
442         case BAD_MESSAGE: {
443             // Keep discarding until disconnection.
444             buffer.skipBytes(buffer.readableBytes());
445             break;
446         }
447         case UPGRADED: {
448             int readableBytes = buffer.readableBytes();
449             if (readableBytes > 0) {
450                 // Keep on consuming as otherwise we may trigger an DecoderException,
451                 // other handler will replace this codec with the upgraded protocol codec to
452                 // take the traffic over at some point then.
453                 // See https://github.com/netty/netty/issues/2173
454                 out.add(buffer.readBytes(readableBytes));
455             }
456             break;
457         }
458         default:
459             break;
460         }
461     }
462 
463     @Override
464     protected void decodeLast(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception {
465         super.decodeLast(ctx, in, out);
466 
467         if (resetRequested.get()) {
468             // If a reset was requested by decodeLast() we need to do it now otherwise we may produce a
469             // LastHttpContent while there was already one.
470             resetNow();
471         }
472         // Handle the last unfinished message.
473         if (message != null) {
474             boolean chunked = HttpUtil.isTransferEncodingChunked(message);
475             if (currentState == State.READ_VARIABLE_LENGTH_CONTENT && !in.isReadable() && !chunked) {
476                 // End of connection.
477                 out.add(LastHttpContent.EMPTY_LAST_CONTENT);
478                 resetNow();
479                 return;
480             }
481 
482             if (currentState == State.READ_HEADER) {
483                 // If we are still in the state of reading headers we need to create a new invalid message that
484                 // signals that the connection was closed before we received the headers.
485                 out.add(invalidMessage(Unpooled.EMPTY_BUFFER,
486                         new PrematureChannelClosureException("Connection closed before received headers")));
487                 resetNow();
488                 return;
489             }
490 
491             // Check if the closure of the connection signifies the end of the content.
492             boolean prematureClosure;
493             if (isDecodingRequest() || chunked) {
494                 // The last request did not wait for a response.
495                 prematureClosure = true;
496             } else {
497                 // Compare the length of the received content and the 'Content-Length' header.
498                 // If the 'Content-Length' header is absent, the length of the content is determined by the end of the
499                 // connection, so it is perfectly fine.
500                 prematureClosure = contentLength() > 0;
501             }
502 
503             if (!prematureClosure) {
504                 out.add(LastHttpContent.EMPTY_LAST_CONTENT);
505             }
506             resetNow();
507         }
508     }
509 
510     @Override
511     public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {
512         if (evt instanceof HttpExpectationFailedEvent) {
513             switch (currentState) {
514             case READ_FIXED_LENGTH_CONTENT:
515             case READ_VARIABLE_LENGTH_CONTENT:
516             case READ_CHUNK_SIZE:
517                 reset();
518                 break;
519             default:
520                 break;
521             }
522         }
523         super.userEventTriggered(ctx, evt);
524     }
525 
526     protected boolean isContentAlwaysEmpty(HttpMessage msg) {
527         if (msg instanceof HttpResponse) {
528             HttpResponse res = (HttpResponse) msg;
529             final HttpResponseStatus status = res.status();
530             final int code = status.code();
531             final HttpStatusClass statusClass = status.codeClass();
532 
533             // Correctly handle return codes of 1xx.
534             //
535             // See:
536             //     - https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html Section 4.4
537             //     - https://github.com/netty/netty/issues/222
538             if (statusClass == HttpStatusClass.INFORMATIONAL) {
539                 // One exception: Hixie 76 websocket handshake response
540                 return !(code == 101 && !res.headers().contains(HttpHeaderNames.SEC_WEBSOCKET_ACCEPT)
541                          && res.headers().contains(HttpHeaderNames.UPGRADE, HttpHeaderValues.WEBSOCKET, true));
542             }
543 
544             switch (code) {
545             case 204: case 304:
546                 return true;
547             default:
548                 return false;
549             }
550         }
551         return false;
552     }
553 
554     /**
555      * Returns true if the server switched to a different protocol than HTTP/1.0 or HTTP/1.1, e.g. HTTP/2 or Websocket.
556      * Returns false if the upgrade happened in a different layer, e.g. upgrade from HTTP/1.1 to HTTP/1.1 over TLS.
557      */
558     protected boolean isSwitchingToNonHttp1Protocol(HttpResponse msg) {
559         if (msg.status().code() != HttpResponseStatus.SWITCHING_PROTOCOLS.code()) {
560             return false;
561         }
562         String newProtocol = msg.headers().get(HttpHeaderNames.UPGRADE);
563         return newProtocol == null ||
564                 !newProtocol.contains(HttpVersion.HTTP_1_0.text()) &&
565                 !newProtocol.contains(HttpVersion.HTTP_1_1.text());
566     }
567 
568     /**
569      * Resets the state of the decoder so that it is ready to decode a new message.
570      * This method is useful for handling a rejected request with {@code Expect: 100-continue} header.
571      */
572     public void reset() {
573         resetRequested.lazySet(true);
574     }
575 
576     private void resetNow() {
577         HttpMessage message = this.message;
578         this.message = null;
579         name = null;
580         value = null;
581         contentLength = Long.MIN_VALUE;
582         lineParser.reset();
583         headerParser.reset();
584         trailer = null;
585         if (!isDecodingRequest()) {
586             HttpResponse res = (HttpResponse) message;
587             if (res != null && isSwitchingToNonHttp1Protocol(res)) {
588                 currentState = State.UPGRADED;
589                 return;
590             }
591         }
592 
593         resetRequested.lazySet(false);
594         currentState = State.SKIP_CONTROL_CHARS;
595     }
596 
597     private HttpMessage invalidMessage(ByteBuf in, Exception cause) {
598         currentState = State.BAD_MESSAGE;
599 
600         // Advance the readerIndex so that ByteToMessageDecoder does not complain
601         // when we produced an invalid message without consuming anything.
602         in.skipBytes(in.readableBytes());
603 
604         if (message == null) {
605             message = createInvalidMessage();
606         }
607         message.setDecoderResult(DecoderResult.failure(cause));
608 
609         HttpMessage ret = message;
610         message = null;
611         return ret;
612     }
613 
614     private HttpContent invalidChunk(ByteBuf in, Exception cause) {
615         currentState = State.BAD_MESSAGE;
616 
617         // Advance the readerIndex so that ByteToMessageDecoder does not complain
618         // when we produced an invalid message without consuming anything.
619         in.skipBytes(in.readableBytes());
620 
621         HttpContent chunk = new DefaultLastHttpContent(Unpooled.EMPTY_BUFFER);
622         chunk.setDecoderResult(DecoderResult.failure(cause));
623         message = null;
624         trailer = null;
625         return chunk;
626     }
627 
628     private State readHeaders(ByteBuf buffer) {
629         final HttpMessage message = this.message;
630         final HttpHeaders headers = message.headers();
631 
632         final HeaderParser headerParser = this.headerParser;
633 
634         ByteBuf line = headerParser.parse(buffer);
635         if (line == null) {
636             return null;
637         }
638         int lineLength = line.readableBytes();
639         while (lineLength > 0) {
640             final byte[] lineContent = line.array();
641             final int startLine = line.arrayOffset() + line.readerIndex();
642             final byte firstChar = lineContent[startLine];
643             if (name != null && (firstChar == ' ' || firstChar == '\t')) {
644                 //please do not make one line from below code
645                 //as it breaks +XX:OptimizeStringConcat optimization
646                 String trimmedLine = langAsciiString(lineContent, startLine, lineLength).trim();
647                 String valueStr = value;
648                 value = valueStr + ' ' + trimmedLine;
649             } else {
650                 if (name != null) {
651                     headers.add(name, value);
652                 }
653                 splitHeader(lineContent, startLine, lineLength);
654             }
655 
656             line = headerParser.parse(buffer);
657             if (line == null) {
658                 return null;
659             }
660             lineLength = line.readableBytes();
661         }
662 
663         // Add the last header.
664         if (name != null) {
665             headers.add(name, value);
666         }
667 
668         // reset name and value fields
669         name = null;
670         value = null;
671 
672         // Done parsing initial line and headers. Set decoder result.
673         HttpMessageDecoderResult decoderResult = new HttpMessageDecoderResult(lineParser.size, headerParser.size);
674         message.setDecoderResult(decoderResult);
675 
676         List<String> contentLengthFields = headers.getAll(HttpHeaderNames.CONTENT_LENGTH);
677         if (!contentLengthFields.isEmpty()) {
678             HttpVersion version = message.protocolVersion();
679             boolean isHttp10OrEarlier = version.majorVersion() < 1 || (version.majorVersion() == 1
680                     && version.minorVersion() == 0);
681             // Guard against multiple Content-Length headers as stated in
682             // https://tools.ietf.org/html/rfc7230#section-3.3.2:
683             contentLength = HttpUtil.normalizeAndGetContentLength(contentLengthFields,
684                     isHttp10OrEarlier, allowDuplicateContentLengths);
685             if (contentLength != -1) {
686                 String lengthValue = contentLengthFields.get(0).trim();
687                 if (contentLengthFields.size() > 1 || // don't unnecessarily re-order headers
688                         !lengthValue.equals(Long.toString(contentLength))) {
689                     headers.set(HttpHeaderNames.CONTENT_LENGTH, contentLength);
690                 }
691             }
692         }
693 
694         if (isContentAlwaysEmpty(message)) {
695             HttpUtil.setTransferEncodingChunked(message, false);
696             return State.SKIP_CONTROL_CHARS;
697         } else if (HttpUtil.isTransferEncodingChunked(message)) {
698             if (!contentLengthFields.isEmpty() && message.protocolVersion() == HttpVersion.HTTP_1_1) {
699                 handleTransferEncodingChunkedWithContentLength(message);
700             }
701             return State.READ_CHUNK_SIZE;
702         } else if (contentLength() >= 0) {
703             return State.READ_FIXED_LENGTH_CONTENT;
704         } else {
705             return State.READ_VARIABLE_LENGTH_CONTENT;
706         }
707     }
708 
709     /**
710      * Invoked when a message with both a "Transfer-Encoding: chunked" and a "Content-Length" header field is detected.
711      * The default behavior is to <i>remove</i> the Content-Length field, but this method could be overridden
712      * to change the behavior (to, e.g., throw an exception and produce an invalid message).
713      * <p>
714      * See: https://tools.ietf.org/html/rfc7230#section-3.3.3
715      * <pre>
716      *     If a message is received with both a Transfer-Encoding and a
717      *     Content-Length header field, the Transfer-Encoding overrides the
718      *     Content-Length.  Such a message might indicate an attempt to
719      *     perform request smuggling (Section 9.5) or response splitting
720      *     (Section 9.4) and ought to be handled as an error.  A sender MUST
721      *     remove the received Content-Length field prior to forwarding such
722      *     a message downstream.
723      * </pre>
724      * Also see:
725      * https://github.com/apache/tomcat/blob/b693d7c1981fa7f51e58bc8c8e72e3fe80b7b773/
726      * java/org/apache/coyote/http11/Http11Processor.java#L747-L755
727      * https://github.com/nginx/nginx/blob/0ad4393e30c119d250415cb769e3d8bc8dce5186/
728      * src/http/ngx_http_request.c#L1946-L1953
729      */
730     protected void handleTransferEncodingChunkedWithContentLength(HttpMessage message) {
731         message.headers().remove(HttpHeaderNames.CONTENT_LENGTH);
732         contentLength = Long.MIN_VALUE;
733     }
734 
735     private long contentLength() {
736         if (contentLength == Long.MIN_VALUE) {
737             contentLength = HttpUtil.getContentLength(message, -1L);
738         }
739         return contentLength;
740     }
741 
742     private LastHttpContent readTrailingHeaders(ByteBuf buffer) {
743         final HeaderParser headerParser = this.headerParser;
744         ByteBuf line = headerParser.parse(buffer);
745         if (line == null) {
746             return null;
747         }
748         LastHttpContent trailer = this.trailer;
749         int lineLength = line.readableBytes();
750         if (lineLength == 0 && trailer == null) {
751             // We have received the empty line which signals the trailer is complete and did not parse any trailers
752             // before. Just return an empty last content to reduce allocations.
753             return LastHttpContent.EMPTY_LAST_CONTENT;
754         }
755 
756         CharSequence lastHeader = null;
757         if (trailer == null) {
758             trailer = this.trailer = new DefaultLastHttpContent(Unpooled.EMPTY_BUFFER, validateHeaders);
759         }
760         while (lineLength > 0) {
761             final byte[] lineContent = line.array();
762             final int startLine = line.arrayOffset() + line.readerIndex();
763             final byte firstChar = lineContent[startLine];
764             if (lastHeader != null && (firstChar == ' ' || firstChar == '\t')) {
765                 List<String> current = trailer.trailingHeaders().getAll(lastHeader);
766                 if (!current.isEmpty()) {
767                     int lastPos = current.size() - 1;
768                     //please do not make one line from below code
769                     //as it breaks +XX:OptimizeStringConcat optimization
770                     String lineTrimmed = langAsciiString(lineContent, startLine, line.readableBytes()).trim();
771                     String currentLastPos = current.get(lastPos);
772                     current.set(lastPos, currentLastPos + lineTrimmed);
773                 }
774             } else {
775                 splitHeader(lineContent, startLine, lineLength);
776                 AsciiString headerName = name;
777                 if (!HttpHeaderNames.CONTENT_LENGTH.contentEqualsIgnoreCase(headerName) &&
778                         !HttpHeaderNames.TRANSFER_ENCODING.contentEqualsIgnoreCase(headerName) &&
779                         !HttpHeaderNames.TRAILER.contentEqualsIgnoreCase(headerName)) {
780                     trailer.trailingHeaders().add(headerName, value);
781                 }
782                 lastHeader = name;
783                 // reset name and value fields
784                 name = null;
785                 value = null;
786             }
787             line = headerParser.parse(buffer);
788             if (line == null) {
789                 return null;
790             }
791             lineLength = line.readableBytes();
792         }
793 
794         this.trailer = null;
795         return trailer;
796     }
797 
798     protected abstract boolean isDecodingRequest();
799     protected abstract HttpMessage createMessage(String[] initialLine) throws Exception;
800     protected abstract HttpMessage createInvalidMessage();
801 
802     /**
803      * It skips any whitespace char and return the number of skipped bytes.
804      */
805     private static int skipWhiteSpaces(byte[] hex, int start, int length) {
806         for (int i = 0; i < length; i++) {
807             if (!isWhitespace(hex[start + i])) {
808                 return i;
809             }
810         }
811         return length;
812     }
813 
814     private static int getChunkSize(byte[] hex, int start, int length) {
815         // trim the leading bytes if white spaces, if any
816         final int skipped = skipWhiteSpaces(hex, start, length);
817         if (skipped == length) {
818             // empty case
819             throw new NumberFormatException();
820         }
821         start += skipped;
822         length -= skipped;
823         int result = 0;
824         for (int i = 0; i < length; i++) {
825             final int digit = StringUtil.decodeHexNibble(hex[start + i]);
826             if (digit == -1) {
827                 // uncommon path
828                 final byte b = hex[start + i];
829                 if (b == ';' || isControlOrWhitespaceAsciiChar(b)) {
830                     if (i == 0) {
831                         // empty case
832                         throw new NumberFormatException();
833                     }
834                     return result;
835                 }
836                 // non-hex char fail-fast path
837                 throw new NumberFormatException();
838             }
839             result *= 16;
840             result += digit;
841         }
842         return result;
843     }
844 
845     private String[] splitInitialLine(ByteBuf asciiBuffer) {
846         final byte[] asciiBytes = asciiBuffer.array();
847 
848         final int arrayOffset = asciiBuffer.arrayOffset();
849 
850         final int startContent = arrayOffset + asciiBuffer.readerIndex();
851 
852         final int end = startContent + asciiBuffer.readableBytes();
853 
854         final int aStart = findNonSPLenient(asciiBytes, startContent, end);
855         final int aEnd = findSPLenient(asciiBytes, aStart, end);
856 
857         final int bStart = findNonSPLenient(asciiBytes, aEnd, end);
858         final int bEnd = findSPLenient(asciiBytes, bStart, end);
859 
860         final int cStart = findNonSPLenient(asciiBytes, bEnd, end);
861         final int cEnd = findEndOfString(asciiBytes, Math.max(cStart - 1, startContent), end);
862 
863         return new String[]{
864                 splitFirstWordInitialLine(asciiBytes, aStart, aEnd - aStart),
865                 splitSecondWordInitialLine(asciiBytes, bStart, bEnd - bStart),
866                 cStart < cEnd ? splitThirdWordInitialLine(asciiBytes, cStart, cEnd - cStart) : StringUtil.EMPTY_STRING};
867     }
868 
869     protected String splitFirstWordInitialLine(final byte[] asciiContent, int start, int length) {
870         return langAsciiString(asciiContent, start, length);
871     }
872 
873     protected String splitSecondWordInitialLine(final byte[] asciiContent, int start, int length) {
874         return langAsciiString(asciiContent, start, length);
875     }
876 
877     protected String splitThirdWordInitialLine(final byte[] asciiContent, int start, int length) {
878         return langAsciiString(asciiContent, start, length);
879     }
880 
881     /**
882      * This method shouldn't exist: look at https://bugs.openjdk.org/browse/JDK-8295496 for more context
883      */
884     private static String langAsciiString(final byte[] asciiContent, int start, int length) {
885         if (length == 0) {
886             return StringUtil.EMPTY_STRING;
887         }
888         // DON'T REMOVE: it helps JIT to use a simpler intrinsic stub for System::arrayCopy based on the call-site
889         if (start == 0) {
890             if (length == asciiContent.length) {
891                 return new String(asciiContent, 0, 0, asciiContent.length);
892             }
893             return new String(asciiContent, 0, 0, length);
894         }
895         return new String(asciiContent, 0, start, length);
896     }
897 
898     private void splitHeader(byte[] line, int start, int length) {
899         final int end = start + length;
900         int nameEnd;
901         final int nameStart = findNonWhitespace(line, start, end);
902         // hoist this load out of the loop, because it won't change!
903         final boolean isDecodingRequest = isDecodingRequest();
904         for (nameEnd = nameStart; nameEnd < end; nameEnd ++) {
905             byte ch = line[nameEnd];
906             // https://tools.ietf.org/html/rfc7230#section-3.2.4
907             //
908             // No whitespace is allowed between the header field-name and colon. In
909             // the past, differences in the handling of such whitespace have led to
910             // security vulnerabilities in request routing and response handling. A
911             // server MUST reject any received request message that contains
912             // whitespace between a header field-name and colon with a response code
913             // of 400 (Bad Request). A proxy MUST remove any such whitespace from a
914             // response message before forwarding the message downstream.
915             if (ch == ':' ||
916                     // In case of decoding a request we will just continue processing and header validation
917                     // is done in the DefaultHttpHeaders implementation.
918                     //
919                     // In the case of decoding a response we will "skip" the whitespace.
920                     (!isDecodingRequest && isOWS(ch))) {
921                 break;
922             }
923         }
924 
925         if (nameEnd == end) {
926             // There was no colon present at all.
927             throw new IllegalArgumentException("No colon found");
928         }
929         int colonEnd;
930         for (colonEnd = nameEnd; colonEnd < end; colonEnd ++) {
931             if (line[colonEnd] == ':') {
932                 colonEnd ++;
933                 break;
934             }
935         }
936         name = splitHeaderName(line, nameStart, nameEnd - nameStart);
937         final int valueStart = findNonWhitespace(line, colonEnd, end);
938         if (valueStart == end) {
939             value = StringUtil.EMPTY_STRING;
940         } else {
941             final int valueEnd = findEndOfString(line, start, end);
942             // no need to make uses of the ByteBuf's toString ASCII method here, and risk to get JIT confused
943             value = langAsciiString(line, valueStart, valueEnd - valueStart);
944         }
945     }
946 
947     protected AsciiString splitHeaderName(byte[] sb, int start, int length) {
948         return new AsciiString(sb, start, length, true);
949     }
950 
951     private static int findNonSPLenient(byte[] sb, int offset, int end) {
952         for (int result = offset; result < end; ++result) {
953             byte c = sb[result];
954             // See https://tools.ietf.org/html/rfc7230#section-3.5
955             if (isSPLenient(c)) {
956                 continue;
957             }
958             if (isWhitespace(c)) {
959                 // Any other whitespace delimiter is invalid
960                 throw new IllegalArgumentException("Invalid separator");
961             }
962             return result;
963         }
964         return end;
965     }
966 
967     private static int findSPLenient(byte[] sb, int offset, int end) {
968         for (int result = offset; result < end; ++result) {
969             if (isSPLenient(sb[result])) {
970                 return result;
971             }
972         }
973         return end;
974     }
975 
976     private static final boolean[] SP_LENIENT_BYTES;
977     private static final boolean[] LATIN_WHITESPACE;
978 
979     static {
980         // See https://tools.ietf.org/html/rfc7230#section-3.5
981         SP_LENIENT_BYTES = new boolean[256];
982         SP_LENIENT_BYTES[128 + ' '] = true;
983         SP_LENIENT_BYTES[128 + 0x09] = true;
984         SP_LENIENT_BYTES[128 + 0x0B] = true;
985         SP_LENIENT_BYTES[128 + 0x0C] = true;
986         SP_LENIENT_BYTES[128 + 0x0D] = true;
987         // TO SAVE PERFORMING Character::isWhitespace ceremony
988         LATIN_WHITESPACE = new boolean[256];
989         for (byte b = Byte.MIN_VALUE; b < Byte.MAX_VALUE; b++) {
990             LATIN_WHITESPACE[128 + b] = Character.isWhitespace(b);
991         }
992     }
993 
994     private static boolean isSPLenient(byte c) {
995         // See https://tools.ietf.org/html/rfc7230#section-3.5
996         return SP_LENIENT_BYTES[c + 128];
997     }
998 
999     private static boolean isWhitespace(byte b) {
1000         return LATIN_WHITESPACE[b + 128];
1001     }
1002 
1003     private static int findNonWhitespace(byte[] sb, int offset, int end) {
1004         for (int result = offset; result < end; ++result) {
1005             byte c = sb[result];
1006             if (!isWhitespace(c)) {
1007                 return result;
1008             } else if (!isOWS(c)) {
1009                 // Only OWS is supported for whitespace
1010                 throw new IllegalArgumentException("Invalid separator, only a single space or horizontal tab allowed," +
1011                         " but received a '" + c + "' (0x" + Integer.toHexString(c) + ")");
1012             }
1013         }
1014         return end;
1015     }
1016 
1017     private static int findEndOfString(byte[] sb, int start, int end) {
1018         for (int result = end - 1; result > start; --result) {
1019             if (!isWhitespace(sb[result])) {
1020                 return result + 1;
1021             }
1022         }
1023         return 0;
1024     }
1025 
1026     private static boolean isOWS(byte ch) {
1027         return ch == ' ' || ch == 0x09;
1028     }
1029 
1030     private static class HeaderParser {
1031         protected final ByteBuf seq;
1032         protected final int maxLength;
1033         int size;
1034 
1035         HeaderParser(ByteBuf seq, int maxLength) {
1036             this.seq = seq;
1037             this.maxLength = maxLength;
1038         }
1039 
1040         public ByteBuf parse(ByteBuf buffer) {
1041             final int readableBytes = buffer.readableBytes();
1042             final int readerIndex = buffer.readerIndex();
1043             final int maxBodySize = maxLength - size;
1044             assert maxBodySize >= 0;
1045             // adding 2 to account for both CR (if present) and LF
1046             // don't remove 2L: it's key to cover maxLength = Integer.MAX_VALUE
1047             final long maxBodySizeWithCRLF = maxBodySize + 2L;
1048             final int toProcess = (int) Math.min(maxBodySizeWithCRLF, readableBytes);
1049             final int toIndexExclusive = readerIndex + toProcess;
1050             assert toIndexExclusive >= readerIndex;
1051             final int indexOfLf = buffer.indexOf(readerIndex, toIndexExclusive, HttpConstants.LF);
1052             if (indexOfLf == -1) {
1053                 if (readableBytes > maxBodySize) {
1054                     // TODO: Respond with Bad Request and discard the traffic
1055                     //    or close the connection.
1056                     //       No need to notify the upstream handlers - just log.
1057                     //       If decoding a response, just throw an exception.
1058                     throw newException(maxLength);
1059                 }
1060                 return null;
1061             }
1062             final int endOfSeqIncluded;
1063             if (indexOfLf > readerIndex && buffer.getByte(indexOfLf - 1) == HttpConstants.CR) {
1064                 // Drop CR if we had a CRLF pair
1065                 endOfSeqIncluded = indexOfLf - 1;
1066             } else {
1067                 endOfSeqIncluded = indexOfLf;
1068             }
1069             final int newSize = endOfSeqIncluded - readerIndex;
1070             if (newSize == 0) {
1071                 seq.clear();
1072                 buffer.readerIndex(indexOfLf + 1);
1073                 return seq;
1074             }
1075             int size = this.size + newSize;
1076             if (size > maxLength) {
1077                 throw newException(maxLength);
1078             }
1079             this.size = size;
1080             seq.clear();
1081             seq.writeBytes(buffer, readerIndex, newSize);
1082             buffer.readerIndex(indexOfLf + 1);
1083             return seq;
1084         }
1085 
1086         public void reset() {
1087             size = 0;
1088         }
1089 
1090         protected TooLongFrameException newException(int maxLength) {
1091             return new TooLongHttpHeaderException("HTTP header is larger than " + maxLength + " bytes.");
1092         }
1093     }
1094 
1095     private final class LineParser extends HeaderParser {
1096 
1097         LineParser(ByteBuf seq, int maxLength) {
1098             super(seq, maxLength);
1099         }
1100 
1101         @Override
1102         public ByteBuf parse(ByteBuf buffer) {
1103             // Suppress a warning because HeaderParser.reset() is supposed to be called
1104             reset();
1105             final int readableBytes = buffer.readableBytes();
1106             if (readableBytes == 0) {
1107                 return null;
1108             }
1109             final int readerIndex = buffer.readerIndex();
1110             if (currentState == State.SKIP_CONTROL_CHARS && skipControlChars(buffer, readableBytes, readerIndex)) {
1111                 return null;
1112             }
1113             return super.parse(buffer);
1114         }
1115 
1116         private boolean skipControlChars(ByteBuf buffer, int readableBytes, int readerIndex) {
1117             assert currentState == State.SKIP_CONTROL_CHARS;
1118             final int maxToSkip = Math.min(maxLength, readableBytes);
1119             final int firstNonControlIndex = buffer.forEachByte(readerIndex, maxToSkip, SKIP_CONTROL_CHARS_BYTES);
1120             if (firstNonControlIndex == -1) {
1121                 buffer.skipBytes(maxToSkip);
1122                 if (readableBytes > maxLength) {
1123                     throw newException(maxLength);
1124                 }
1125                 return true;
1126             }
1127             // from now on we don't care about control chars
1128             buffer.readerIndex(firstNonControlIndex);
1129             currentState = State.READ_INITIAL;
1130             return false;
1131         }
1132 
1133         @Override
1134         protected TooLongFrameException newException(int maxLength) {
1135             return new TooLongHttpLineException("An HTTP line is larger than " + maxLength + " bytes.");
1136         }
1137     }
1138 
1139     private static final boolean[] ISO_CONTROL_OR_WHITESPACE;
1140 
1141     static {
1142         ISO_CONTROL_OR_WHITESPACE = new boolean[256];
1143         for (byte b = Byte.MIN_VALUE; b < Byte.MAX_VALUE; b++) {
1144             ISO_CONTROL_OR_WHITESPACE[128 + b] = Character.isISOControl(b) || isWhitespace(b);
1145         }
1146     }
1147 
1148     private static final ByteProcessor SKIP_CONTROL_CHARS_BYTES = new ByteProcessor() {
1149 
1150         @Override
1151         public boolean process(byte value) {
1152             return ISO_CONTROL_OR_WHITESPACE[128 + value];
1153         }
1154     };
1155 
1156     private static boolean isControlOrWhitespaceAsciiChar(byte b) {
1157         return ISO_CONTROL_OR_WHITESPACE[128 + b];
1158     }
1159 }