View Javadoc
1   /*
2    * Copyright 2012 The Netty Project
3    *
4    * The Netty Project licenses this file to you under the Apache License,
5    * version 2.0 (the "License"); you may not use this file except in compliance
6    * with the License. You may obtain a copy of the License at:
7    *
8    *   https://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13   * License for the specific language governing permissions and limitations
14   * under the License.
15   */
16  package io.netty.handler.codec.http;
17  
18  import static io.netty.util.internal.ObjectUtil.checkPositive;
19  
20  import io.netty.buffer.ByteBuf;
21  import io.netty.buffer.Unpooled;
22  import io.netty.channel.ChannelHandlerContext;
23  import io.netty.channel.ChannelPipeline;
24  import io.netty.handler.codec.ByteToMessageDecoder;
25  import io.netty.handler.codec.DecoderResult;
26  import io.netty.handler.codec.PrematureChannelClosureException;
27  import io.netty.handler.codec.TooLongFrameException;
28  import io.netty.util.ByteProcessor;
29  import io.netty.util.internal.AppendableCharSequence;
30  
31  import java.util.List;
32  
33  /**
34   * Decodes {@link ByteBuf}s into {@link HttpMessage}s and
35   * {@link HttpContent}s.
36   *
37   * <h3>Parameters that prevents excessive memory consumption</h3>
38   * <table border="1">
39   * <tr>
40   * <th>Name</th><th>Default value</th><th>Meaning</th>
41   * </tr>
42   * <tr>
43   * <td>{@code maxInitialLineLength}</td>
44   * <td>{@value #DEFAULT_MAX_INITIAL_LINE_LENGTH}</td>
45   * <td>The maximum length of the initial line
46   *     (e.g. {@code "GET / HTTP/1.0"} or {@code "HTTP/1.0 200 OK"})
47   *     If the length of the initial line exceeds this value, a
48   *     {@link TooLongHttpLineException} will be raised.</td>
49   * </tr>
50   * <tr>
51   * <td>{@code maxHeaderSize}</td>
52   * <td>{@value #DEFAULT_MAX_HEADER_SIZE}</td>
53   * <td>The maximum length of all headers.  If the sum of the length of each
54   *     header exceeds this value, a {@link TooLongHttpHeaderException} will be raised.</td>
55   * </tr>
56   * <tr>
57   * <td>{@code maxChunkSize}</td>
58   * <td>{@value #DEFAULT_MAX_CHUNK_SIZE}</td>
59   * <td>The maximum length of the content or each chunk.  If the content length
60   *     (or the length of each chunk) exceeds this value, the content or chunk
61   *     will be split into multiple {@link HttpContent}s whose length is
62   *     {@code maxChunkSize} at maximum.</td>
63   * </tr>
64   * </table>
65   *
66   * <h3>Parameters that control parsing behavior</h3>
67   * <table border="1">
68   * <tr>
69   * <th>Name</th><th>Default value</th><th>Meaning</th>
70   * </tr>
71   * <tr>
72   * <td>{@code allowDuplicateContentLengths}</td>
73   * <td>{@value #DEFAULT_ALLOW_DUPLICATE_CONTENT_LENGTHS}</td>
74   * <td>When set to {@code false}, will reject any messages that contain multiple Content-Length header fields.
75   *     When set to {@code true}, will allow multiple Content-Length headers only if they are all the same decimal value.
76   *     The duplicated field-values will be replaced with a single valid Content-Length field.
77   *     See <a href="https://tools.ietf.org/html/rfc7230#section-3.3.2">RFC 7230, Section 3.3.2</a>.</td>
78   * </tr>
79   * <tr>
80   * <td>{@code allowPartialChunks}</td>
81   * <td>{@value #DEFAULT_ALLOW_PARTIAL_CHUNKS}</td>
82   * <td>If the length of a chunk exceeds the {@link ByteBuf}s readable bytes and {@code allowPartialChunks}
83   *     is set to {@code true}, the chunk will be split into multiple {@link HttpContent}s.
84   *     Otherwise, if the chunk size does not exceed {@code maxChunkSize} and {@code allowPartialChunks}
85   *     is set to {@code false}, the {@link ByteBuf} is not decoded into an {@link HttpContent} until
86   *     the readable bytes are greater or equal to the chunk size.</td>
87   * </tr>
88   * </table>
89   *
90   * <h3>Chunked Content</h3>
91   *
92   * If the content of an HTTP message is greater than {@code maxChunkSize} or
93   * the transfer encoding of the HTTP message is 'chunked', this decoder
94   * generates one {@link HttpMessage} instance and its following
95   * {@link HttpContent}s per single HTTP message to avoid excessive memory
96   * consumption. For example, the following HTTP message:
97   * <pre>
98   * GET / HTTP/1.1
99   * Transfer-Encoding: chunked
100  *
101  * 1a
102  * abcdefghijklmnopqrstuvwxyz
103  * 10
104  * 1234567890abcdef
105  * 0
106  * Content-MD5: ...
107  * <i>[blank line]</i>
108  * </pre>
109  * triggers {@link HttpRequestDecoder} to generate 3 objects:
110  * <ol>
111  * <li>An {@link HttpRequest},</li>
112  * <li>The first {@link HttpContent} whose content is {@code 'abcdefghijklmnopqrstuvwxyz'},</li>
113  * <li>The second {@link LastHttpContent} whose content is {@code '1234567890abcdef'}, which marks
114  * the end of the content.</li>
115  * </ol>
116  *
117  * If you prefer not to handle {@link HttpContent}s by yourself for your
118  * convenience, insert {@link HttpObjectAggregator} after this decoder in the
119  * {@link ChannelPipeline}.  However, please note that your server might not
120  * be as memory efficient as without the aggregator.
121  *
122  * <h3>Extensibility</h3>
123  *
124  * Please note that this decoder is designed to be extended to implement
125  * a protocol derived from HTTP, such as
126  * <a href="https://en.wikipedia.org/wiki/Real_Time_Streaming_Protocol">RTSP</a> and
127  * <a href="https://en.wikipedia.org/wiki/Internet_Content_Adaptation_Protocol">ICAP</a>.
128  * To implement the decoder of such a derived protocol, extend this class and
129  * implement all abstract methods properly.
130  */
131 public abstract class HttpObjectDecoder extends ByteToMessageDecoder {
132     public static final int DEFAULT_MAX_INITIAL_LINE_LENGTH = 4096;
133     public static final int DEFAULT_MAX_HEADER_SIZE = 8192;
134     public static final boolean DEFAULT_CHUNKED_SUPPORTED = true;
135     public static final boolean DEFAULT_ALLOW_PARTIAL_CHUNKS = true;
136     public static final int DEFAULT_MAX_CHUNK_SIZE = 8192;
137     public static final boolean DEFAULT_VALIDATE_HEADERS = true;
138     public static final int DEFAULT_INITIAL_BUFFER_SIZE = 128;
139     public static final boolean DEFAULT_ALLOW_DUPLICATE_CONTENT_LENGTHS = false;
140 
141     private static final String EMPTY_VALUE = "";
142 
143     private final int maxChunkSize;
144     private final boolean chunkedSupported;
145     private final boolean allowPartialChunks;
146     protected final boolean validateHeaders;
147     private final boolean allowDuplicateContentLengths;
148     private final HeaderParser headerParser;
149     private final LineParser lineParser;
150 
151     private HttpMessage message;
152     private long chunkSize;
153     private long contentLength = Long.MIN_VALUE;
154     private volatile boolean resetRequested;
155 
156     // These will be updated by splitHeader(...)
157     private CharSequence name;
158     private CharSequence value;
159 
160     private LastHttpContent trailer;
161 
162     /**
163      * The internal state of {@link HttpObjectDecoder}.
164      * <em>Internal use only</em>.
165      */
166     private enum State {
167         SKIP_CONTROL_CHARS,
168         READ_INITIAL,
169         READ_HEADER,
170         READ_VARIABLE_LENGTH_CONTENT,
171         READ_FIXED_LENGTH_CONTENT,
172         READ_CHUNK_SIZE,
173         READ_CHUNKED_CONTENT,
174         READ_CHUNK_DELIMITER,
175         READ_CHUNK_FOOTER,
176         BAD_MESSAGE,
177         UPGRADED
178     }
179 
180     private State currentState = State.SKIP_CONTROL_CHARS;
181 
182     /**
183      * Creates a new instance with the default
184      * {@code maxInitialLineLength (4096}}, {@code maxHeaderSize (8192)}, and
185      * {@code maxChunkSize (8192)}.
186      */
187     protected HttpObjectDecoder() {
188         this(DEFAULT_MAX_INITIAL_LINE_LENGTH, DEFAULT_MAX_HEADER_SIZE, DEFAULT_MAX_CHUNK_SIZE,
189              DEFAULT_CHUNKED_SUPPORTED);
190     }
191 
192     /**
193      * Creates a new instance with the specified parameters.
194      */
195     protected HttpObjectDecoder(
196             int maxInitialLineLength, int maxHeaderSize, int maxChunkSize, boolean chunkedSupported) {
197         this(maxInitialLineLength, maxHeaderSize, maxChunkSize, chunkedSupported, DEFAULT_VALIDATE_HEADERS);
198     }
199 
200     /**
201      * Creates a new instance with the specified parameters.
202      */
203     protected HttpObjectDecoder(
204             int maxInitialLineLength, int maxHeaderSize, int maxChunkSize,
205             boolean chunkedSupported, boolean validateHeaders) {
206         this(maxInitialLineLength, maxHeaderSize, maxChunkSize, chunkedSupported, validateHeaders,
207              DEFAULT_INITIAL_BUFFER_SIZE);
208     }
209 
210     /**
211      * Creates a new instance with the specified parameters.
212      */
213     protected HttpObjectDecoder(
214             int maxInitialLineLength, int maxHeaderSize, int maxChunkSize,
215             boolean chunkedSupported, boolean validateHeaders, int initialBufferSize) {
216         this(maxInitialLineLength, maxHeaderSize, maxChunkSize, chunkedSupported, validateHeaders, initialBufferSize,
217              DEFAULT_ALLOW_DUPLICATE_CONTENT_LENGTHS);
218     }
219 
220     /**
221      * Creates a new instance with the specified parameters.
222      */
223     protected HttpObjectDecoder(
224             int maxInitialLineLength, int maxHeaderSize, int maxChunkSize,
225             boolean chunkedSupported, boolean validateHeaders, int initialBufferSize,
226             boolean allowDuplicateContentLengths) {
227         this(maxInitialLineLength, maxHeaderSize, maxChunkSize, chunkedSupported, validateHeaders, initialBufferSize,
228             allowDuplicateContentLengths, DEFAULT_ALLOW_PARTIAL_CHUNKS);
229     }
230 
231     /**
232      * Creates a new instance with the specified parameters.
233      */
234     protected HttpObjectDecoder(
235             int maxInitialLineLength, int maxHeaderSize, int maxChunkSize,
236             boolean chunkedSupported, boolean validateHeaders, int initialBufferSize,
237             boolean allowDuplicateContentLengths, boolean allowPartialChunks) {
238         checkPositive(maxInitialLineLength, "maxInitialLineLength");
239         checkPositive(maxHeaderSize, "maxHeaderSize");
240         checkPositive(maxChunkSize, "maxChunkSize");
241 
242         AppendableCharSequence seq = new AppendableCharSequence(initialBufferSize);
243         lineParser = new LineParser(seq, maxInitialLineLength);
244         headerParser = new HeaderParser(seq, maxHeaderSize);
245         this.maxChunkSize = maxChunkSize;
246         this.chunkedSupported = chunkedSupported;
247         this.validateHeaders = validateHeaders;
248         this.allowDuplicateContentLengths = allowDuplicateContentLengths;
249         this.allowPartialChunks = allowPartialChunks;
250     }
251 
252     @Override
253     protected void decode(ChannelHandlerContext ctx, ByteBuf buffer, List<Object> out) throws Exception {
254         if (resetRequested) {
255             resetNow();
256         }
257 
258         switch (currentState) {
259         case SKIP_CONTROL_CHARS:
260             // Fall-through
261         case READ_INITIAL: try {
262             AppendableCharSequence line = lineParser.parse(buffer);
263             if (line == null) {
264                 return;
265             }
266             String[] initialLine = splitInitialLine(line);
267             if (initialLine.length < 3) {
268                 // Invalid initial line - ignore.
269                 currentState = State.SKIP_CONTROL_CHARS;
270                 return;
271             }
272 
273             message = createMessage(initialLine);
274             currentState = State.READ_HEADER;
275             // fall-through
276         } catch (Exception e) {
277             out.add(invalidMessage(buffer, e));
278             return;
279         }
280         case READ_HEADER: try {
281             State nextState = readHeaders(buffer);
282             if (nextState == null) {
283                 return;
284             }
285             currentState = nextState;
286             switch (nextState) {
287             case SKIP_CONTROL_CHARS:
288                 // fast-path
289                 // No content is expected.
290                 out.add(message);
291                 out.add(LastHttpContent.EMPTY_LAST_CONTENT);
292                 resetNow();
293                 return;
294             case READ_CHUNK_SIZE:
295                 if (!chunkedSupported) {
296                     throw new IllegalArgumentException("Chunked messages not supported");
297                 }
298                 // Chunked encoding - generate HttpMessage first.  HttpChunks will follow.
299                 out.add(message);
300                 return;
301             default:
302                 /**
303                  * <a href="https://tools.ietf.org/html/rfc7230#section-3.3.3">RFC 7230, 3.3.3</a> states that if a
304                  * request does not have either a transfer-encoding or a content-length header then the message body
305                  * length is 0. However for a response the body length is the number of octets received prior to the
306                  * server closing the connection. So we treat this as variable length chunked encoding.
307                  */
308                 long contentLength = contentLength();
309                 if (contentLength == 0 || contentLength == -1 && isDecodingRequest()) {
310                     out.add(message);
311                     out.add(LastHttpContent.EMPTY_LAST_CONTENT);
312                     resetNow();
313                     return;
314                 }
315 
316                 assert nextState == State.READ_FIXED_LENGTH_CONTENT ||
317                         nextState == State.READ_VARIABLE_LENGTH_CONTENT;
318 
319                 out.add(message);
320 
321                 if (nextState == State.READ_FIXED_LENGTH_CONTENT) {
322                     // chunkSize will be decreased as the READ_FIXED_LENGTH_CONTENT state reads data chunk by chunk.
323                     chunkSize = contentLength;
324                 }
325 
326                 // We return here, this forces decode to be called again where we will decode the content
327                 return;
328             }
329         } catch (Exception e) {
330             out.add(invalidMessage(buffer, e));
331             return;
332         }
333         case READ_VARIABLE_LENGTH_CONTENT: {
334             // Keep reading data as a chunk until the end of connection is reached.
335             int toRead = Math.min(buffer.readableBytes(), maxChunkSize);
336             if (toRead > 0) {
337                 ByteBuf content = buffer.readRetainedSlice(toRead);
338                 out.add(new DefaultHttpContent(content));
339             }
340             return;
341         }
342         case READ_FIXED_LENGTH_CONTENT: {
343             int readLimit = buffer.readableBytes();
344 
345             // Check if the buffer is readable first as we use the readable byte count
346             // to create the HttpChunk. This is needed as otherwise we may end up with
347             // create an HttpChunk instance that contains an empty buffer and so is
348             // handled like it is the last HttpChunk.
349             //
350             // See https://github.com/netty/netty/issues/433
351             if (readLimit == 0) {
352                 return;
353             }
354 
355             int toRead = Math.min(readLimit, maxChunkSize);
356             if (toRead > chunkSize) {
357                 toRead = (int) chunkSize;
358             }
359             ByteBuf content = buffer.readRetainedSlice(toRead);
360             chunkSize -= toRead;
361 
362             if (chunkSize == 0) {
363                 // Read all content.
364                 out.add(new DefaultLastHttpContent(content, validateHeaders));
365                 resetNow();
366             } else {
367                 out.add(new DefaultHttpContent(content));
368             }
369             return;
370         }
371         /**
372          * everything else after this point takes care of reading chunked content. basically, read chunk size,
373          * read chunk, read and ignore the CRLF and repeat until 0
374          */
375         case READ_CHUNK_SIZE: try {
376             AppendableCharSequence line = lineParser.parse(buffer);
377             if (line == null) {
378                 return;
379             }
380             int chunkSize = getChunkSize(line.toString());
381             this.chunkSize = chunkSize;
382             if (chunkSize == 0) {
383                 currentState = State.READ_CHUNK_FOOTER;
384                 return;
385             }
386             currentState = State.READ_CHUNKED_CONTENT;
387             // fall-through
388         } catch (Exception e) {
389             out.add(invalidChunk(buffer, e));
390             return;
391         }
392         case READ_CHUNKED_CONTENT: {
393             assert chunkSize <= Integer.MAX_VALUE;
394             int toRead = Math.min((int) chunkSize, maxChunkSize);
395             if (!allowPartialChunks && buffer.readableBytes() < toRead) {
396                 return;
397             }
398             toRead = Math.min(toRead, buffer.readableBytes());
399             if (toRead == 0) {
400                 return;
401             }
402             HttpContent chunk = new DefaultHttpContent(buffer.readRetainedSlice(toRead));
403             chunkSize -= toRead;
404 
405             out.add(chunk);
406 
407             if (chunkSize != 0) {
408                 return;
409             }
410             currentState = State.READ_CHUNK_DELIMITER;
411             // fall-through
412         }
413         case READ_CHUNK_DELIMITER: {
414             final int wIdx = buffer.writerIndex();
415             int rIdx = buffer.readerIndex();
416             while (wIdx > rIdx) {
417                 byte next = buffer.getByte(rIdx++);
418                 if (next == HttpConstants.LF) {
419                     currentState = State.READ_CHUNK_SIZE;
420                     break;
421                 }
422             }
423             buffer.readerIndex(rIdx);
424             return;
425         }
426         case READ_CHUNK_FOOTER: try {
427             LastHttpContent trailer = readTrailingHeaders(buffer);
428             if (trailer == null) {
429                 return;
430             }
431             out.add(trailer);
432             resetNow();
433             return;
434         } catch (Exception e) {
435             out.add(invalidChunk(buffer, e));
436             return;
437         }
438         case BAD_MESSAGE: {
439             // Keep discarding until disconnection.
440             buffer.skipBytes(buffer.readableBytes());
441             break;
442         }
443         case UPGRADED: {
444             int readableBytes = buffer.readableBytes();
445             if (readableBytes > 0) {
446                 // Keep on consuming as otherwise we may trigger an DecoderException,
447                 // other handler will replace this codec with the upgraded protocol codec to
448                 // take the traffic over at some point then.
449                 // See https://github.com/netty/netty/issues/2173
450                 out.add(buffer.readBytes(readableBytes));
451             }
452             break;
453         }
454         default:
455             break;
456         }
457     }
458 
459     @Override
460     protected void decodeLast(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception {
461         super.decodeLast(ctx, in, out);
462 
463         if (resetRequested) {
464             // If a reset was requested by decodeLast() we need to do it now otherwise we may produce a
465             // LastHttpContent while there was already one.
466             resetNow();
467         }
468         // Handle the last unfinished message.
469         if (message != null) {
470             boolean chunked = HttpUtil.isTransferEncodingChunked(message);
471             if (currentState == State.READ_VARIABLE_LENGTH_CONTENT && !in.isReadable() && !chunked) {
472                 // End of connection.
473                 out.add(LastHttpContent.EMPTY_LAST_CONTENT);
474                 resetNow();
475                 return;
476             }
477 
478             if (currentState == State.READ_HEADER) {
479                 // If we are still in the state of reading headers we need to create a new invalid message that
480                 // signals that the connection was closed before we received the headers.
481                 out.add(invalidMessage(Unpooled.EMPTY_BUFFER,
482                         new PrematureChannelClosureException("Connection closed before received headers")));
483                 resetNow();
484                 return;
485             }
486 
487             // Check if the closure of the connection signifies the end of the content.
488             boolean prematureClosure;
489             if (isDecodingRequest() || chunked) {
490                 // The last request did not wait for a response.
491                 prematureClosure = true;
492             } else {
493                 // Compare the length of the received content and the 'Content-Length' header.
494                 // If the 'Content-Length' header is absent, the length of the content is determined by the end of the
495                 // connection, so it is perfectly fine.
496                 prematureClosure = contentLength() > 0;
497             }
498 
499             if (!prematureClosure) {
500                 out.add(LastHttpContent.EMPTY_LAST_CONTENT);
501             }
502             resetNow();
503         }
504     }
505 
506     @Override
507     public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {
508         if (evt instanceof HttpExpectationFailedEvent) {
509             switch (currentState) {
510             case READ_FIXED_LENGTH_CONTENT:
511             case READ_VARIABLE_LENGTH_CONTENT:
512             case READ_CHUNK_SIZE:
513                 reset();
514                 break;
515             default:
516                 break;
517             }
518         }
519         super.userEventTriggered(ctx, evt);
520     }
521 
522     protected boolean isContentAlwaysEmpty(HttpMessage msg) {
523         if (msg instanceof HttpResponse) {
524             HttpResponse res = (HttpResponse) msg;
525             int code = res.status().code();
526 
527             // Correctly handle return codes of 1xx.
528             //
529             // See:
530             //     - https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html Section 4.4
531             //     - https://github.com/netty/netty/issues/222
532             if (code >= 100 && code < 200) {
533                 // One exception: Hixie 76 websocket handshake response
534                 return !(code == 101 && !res.headers().contains(HttpHeaderNames.SEC_WEBSOCKET_ACCEPT)
535                          && res.headers().contains(HttpHeaderNames.UPGRADE, HttpHeaderValues.WEBSOCKET, true));
536             }
537 
538             switch (code) {
539             case 204: case 304:
540                 return true;
541             default:
542                 return false;
543             }
544         }
545         return false;
546     }
547 
548     /**
549      * Returns true if the server switched to a different protocol than HTTP/1.0 or HTTP/1.1, e.g. HTTP/2 or Websocket.
550      * Returns false if the upgrade happened in a different layer, e.g. upgrade from HTTP/1.1 to HTTP/1.1 over TLS.
551      */
552     protected boolean isSwitchingToNonHttp1Protocol(HttpResponse msg) {
553         if (msg.status().code() != HttpResponseStatus.SWITCHING_PROTOCOLS.code()) {
554             return false;
555         }
556         String newProtocol = msg.headers().get(HttpHeaderNames.UPGRADE);
557         return newProtocol == null ||
558                 !newProtocol.contains(HttpVersion.HTTP_1_0.text()) &&
559                 !newProtocol.contains(HttpVersion.HTTP_1_1.text());
560     }
561 
562     /**
563      * Resets the state of the decoder so that it is ready to decode a new message.
564      * This method is useful for handling a rejected request with {@code Expect: 100-continue} header.
565      */
566     public void reset() {
567         resetRequested = true;
568     }
569 
570     private void resetNow() {
571         HttpMessage message = this.message;
572         this.message = null;
573         name = null;
574         value = null;
575         contentLength = Long.MIN_VALUE;
576         lineParser.reset();
577         headerParser.reset();
578         trailer = null;
579         if (!isDecodingRequest()) {
580             HttpResponse res = (HttpResponse) message;
581             if (res != null && isSwitchingToNonHttp1Protocol(res)) {
582                 currentState = State.UPGRADED;
583                 return;
584             }
585         }
586 
587         resetRequested = false;
588         currentState = State.SKIP_CONTROL_CHARS;
589     }
590 
591     private HttpMessage invalidMessage(ByteBuf in, Exception cause) {
592         currentState = State.BAD_MESSAGE;
593 
594         // Advance the readerIndex so that ByteToMessageDecoder does not complain
595         // when we produced an invalid message without consuming anything.
596         in.skipBytes(in.readableBytes());
597 
598         if (message == null) {
599             message = createInvalidMessage();
600         }
601         message.setDecoderResult(DecoderResult.failure(cause));
602 
603         HttpMessage ret = message;
604         message = null;
605         return ret;
606     }
607 
608     private HttpContent invalidChunk(ByteBuf in, Exception cause) {
609         currentState = State.BAD_MESSAGE;
610 
611         // Advance the readerIndex so that ByteToMessageDecoder does not complain
612         // when we produced an invalid message without consuming anything.
613         in.skipBytes(in.readableBytes());
614 
615         HttpContent chunk = new DefaultLastHttpContent(Unpooled.EMPTY_BUFFER);
616         chunk.setDecoderResult(DecoderResult.failure(cause));
617         message = null;
618         trailer = null;
619         return chunk;
620     }
621 
622     private State readHeaders(ByteBuf buffer) {
623         final HttpMessage message = this.message;
624         final HttpHeaders headers = message.headers();
625 
626         AppendableCharSequence line = headerParser.parse(buffer);
627         if (line == null) {
628             return null;
629         }
630         if (line.length() > 0) {
631             do {
632                 char firstChar = line.charAtUnsafe(0);
633                 if (name != null && (firstChar == ' ' || firstChar == '\t')) {
634                     //please do not make one line from below code
635                     //as it breaks +XX:OptimizeStringConcat optimization
636                     String trimmedLine = line.toString().trim();
637                     String valueStr = String.valueOf(value);
638                     value = valueStr + ' ' + trimmedLine;
639                 } else {
640                     if (name != null) {
641                         headers.add(name, value);
642                     }
643                     splitHeader(line);
644                 }
645 
646                 line = headerParser.parse(buffer);
647                 if (line == null) {
648                     return null;
649                 }
650             } while (line.length() > 0);
651         }
652 
653         // Add the last header.
654         if (name != null) {
655             headers.add(name, value);
656         }
657 
658         // reset name and value fields
659         name = null;
660         value = null;
661 
662         // Done parsing initial line and headers. Set decoder result.
663         HttpMessageDecoderResult decoderResult = new HttpMessageDecoderResult(lineParser.size, headerParser.size);
664         message.setDecoderResult(decoderResult);
665 
666         List<String> contentLengthFields = headers.getAll(HttpHeaderNames.CONTENT_LENGTH);
667         if (!contentLengthFields.isEmpty()) {
668             HttpVersion version = message.protocolVersion();
669             boolean isHttp10OrEarlier = version.majorVersion() < 1 || (version.majorVersion() == 1
670                     && version.minorVersion() == 0);
671             // Guard against multiple Content-Length headers as stated in
672             // https://tools.ietf.org/html/rfc7230#section-3.3.2:
673             contentLength = HttpUtil.normalizeAndGetContentLength(contentLengthFields,
674                     isHttp10OrEarlier, allowDuplicateContentLengths);
675             if (contentLength != -1) {
676                 headers.set(HttpHeaderNames.CONTENT_LENGTH, contentLength);
677             }
678         }
679 
680         if (isContentAlwaysEmpty(message)) {
681             HttpUtil.setTransferEncodingChunked(message, false);
682             return State.SKIP_CONTROL_CHARS;
683         } else if (HttpUtil.isTransferEncodingChunked(message)) {
684             if (!contentLengthFields.isEmpty() && message.protocolVersion() == HttpVersion.HTTP_1_1) {
685                 handleTransferEncodingChunkedWithContentLength(message);
686             }
687             return State.READ_CHUNK_SIZE;
688         } else if (contentLength() >= 0) {
689             return State.READ_FIXED_LENGTH_CONTENT;
690         } else {
691             return State.READ_VARIABLE_LENGTH_CONTENT;
692         }
693     }
694 
695     /**
696      * Invoked when a message with both a "Transfer-Encoding: chunked" and a "Content-Length" header field is detected.
697      * The default behavior is to <i>remove</i> the Content-Length field, but this method could be overridden
698      * to change the behavior (to, e.g., throw an exception and produce an invalid message).
699      * <p>
700      * See: https://tools.ietf.org/html/rfc7230#section-3.3.3
701      * <pre>
702      *     If a message is received with both a Transfer-Encoding and a
703      *     Content-Length header field, the Transfer-Encoding overrides the
704      *     Content-Length.  Such a message might indicate an attempt to
705      *     perform request smuggling (Section 9.5) or response splitting
706      *     (Section 9.4) and ought to be handled as an error.  A sender MUST
707      *     remove the received Content-Length field prior to forwarding such
708      *     a message downstream.
709      * </pre>
710      * Also see:
711      * https://github.com/apache/tomcat/blob/b693d7c1981fa7f51e58bc8c8e72e3fe80b7b773/
712      * java/org/apache/coyote/http11/Http11Processor.java#L747-L755
713      * https://github.com/nginx/nginx/blob/0ad4393e30c119d250415cb769e3d8bc8dce5186/
714      * src/http/ngx_http_request.c#L1946-L1953
715      */
716     protected void handleTransferEncodingChunkedWithContentLength(HttpMessage message) {
717         message.headers().remove(HttpHeaderNames.CONTENT_LENGTH);
718         contentLength = Long.MIN_VALUE;
719     }
720 
721     private long contentLength() {
722         if (contentLength == Long.MIN_VALUE) {
723             contentLength = HttpUtil.getContentLength(message, -1L);
724         }
725         return contentLength;
726     }
727 
728     private LastHttpContent readTrailingHeaders(ByteBuf buffer) {
729         AppendableCharSequence line = headerParser.parse(buffer);
730         if (line == null) {
731             return null;
732         }
733         LastHttpContent trailer = this.trailer;
734         if (line.length() == 0 && trailer == null) {
735             // We have received the empty line which signals the trailer is complete and did not parse any trailers
736             // before. Just return an empty last content to reduce allocations.
737             return LastHttpContent.EMPTY_LAST_CONTENT;
738         }
739 
740         CharSequence lastHeader = null;
741         if (trailer == null) {
742             trailer = this.trailer = new DefaultLastHttpContent(Unpooled.EMPTY_BUFFER, validateHeaders);
743         }
744         while (line.length() > 0) {
745             char firstChar = line.charAtUnsafe(0);
746             if (lastHeader != null && (firstChar == ' ' || firstChar == '\t')) {
747                 List<String> current = trailer.trailingHeaders().getAll(lastHeader);
748                 if (!current.isEmpty()) {
749                     int lastPos = current.size() - 1;
750                     //please do not make one line from below code
751                     //as it breaks +XX:OptimizeStringConcat optimization
752                     String lineTrimmed = line.toString().trim();
753                     String currentLastPos = current.get(lastPos);
754                     current.set(lastPos, currentLastPos + lineTrimmed);
755                 }
756             } else {
757                 splitHeader(line);
758                 CharSequence headerName = name;
759                 if (!HttpHeaderNames.CONTENT_LENGTH.contentEqualsIgnoreCase(headerName) &&
760                         !HttpHeaderNames.TRANSFER_ENCODING.contentEqualsIgnoreCase(headerName) &&
761                         !HttpHeaderNames.TRAILER.contentEqualsIgnoreCase(headerName)) {
762                     trailer.trailingHeaders().add(headerName, value);
763                 }
764                 lastHeader = name;
765                 // reset name and value fields
766                 name = null;
767                 value = null;
768             }
769             line = headerParser.parse(buffer);
770             if (line == null) {
771                 return null;
772             }
773         }
774 
775         this.trailer = null;
776         return trailer;
777     }
778 
779     protected abstract boolean isDecodingRequest();
780     protected abstract HttpMessage createMessage(String[] initialLine) throws Exception;
781     protected abstract HttpMessage createInvalidMessage();
782 
783     private static int getChunkSize(String hex) {
784         hex = hex.trim();
785         for (int i = 0; i < hex.length(); i ++) {
786             char c = hex.charAt(i);
787             if (c == ';' || Character.isWhitespace(c) || Character.isISOControl(c)) {
788                 hex = hex.substring(0, i);
789                 break;
790             }
791         }
792 
793         return Integer.parseInt(hex, 16);
794     }
795 
796     private static String[] splitInitialLine(AppendableCharSequence sb) {
797         int aStart;
798         int aEnd;
799         int bStart;
800         int bEnd;
801         int cStart;
802         int cEnd;
803 
804         aStart = findNonSPLenient(sb, 0);
805         aEnd = findSPLenient(sb, aStart);
806 
807         bStart = findNonSPLenient(sb, aEnd);
808         bEnd = findSPLenient(sb, bStart);
809 
810         cStart = findNonSPLenient(sb, bEnd);
811         cEnd = findEndOfString(sb);
812 
813         return new String[] {
814                 sb.subStringUnsafe(aStart, aEnd),
815                 sb.subStringUnsafe(bStart, bEnd),
816                 cStart < cEnd? sb.subStringUnsafe(cStart, cEnd) : "" };
817     }
818 
819     private void splitHeader(AppendableCharSequence sb) {
820         final int length = sb.length();
821         int nameStart;
822         int nameEnd;
823         int colonEnd;
824         int valueStart;
825         int valueEnd;
826 
827         nameStart = findNonWhitespace(sb, 0);
828         for (nameEnd = nameStart; nameEnd < length; nameEnd ++) {
829             char ch = sb.charAtUnsafe(nameEnd);
830             // https://tools.ietf.org/html/rfc7230#section-3.2.4
831             //
832             // No whitespace is allowed between the header field-name and colon. In
833             // the past, differences in the handling of such whitespace have led to
834             // security vulnerabilities in request routing and response handling. A
835             // server MUST reject any received request message that contains
836             // whitespace between a header field-name and colon with a response code
837             // of 400 (Bad Request). A proxy MUST remove any such whitespace from a
838             // response message before forwarding the message downstream.
839             if (ch == ':' ||
840                     // In case of decoding a request we will just continue processing and header validation
841                     // is done in the DefaultHttpHeaders implementation.
842                     //
843                     // In the case of decoding a response we will "skip" the whitespace.
844                     (!isDecodingRequest() && isOWS(ch))) {
845                 break;
846             }
847         }
848 
849         if (nameEnd == length) {
850             // There was no colon present at all.
851             throw new IllegalArgumentException("No colon found");
852         }
853 
854         for (colonEnd = nameEnd; colonEnd < length; colonEnd ++) {
855             if (sb.charAtUnsafe(colonEnd) == ':') {
856                 colonEnd ++;
857                 break;
858             }
859         }
860 
861         name = sb.subStringUnsafe(nameStart, nameEnd);
862         valueStart = findNonWhitespace(sb, colonEnd);
863         if (valueStart == length) {
864             value = EMPTY_VALUE;
865         } else {
866             valueEnd = findEndOfString(sb);
867             value = sb.subStringUnsafe(valueStart, valueEnd);
868         }
869     }
870 
871     private static int findNonSPLenient(AppendableCharSequence sb, int offset) {
872         for (int result = offset; result < sb.length(); ++result) {
873             char c = sb.charAtUnsafe(result);
874             // See https://tools.ietf.org/html/rfc7230#section-3.5
875             if (isSPLenient(c)) {
876                 continue;
877             }
878             if (Character.isWhitespace(c)) {
879                 // Any other whitespace delimiter is invalid
880                 throw new IllegalArgumentException("Invalid separator");
881             }
882             return result;
883         }
884         return sb.length();
885     }
886 
887     private static int findSPLenient(AppendableCharSequence sb, int offset) {
888         for (int result = offset; result < sb.length(); ++result) {
889             if (isSPLenient(sb.charAtUnsafe(result))) {
890                 return result;
891             }
892         }
893         return sb.length();
894     }
895 
896     private static boolean isSPLenient(char c) {
897         // See https://tools.ietf.org/html/rfc7230#section-3.5
898         return c == ' ' || c == (char) 0x09 || c == (char) 0x0B || c == (char) 0x0C || c == (char) 0x0D;
899     }
900 
901     private static int findNonWhitespace(AppendableCharSequence sb, int offset) {
902         for (int result = offset; result < sb.length(); ++result) {
903             char c = sb.charAtUnsafe(result);
904             if (!Character.isWhitespace(c)) {
905                 return result;
906             } else if (!isOWS(c)) {
907                 // Only OWS is supported for whitespace
908                 throw new IllegalArgumentException("Invalid separator, only a single space or horizontal tab allowed," +
909                         " but received a '" + c + "' (0x" + Integer.toHexString(c) + ")");
910             }
911         }
912         return sb.length();
913     }
914 
915     private static int findEndOfString(AppendableCharSequence sb) {
916         for (int result = sb.length() - 1; result > 0; --result) {
917             if (!Character.isWhitespace(sb.charAtUnsafe(result))) {
918                 return result + 1;
919             }
920         }
921         return 0;
922     }
923 
924     private static boolean isOWS(char ch) {
925         return ch == ' ' || ch == (char) 0x09;
926     }
927 
928     private static class HeaderParser implements ByteProcessor {
929         private final AppendableCharSequence seq;
930         private final int maxLength;
931         int size;
932 
933         HeaderParser(AppendableCharSequence seq, int maxLength) {
934             this.seq = seq;
935             this.maxLength = maxLength;
936         }
937 
938         public AppendableCharSequence parse(ByteBuf buffer) {
939             final int oldSize = size;
940             seq.reset();
941             int i = buffer.forEachByte(this);
942             if (i == -1) {
943                 size = oldSize;
944                 return null;
945             }
946             buffer.readerIndex(i + 1);
947             return seq;
948         }
949 
950         public void reset() {
951             size = 0;
952         }
953 
954         @Override
955         public boolean process(byte value) throws Exception {
956             char nextByte = (char) (value & 0xFF);
957             if (nextByte == HttpConstants.LF) {
958                 int len = seq.length();
959                 // Drop CR if we had a CRLF pair
960                 if (len >= 1 && seq.charAtUnsafe(len - 1) == HttpConstants.CR) {
961                     -- size;
962                     seq.setLength(len - 1);
963                 }
964                 return false;
965             }
966 
967             increaseCount();
968 
969             seq.append(nextByte);
970             return true;
971         }
972 
973         protected final void increaseCount() {
974             if (++ size > maxLength) {
975                 // TODO: Respond with Bad Request and discard the traffic
976                 //    or close the connection.
977                 //       No need to notify the upstream handlers - just log.
978                 //       If decoding a response, just throw an exception.
979                 throw newException(maxLength);
980             }
981         }
982 
983         protected TooLongFrameException newException(int maxLength) {
984             return new TooLongHttpHeaderException("HTTP header is larger than " + maxLength + " bytes.");
985         }
986     }
987 
988     private final class LineParser extends HeaderParser {
989 
990         LineParser(AppendableCharSequence seq, int maxLength) {
991             super(seq, maxLength);
992         }
993 
994         @Override
995         public AppendableCharSequence parse(ByteBuf buffer) {
996             // Suppress a warning because HeaderParser.reset() is supposed to be called
997             reset();    // lgtm[java/subtle-inherited-call]
998             return super.parse(buffer);
999         }
1000 
1001         @Override
1002         public boolean process(byte value) throws Exception {
1003             if (currentState == State.SKIP_CONTROL_CHARS) {
1004                 char c = (char) (value & 0xFF);
1005                 if (Character.isISOControl(c) || Character.isWhitespace(c)) {
1006                     increaseCount();
1007                     return true;
1008                 }
1009                 currentState = State.READ_INITIAL;
1010             }
1011             return super.process(value);
1012         }
1013 
1014         @Override
1015         protected TooLongFrameException newException(int maxLength) {
1016             return new TooLongHttpLineException("An HTTP line is larger than " + maxLength + " bytes.");
1017         }
1018     }
1019 }