View Javadoc
1   /*
2    * Copyright 2012 The Netty Project
3    *
4    * The Netty Project licenses this file to you under the Apache License,
5    * version 2.0 (the "License"); you may not use this file except in compliance
6    * with the License. You may obtain a copy of the License at:
7    *
8    *   https://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13   * License for the specific language governing permissions and limitations
14   * under the License.
15   */
16  package io.netty.handler.codec.http;
17  
18  import io.netty.buffer.ByteBuf;
19  import io.netty.buffer.ByteBufUtil;
20  import io.netty.buffer.Unpooled;
21  import io.netty.channel.ChannelHandlerContext;
22  import io.netty.channel.FileRegion;
23  import io.netty.handler.codec.MessageToMessageEncoder;
24  import io.netty.util.CharsetUtil;
25  import io.netty.util.internal.StringUtil;
26  
27  import java.util.Iterator;
28  import java.util.List;
29  import java.util.Map.Entry;
30  
31  import static io.netty.buffer.Unpooled.directBuffer;
32  import static io.netty.buffer.Unpooled.unreleasableBuffer;
33  import static io.netty.handler.codec.http.HttpConstants.CR;
34  import static io.netty.handler.codec.http.HttpConstants.LF;
35  
36  /**
37   * Encodes an {@link HttpMessage} or an {@link HttpContent} into
38   * a {@link ByteBuf}.
39   *
40   * <h3>Extensibility</h3>
41   *
42   * Please note that this encoder is designed to be extended to implement
43   * a protocol derived from HTTP, such as
44   * <a href="https://en.wikipedia.org/wiki/Real_Time_Streaming_Protocol">RTSP</a> and
45   * <a href="https://en.wikipedia.org/wiki/Internet_Content_Adaptation_Protocol">ICAP</a>.
46   * To implement the encoder of such a derived protocol, extend this class and
47   * implement all abstract methods properly.
48   */
49  public abstract class HttpObjectEncoder<H extends HttpMessage> extends MessageToMessageEncoder<Object> {
50      static final int CRLF_SHORT = (CR << 8) | LF;
51      private static final int ZERO_CRLF_MEDIUM = ('0' << 16) | CRLF_SHORT;
52      private static final byte[] ZERO_CRLF_CRLF = { '0', CR, LF, CR, LF };
53      private static final ByteBuf CRLF_BUF = unreleasableBuffer(
54              directBuffer(2).writeByte(CR).writeByte(LF)).asReadOnly();
55      private static final ByteBuf ZERO_CRLF_CRLF_BUF = unreleasableBuffer(
56              directBuffer(ZERO_CRLF_CRLF.length).writeBytes(ZERO_CRLF_CRLF)).asReadOnly();
57      private static final float HEADERS_WEIGHT_NEW = 1 / 5f;
58      private static final float HEADERS_WEIGHT_HISTORICAL = 1 - HEADERS_WEIGHT_NEW;
59      private static final float TRAILERS_WEIGHT_NEW = HEADERS_WEIGHT_NEW;
60      private static final float TRAILERS_WEIGHT_HISTORICAL = HEADERS_WEIGHT_HISTORICAL;
61  
62      private static final int ST_INIT = 0;
63      private static final int ST_CONTENT_NON_CHUNK = 1;
64      private static final int ST_CONTENT_CHUNK = 2;
65      private static final int ST_CONTENT_ALWAYS_EMPTY = 3;
66  
67      @SuppressWarnings("RedundantFieldInitialization")
68      private int state = ST_INIT;
69  
70      /**
71       * Used to calculate an exponential moving average of the encoded size of the initial line and the headers for
72       * a guess for future buffer allocations.
73       */
74      private float headersEncodedSizeAccumulator = 256;
75  
76      /**
77       * Used to calculate an exponential moving average of the encoded size of the trailers for
78       * a guess for future buffer allocations.
79       */
80      private float trailersEncodedSizeAccumulator = 256;
81  
82      @Override
83      protected void encode(ChannelHandlerContext ctx, Object msg, List<Object> out) throws Exception {
84          ByteBuf buf = null;
85          if (msg instanceof HttpMessage) {
86              if (state != ST_INIT) {
87                  throw new IllegalStateException("unexpected message type: " + StringUtil.simpleClassName(msg)
88                          + ", state: " + state);
89              }
90  
91              @SuppressWarnings({ "unchecked", "CastConflictsWithInstanceof" })
92              H m = (H) msg;
93  
94              buf = ctx.alloc().buffer((int) headersEncodedSizeAccumulator);
95              // Encode the message.
96              encodeInitialLine(buf, m);
97              state = isContentAlwaysEmpty(m) ? ST_CONTENT_ALWAYS_EMPTY :
98                      HttpUtil.isTransferEncodingChunked(m) ? ST_CONTENT_CHUNK : ST_CONTENT_NON_CHUNK;
99  
100             sanitizeHeadersBeforeEncode(m, state == ST_CONTENT_ALWAYS_EMPTY);
101 
102             encodeHeaders(m.headers(), buf);
103             ByteBufUtil.writeShortBE(buf, CRLF_SHORT);
104 
105             headersEncodedSizeAccumulator = HEADERS_WEIGHT_NEW * padSizeForAccumulation(buf.readableBytes()) +
106                                             HEADERS_WEIGHT_HISTORICAL * headersEncodedSizeAccumulator;
107         }
108 
109         // Bypass the encoder in case of an empty buffer, so that the following idiom works:
110         //
111         //     ch.write(Unpooled.EMPTY_BUFFER).addListener(ChannelFutureListener.CLOSE);
112         //
113         // See https://github.com/netty/netty/issues/2983 for more information.
114         if (msg instanceof ByteBuf) {
115             final ByteBuf potentialEmptyBuf = (ByteBuf) msg;
116             if (!potentialEmptyBuf.isReadable()) {
117                 out.add(potentialEmptyBuf.retain());
118                 return;
119             }
120         }
121 
122         if (msg instanceof HttpContent || msg instanceof ByteBuf || msg instanceof FileRegion) {
123             switch (state) {
124                 case ST_INIT:
125                     throw new IllegalStateException("unexpected message type: " + StringUtil.simpleClassName(msg)
126                         + ", state: " + state);
127                 case ST_CONTENT_NON_CHUNK:
128                     final long contentLength = contentLength(msg);
129                     if (contentLength > 0) {
130                         if (buf != null && buf.writableBytes() >= contentLength && msg instanceof HttpContent) {
131                             // merge into other buffer for performance reasons
132                             buf.writeBytes(((HttpContent) msg).content());
133                             out.add(buf);
134                         } else {
135                             if (buf != null) {
136                                 out.add(buf);
137                             }
138                             out.add(encodeAndRetain(msg));
139                         }
140 
141                         if (msg instanceof LastHttpContent) {
142                             state = ST_INIT;
143                         }
144 
145                         break;
146                     }
147 
148                     // fall-through!
149                 case ST_CONTENT_ALWAYS_EMPTY:
150 
151                     if (buf != null) {
152                         // We allocated a buffer so add it now.
153                         out.add(buf);
154                     } else {
155                         // Need to produce some output otherwise an
156                         // IllegalStateException will be thrown as we did not write anything
157                         // Its ok to just write an EMPTY_BUFFER as if there are reference count issues these will be
158                         // propagated as the caller of the encode(...) method will release the original
159                         // buffer.
160                         // Writing an empty buffer will not actually write anything on the wire, so if there is a user
161                         // error with msg it will not be visible externally
162                         out.add(Unpooled.EMPTY_BUFFER);
163                     }
164 
165                     break;
166                 case ST_CONTENT_CHUNK:
167                     if (buf != null) {
168                         // We allocated a buffer so add it now.
169                         out.add(buf);
170                     }
171                     encodeChunkedContent(ctx, msg, contentLength(msg), out);
172 
173                     break;
174                 default:
175                     throw new Error();
176             }
177 
178             if (msg instanceof LastHttpContent) {
179                 state = ST_INIT;
180             }
181         } else if (buf != null) {
182             out.add(buf);
183         }
184     }
185 
186     /**
187      * Encode the {@link HttpHeaders} into a {@link ByteBuf}.
188      */
189     protected void encodeHeaders(HttpHeaders headers, ByteBuf buf) {
190         Iterator<Entry<CharSequence, CharSequence>> iter = headers.iteratorCharSequence();
191         while (iter.hasNext()) {
192             Entry<CharSequence, CharSequence> header = iter.next();
193             HttpHeadersEncoder.encoderHeader(header.getKey(), header.getValue(), buf);
194         }
195     }
196 
197     private void encodeChunkedContent(ChannelHandlerContext ctx, Object msg, long contentLength, List<Object> out) {
198         if (contentLength > 0) {
199             String lengthHex = Long.toHexString(contentLength);
200             ByteBuf buf = ctx.alloc().buffer(lengthHex.length() + 2);
201             buf.writeCharSequence(lengthHex, CharsetUtil.US_ASCII);
202             ByteBufUtil.writeShortBE(buf, CRLF_SHORT);
203             out.add(buf);
204             out.add(encodeAndRetain(msg));
205             out.add(CRLF_BUF.duplicate());
206         }
207 
208         if (msg instanceof LastHttpContent) {
209             HttpHeaders headers = ((LastHttpContent) msg).trailingHeaders();
210             if (headers.isEmpty()) {
211                 out.add(ZERO_CRLF_CRLF_BUF.duplicate());
212             } else {
213                 ByteBuf buf = ctx.alloc().buffer((int) trailersEncodedSizeAccumulator);
214                 ByteBufUtil.writeMediumBE(buf, ZERO_CRLF_MEDIUM);
215                 encodeHeaders(headers, buf);
216                 ByteBufUtil.writeShortBE(buf, CRLF_SHORT);
217                 trailersEncodedSizeAccumulator = TRAILERS_WEIGHT_NEW * padSizeForAccumulation(buf.readableBytes()) +
218                                                  TRAILERS_WEIGHT_HISTORICAL * trailersEncodedSizeAccumulator;
219                 out.add(buf);
220             }
221         } else if (contentLength == 0) {
222             // Need to produce some output otherwise an
223             // IllegalStateException will be thrown
224             out.add(encodeAndRetain(msg));
225         }
226     }
227 
228     /**
229      * Allows to sanitize headers of the message before encoding these.
230      */
231     protected void sanitizeHeadersBeforeEncode(@SuppressWarnings("unused") H msg, boolean isAlwaysEmpty) {
232         // noop
233     }
234 
235     /**
236      * Determine whether a message has a content or not. Some message may have headers indicating
237      * a content without having an actual content, e.g the response to an HEAD or CONNECT request.
238      *
239      * @param msg the message to test
240      * @return {@code true} to signal the message has no content
241      */
242     protected boolean isContentAlwaysEmpty(@SuppressWarnings("unused") H msg) {
243         return false;
244     }
245 
246     @Override
247     public boolean acceptOutboundMessage(Object msg) throws Exception {
248         return msg instanceof HttpObject || msg instanceof ByteBuf || msg instanceof FileRegion;
249     }
250 
251     private static Object encodeAndRetain(Object msg) {
252         if (msg instanceof ByteBuf) {
253             return ((ByteBuf) msg).retain();
254         }
255         if (msg instanceof HttpContent) {
256             return ((HttpContent) msg).content().retain();
257         }
258         if (msg instanceof FileRegion) {
259             return ((FileRegion) msg).retain();
260         }
261         throw new IllegalStateException("unexpected message type: " + StringUtil.simpleClassName(msg));
262     }
263 
264     private static long contentLength(Object msg) {
265         if (msg instanceof HttpContent) {
266             return ((HttpContent) msg).content().readableBytes();
267         }
268         if (msg instanceof ByteBuf) {
269             return ((ByteBuf) msg).readableBytes();
270         }
271         if (msg instanceof FileRegion) {
272             return ((FileRegion) msg).count();
273         }
274         throw new IllegalStateException("unexpected message type: " + StringUtil.simpleClassName(msg));
275     }
276 
277     /**
278      * Add some additional overhead to the buffer. The rational is that it is better to slightly over allocate and waste
279      * some memory, rather than under allocate and require a resize/copy.
280      * @param readableBytes The readable bytes in the buffer.
281      * @return The {@code readableBytes} with some additional padding.
282      */
283     private static int padSizeForAccumulation(int readableBytes) {
284         return (readableBytes << 2) / 3;
285     }
286 
287     @Deprecated
288     protected static void encodeAscii(String s, ByteBuf buf) {
289         buf.writeCharSequence(s, CharsetUtil.US_ASCII);
290     }
291 
292     protected abstract void encodeInitialLine(ByteBuf buf, H message) throws Exception;
293 }