View Javadoc
1   /*
2    * Copyright 2015 The Netty Project
3    *
4    * The Netty Project licenses this file to you under the Apache License,
5    * version 2.0 (the "License"); you may not use this file except in compliance
6    * with the License. You may obtain a copy of the License at:
7    *
8    *   https://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13   * License for the specific language governing permissions and limitations
14   * under the License.
15   */
16  
17  /*
18   * Copyright 2014 Twitter, Inc.
19   *
20   * Licensed under the Apache License, Version 2.0 (the "License");
21   * you may not use this file except in compliance with the License.
22   * You may obtain a copy of the License at
23   *
24   *     https://www.apache.org/licenses/LICENSE-2.0
25   *
26   * Unless required by applicable law or agreed to in writing, software
27   * distributed under the License is distributed on an "AS IS" BASIS,
28   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
29   * See the License for the specific language governing permissions and
30   * limitations under the License.
31   */
32  package io.netty.handler.codec.http2;
33  
34  import io.netty.buffer.ByteBuf;
35  import io.netty.handler.codec.http2.HpackUtil.IndexType;
36  import io.netty.handler.codec.http2.Http2HeadersEncoder.SensitivityDetector;
37  import io.netty.util.AsciiString;
38  import io.netty.util.CharsetUtil;
39  
40  import java.util.Map;
41  
42  import static io.netty.handler.codec.http2.HpackUtil.equalsConstantTime;
43  import static io.netty.handler.codec.http2.HpackUtil.equalsVariableTime;
44  import static io.netty.handler.codec.http2.Http2CodecUtil.DEFAULT_HEADER_TABLE_SIZE;
45  import static io.netty.handler.codec.http2.Http2CodecUtil.MAX_HEADER_LIST_SIZE;
46  import static io.netty.handler.codec.http2.Http2CodecUtil.MAX_HEADER_TABLE_SIZE;
47  import static io.netty.handler.codec.http2.Http2CodecUtil.MIN_HEADER_LIST_SIZE;
48  import static io.netty.handler.codec.http2.Http2CodecUtil.MIN_HEADER_TABLE_SIZE;
49  import static io.netty.handler.codec.http2.Http2CodecUtil.headerListSizeExceeded;
50  import static io.netty.handler.codec.http2.Http2Error.PROTOCOL_ERROR;
51  import static io.netty.handler.codec.http2.Http2Exception.connectionError;
52  import static io.netty.util.internal.MathUtil.findNextPositivePowerOfTwo;
53  import static java.lang.Math.max;
54  import static java.lang.Math.min;
55  
56  /**
57   * An HPACK encoder.
58   *
59   * <p>Implementation note:  This class is security sensitive, and depends on users correctly identifying their headers
60   * as security sensitive or not.  If a header is considered not sensitive, methods names "insensitive" are used which
61   * are fast, but don't provide any security guarantees.
62   */
63  final class HpackEncoder {
64      static final int NOT_FOUND = -1;
65      static final int HUFF_CODE_THRESHOLD = 512;
66      // a hash map of header fields keyed by header name
67      private final NameEntry[] nameEntries;
68  
69      // a hash map of header fields keyed by header name and value
70      private final NameValueEntry[] nameValueEntries;
71  
72      private final NameValueEntry head = new NameValueEntry(-1, AsciiString.EMPTY_STRING,
73        AsciiString.EMPTY_STRING, Integer.MAX_VALUE, null);
74  
75      private NameValueEntry latest = head;
76  
77      private final HpackHuffmanEncoder hpackHuffmanEncoder = new HpackHuffmanEncoder();
78      private final byte hashMask;
79      private final boolean ignoreMaxHeaderListSize;
80      private final int huffCodeThreshold;
81      private long size;
82      private long maxHeaderTableSize;
83      private long maxHeaderListSize;
84  
85      /**
86       * Creates a new encoder.
87       */
88      HpackEncoder() {
89          this(false);
90      }
91  
92      /**
93       * Creates a new encoder.
94       */
95      HpackEncoder(boolean ignoreMaxHeaderListSize) {
96          this(ignoreMaxHeaderListSize, 64, HUFF_CODE_THRESHOLD);
97      }
98  
99      /**
100      * Creates a new encoder.
101      */
102     HpackEncoder(boolean ignoreMaxHeaderListSize, int arraySizeHint, int huffCodeThreshold) {
103         this.ignoreMaxHeaderListSize = ignoreMaxHeaderListSize;
104         maxHeaderTableSize = DEFAULT_HEADER_TABLE_SIZE;
105         maxHeaderListSize = MAX_HEADER_LIST_SIZE;
106         // Enforce a bound of [2, 128] because hashMask is a byte. The max possible value of hashMask is one less
107         // than the length of this array, and we want the mask to be > 0.
108         nameEntries = new NameEntry[findNextPositivePowerOfTwo(max(2, min(arraySizeHint, 128)))];
109         nameValueEntries = new NameValueEntry[nameEntries.length];
110         hashMask = (byte) (nameEntries.length - 1);
111         this.huffCodeThreshold = huffCodeThreshold;
112     }
113 
114     /**
115      * Encode the header field into the header block.
116      * <p>
117      * <strong>The given {@link CharSequence}s must be immutable!</strong>
118      */
119     public void encodeHeaders(int streamId, ByteBuf out, Http2Headers headers, SensitivityDetector sensitivityDetector)
120       throws Http2Exception {
121         if (ignoreMaxHeaderListSize) {
122             encodeHeadersIgnoreMaxHeaderListSize(out, headers, sensitivityDetector);
123         } else {
124             encodeHeadersEnforceMaxHeaderListSize(streamId, out, headers, sensitivityDetector);
125         }
126     }
127 
128     private void encodeHeadersEnforceMaxHeaderListSize(int streamId, ByteBuf out, Http2Headers headers,
129                                                        SensitivityDetector sensitivityDetector)
130       throws Http2Exception {
131         long headerSize = 0;
132         // To ensure we stay consistent with our peer check the size is valid before we potentially modify HPACK state.
133         for (Map.Entry<CharSequence, CharSequence> header : headers) {
134             CharSequence name = header.getKey();
135             CharSequence value = header.getValue();
136             // OK to increment now and check for bounds after because this value is limited to unsigned int and will not
137             // overflow.
138             headerSize += HpackHeaderField.sizeOf(name, value);
139             if (headerSize > maxHeaderListSize) {
140                 headerListSizeExceeded(streamId, maxHeaderListSize, false);
141             }
142         }
143         encodeHeadersIgnoreMaxHeaderListSize(out, headers, sensitivityDetector);
144     }
145 
146     private void encodeHeadersIgnoreMaxHeaderListSize(ByteBuf out, Http2Headers headers,
147                                                       SensitivityDetector sensitivityDetector) {
148         for (Map.Entry<CharSequence, CharSequence> header : headers) {
149             CharSequence name = header.getKey();
150             CharSequence value = header.getValue();
151             encodeHeader(out, name, value, sensitivityDetector.isSensitive(name, value),
152               HpackHeaderField.sizeOf(name, value));
153         }
154     }
155 
156     /**
157      * Encode the header field into the header block.
158      * <p>
159      * <strong>The given {@link CharSequence}s must be immutable!</strong>
160      */
161     private void encodeHeader(ByteBuf out, CharSequence name, CharSequence value, boolean sensitive, long headerSize) {
162         // If the header value is sensitive then it must never be indexed
163         if (sensitive) {
164             int nameIndex = getNameIndex(name);
165             encodeLiteral(out, name, value, IndexType.NEVER, nameIndex);
166             return;
167         }
168 
169         // If the peer will only use the static table
170         if (maxHeaderTableSize == 0) {
171             int staticTableIndex = HpackStaticTable.getIndexInsensitive(name, value);
172             if (staticTableIndex == HpackStaticTable.NOT_FOUND) {
173                 int nameIndex = HpackStaticTable.getIndex(name);
174                 encodeLiteral(out, name, value, IndexType.NONE, nameIndex);
175             } else {
176                 encodeInteger(out, 0x80, 7, staticTableIndex);
177             }
178             return;
179         }
180 
181         // If the headerSize is greater than the max table size then it must be encoded literally
182         if (headerSize > maxHeaderTableSize) {
183             int nameIndex = getNameIndex(name);
184             encodeLiteral(out, name, value, IndexType.NONE, nameIndex);
185             return;
186         }
187 
188         int nameHash = AsciiString.hashCode(name);
189         int valueHash = AsciiString.hashCode(value);
190         NameValueEntry headerField = getEntryInsensitive(name, nameHash, value, valueHash);
191         if (headerField != null) {
192             // Section 6.1. Indexed Header Field Representation
193             encodeInteger(out, 0x80, 7, getIndexPlusOffset(headerField.counter));
194         } else {
195             int staticTableIndex = HpackStaticTable.getIndexInsensitive(name, value);
196             if (staticTableIndex != HpackStaticTable.NOT_FOUND) {
197                 // Section 6.1. Indexed Header Field Representation
198                 encodeInteger(out, 0x80, 7, staticTableIndex);
199             } else {
200                 ensureCapacity(headerSize);
201                 encodeAndAddEntries(out, name, nameHash, value, valueHash);
202                 size += headerSize;
203             }
204         }
205     }
206 
207     private void encodeAndAddEntries(ByteBuf out, CharSequence name, int nameHash, CharSequence value, int valueHash) {
208         int staticTableIndex = HpackStaticTable.getIndex(name);
209         int nextCounter = latestCounter() - 1;
210         if (staticTableIndex == HpackStaticTable.NOT_FOUND) {
211             NameEntry e = getEntry(name, nameHash);
212             if (e == null) {
213                 encodeLiteral(out, name, value, IndexType.INCREMENTAL, NOT_FOUND);
214                 addNameEntry(name, nameHash, nextCounter);
215                 addNameValueEntry(name, value, nameHash, valueHash, nextCounter);
216             } else {
217                 encodeLiteral(out, name, value, IndexType.INCREMENTAL, getIndexPlusOffset(e.counter));
218                 addNameValueEntry(e.name, value, nameHash, valueHash, nextCounter);
219 
220                 // The name entry should always point to the latest counter.
221                 e.counter = nextCounter;
222             }
223         } else {
224             encodeLiteral(out, name, value, IndexType.INCREMENTAL, staticTableIndex);
225             // use the name from the static table to optimize memory usage.
226             addNameValueEntry(
227               HpackStaticTable.getEntry(staticTableIndex).name, value, nameHash, valueHash, nextCounter);
228         }
229     }
230 
231     /**
232      * Set the maximum table size.
233      */
234     public void setMaxHeaderTableSize(ByteBuf out, long maxHeaderTableSize) throws Http2Exception {
235         if (maxHeaderTableSize < MIN_HEADER_TABLE_SIZE || maxHeaderTableSize > MAX_HEADER_TABLE_SIZE) {
236             throw connectionError(PROTOCOL_ERROR, "Header Table Size must be >= %d and <= %d but was %d",
237               MIN_HEADER_TABLE_SIZE, MAX_HEADER_TABLE_SIZE, maxHeaderTableSize);
238         }
239         if (this.maxHeaderTableSize == maxHeaderTableSize) {
240             return;
241         }
242         this.maxHeaderTableSize = maxHeaderTableSize;
243         ensureCapacity(0);
244         // Casting to integer is safe as we verified the maxHeaderTableSize is a valid unsigned int.
245         encodeInteger(out, 0x20, 5, maxHeaderTableSize);
246     }
247 
248     /**
249      * Return the maximum table size.
250      */
251     public long getMaxHeaderTableSize() {
252         return maxHeaderTableSize;
253     }
254 
255     public void setMaxHeaderListSize(long maxHeaderListSize) throws Http2Exception {
256         if (maxHeaderListSize < MIN_HEADER_LIST_SIZE || maxHeaderListSize > MAX_HEADER_LIST_SIZE) {
257             throw connectionError(PROTOCOL_ERROR, "Header List Size must be >= %d and <= %d but was %d",
258               MIN_HEADER_LIST_SIZE, MAX_HEADER_LIST_SIZE, maxHeaderListSize);
259         }
260         this.maxHeaderListSize = maxHeaderListSize;
261     }
262 
263     public long getMaxHeaderListSize() {
264         return maxHeaderListSize;
265     }
266 
267     /**
268      * Encode integer according to <a href="https://tools.ietf.org/html/rfc7541#section-5.1">Section 5.1</a>.
269      */
270     private static void encodeInteger(ByteBuf out, int mask, int n, int i) {
271         encodeInteger(out, mask, n, (long) i);
272     }
273 
274     /**
275      * Encode integer according to <a href="https://tools.ietf.org/html/rfc7541#section-5.1">Section 5.1</a>.
276      */
277     private static void encodeInteger(ByteBuf out, int mask, int n, long i) {
278         assert n >= 0 && n <= 8 : "N: " + n;
279         int nbits = 0xFF >>> 8 - n;
280         if (i < nbits) {
281             out.writeByte((int) (mask | i));
282         } else {
283             out.writeByte(mask | nbits);
284             long length = i - nbits;
285             for (; (length & ~0x7F) != 0; length >>>= 7) {
286                 out.writeByte((int) (length & 0x7F | 0x80));
287             }
288             out.writeByte((int) length);
289         }
290     }
291 
292     /**
293      * Encode string literal according to Section 5.2.
294      */
295     private void encodeStringLiteral(ByteBuf out, CharSequence string) {
296         int huffmanLength;
297         if (string.length() >= huffCodeThreshold
298           && (huffmanLength = hpackHuffmanEncoder.getEncodedLength(string)) < string.length()) {
299             encodeInteger(out, 0x80, 7, huffmanLength);
300             hpackHuffmanEncoder.encode(out, string);
301         } else {
302             encodeInteger(out, 0x00, 7, string.length());
303             if (string instanceof AsciiString) {
304                 // Fast-path
305                 AsciiString asciiString = (AsciiString) string;
306                 out.writeBytes(asciiString.array(), asciiString.arrayOffset(), asciiString.length());
307             } else {
308                 // Only ASCII is allowed in http2 headers, so it is fine to use this.
309                 // https://tools.ietf.org/html/rfc7540#section-8.1.2
310                 out.writeCharSequence(string, CharsetUtil.ISO_8859_1);
311             }
312         }
313     }
314 
315     /**
316      * Encode literal header field according to Section 6.2.
317      */
318     private void encodeLiteral(ByteBuf out, CharSequence name, CharSequence value, IndexType indexType,
319                                int nameIndex) {
320         boolean nameIndexValid = nameIndex != NOT_FOUND;
321         switch (indexType) {
322             case INCREMENTAL:
323                 encodeInteger(out, 0x40, 6, nameIndexValid ? nameIndex : 0);
324                 break;
325             case NONE:
326                 encodeInteger(out, 0x00, 4, nameIndexValid ? nameIndex : 0);
327                 break;
328             case NEVER:
329                 encodeInteger(out, 0x10, 4, nameIndexValid ? nameIndex : 0);
330                 break;
331             default:
332                 throw new Error("should not reach here");
333         }
334         if (!nameIndexValid) {
335             encodeStringLiteral(out, name);
336         }
337         encodeStringLiteral(out, value);
338     }
339 
340     private int getNameIndex(CharSequence name) {
341         int index = HpackStaticTable.getIndex(name);
342         if (index != HpackStaticTable.NOT_FOUND) {
343             return index;
344         }
345         NameEntry e = getEntry(name, AsciiString.hashCode(name));
346         return e == null ? NOT_FOUND : getIndexPlusOffset(e.counter);
347     }
348 
349     /**
350      * Ensure that the dynamic table has enough room to hold 'headerSize' more bytes. Removes the
351      * oldest entry from the dynamic table until sufficient space is available.
352      */
353     private void ensureCapacity(long headerSize) {
354         while (maxHeaderTableSize - size < headerSize) {
355             remove();
356         }
357     }
358 
359     /**
360      * Return the number of header fields in the dynamic table. Exposed for testing.
361      */
362     int length() {
363         return isEmpty() ? 0 : getIndex(head.after.counter);
364     }
365 
366     /**
367      * Return the size of the dynamic table. Exposed for testing.
368      */
369     long size() {
370         return size;
371     }
372 
373     /**
374      * Return the header field at the given index. Exposed for testing.
375      */
376     HpackHeaderField getHeaderField(int index) {
377         NameValueEntry entry = head;
378         while (index++ < length()) {
379             entry = entry.after;
380         }
381         return entry;
382     }
383 
384     /**
385      * Returns the header entry with the lowest index value for the header field. Returns null if
386      * header field is not in the dynamic table.
387      */
388     private NameValueEntry getEntryInsensitive(CharSequence name, int nameHash, CharSequence value, int valueHash) {
389         int h = hash(nameHash, valueHash);
390         for (NameValueEntry e = nameValueEntries[bucket(h)]; e != null; e = e.next) {
391             if (e.hash == h && equalsVariableTime(value, e.value) && equalsVariableTime(name, e.name)) {
392                 return e;
393             }
394         }
395         return null;
396     }
397 
398     /**
399      * Returns the lowest index value for the header field name in the dynamic table. Returns -1 if
400      * the header field name is not in the dynamic table.
401      */
402     private NameEntry getEntry(CharSequence name, int nameHash) {
403         for (NameEntry e = nameEntries[bucket(nameHash)]; e != null; e = e.next) {
404             if (e.hash == nameHash && equalsConstantTime(name, e.name) != 0) {
405                 return e;
406             }
407         }
408         return null;
409     }
410 
411     private int getIndexPlusOffset(int counter) {
412         return getIndex(counter) + HpackStaticTable.length;
413     }
414 
415     /**
416      * Compute the index into the dynamic table given the counter in the header entry.
417      */
418     private int getIndex(int counter) {
419         return counter - latestCounter() + 1;
420     }
421 
422     private int latestCounter() {
423         return latest.counter;
424     }
425 
426     private void addNameEntry(CharSequence name, int nameHash, int nextCounter) {
427         int bucket = bucket(nameHash);
428         nameEntries[bucket] = new NameEntry(nameHash, name, nextCounter, nameEntries[bucket]);
429     }
430 
431     private void addNameValueEntry(CharSequence name, CharSequence value,
432                                    int nameHash, int valueHash, int nextCounter) {
433         int hash = hash(nameHash, valueHash);
434         int bucket = bucket(hash);
435         NameValueEntry e = new NameValueEntry(hash, name, value, nextCounter, nameValueEntries[bucket]);
436         nameValueEntries[bucket] = e;
437         latest.after = e;
438         latest = e;
439     }
440 
441     /**
442      * Remove the oldest header field from the dynamic table.
443      */
444     private void remove() {
445         NameValueEntry eldest = head.after;
446         removeNameValueEntry(eldest);
447         removeNameEntryMatchingCounter(eldest.name, eldest.counter);
448         head.after = eldest.after;
449         eldest.unlink();
450         size -= eldest.size();
451         if (isEmpty()) {
452             latest = head;
453         }
454     }
455 
456     private boolean isEmpty() {
457         return size == 0;
458     }
459 
460     private void removeNameValueEntry(NameValueEntry eldest) {
461         int bucket = bucket(eldest.hash);
462         NameValueEntry e = nameValueEntries[bucket];
463         if (e == eldest) {
464             nameValueEntries[bucket] = eldest.next;
465         } else {
466             while (e.next != eldest) {
467                 e = e.next;
468             }
469             e.next = eldest.next;
470         }
471     }
472 
473     private void removeNameEntryMatchingCounter(CharSequence name, int counter) {
474         int hash = AsciiString.hashCode(name);
475         int bucket = bucket(hash);
476         NameEntry e = nameEntries[bucket];
477         if (e == null) {
478             return;
479         }
480         if (counter == e.counter) {
481             nameEntries[bucket] = e.next;
482             e.unlink();
483         } else {
484             NameEntry prev = e;
485             e = e.next;
486             while (e != null) {
487                 if (counter == e.counter) {
488                     prev.next = e.next;
489                     e.unlink();
490                     break;
491                 }
492                 prev = e;
493                 e = e.next;
494             }
495         }
496     }
497 
498     /**
499      * Returns the bucket of the hash table for the hash code h.
500      */
501     private int bucket(int h) {
502         return h & hashMask;
503     }
504 
505     private static int hash(int nameHash, int valueHash) {
506         return 31 * nameHash + valueHash;
507     }
508 
509     private static final class NameEntry {
510         NameEntry next;
511 
512         final CharSequence name;
513 
514         final int hash;
515 
516         // This is used to compute the index in the dynamic table.
517         int counter;
518 
519         NameEntry(int hash, CharSequence name, int counter, NameEntry next) {
520             this.hash = hash;
521             this.name = name;
522             this.counter = counter;
523             this.next = next;
524         }
525 
526         void unlink() {
527             next = null; // null references to prevent nepotism in generational GC.
528         }
529     }
530 
531     private static final class NameValueEntry extends HpackHeaderField {
532         // This field comprises the linked list used for implementing the eviction policy.
533         NameValueEntry after;
534 
535         NameValueEntry next;
536 
537         // hash of both name and value
538         final int hash;
539 
540         // This is used to compute the index in the dynamic table.
541         final int counter;
542 
543         NameValueEntry(int hash, CharSequence name, CharSequence value, int counter, NameValueEntry next) {
544             super(name, value);
545             this.next = next;
546             this.hash = hash;
547             this.counter = counter;
548         }
549 
550         void unlink() {
551             after = null; // null references to prevent nepotism in generational GC.
552             next = null;
553         }
554     }
555 }