View Javadoc
1   /*
2    * Copyright 2015 The Netty Project
3    *
4    * The Netty Project licenses this file to you under the Apache License,
5    * version 2.0 (the "License"); you may not use this file except in compliance
6    * with the License. You may obtain a copy of the License at:
7    *
8    *   https://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13   * License for the specific language governing permissions and limitations
14   * under the License.
15   */
16  
17  /*
18   * Copyright 2014 Twitter, Inc.
19   *
20   * Licensed under the Apache License, Version 2.0 (the "License");
21   * you may not use this file except in compliance with the License.
22   * You may obtain a copy of the License at
23   *
24   *     https://www.apache.org/licenses/LICENSE-2.0
25   *
26   * Unless required by applicable law or agreed to in writing, software
27   * distributed under the License is distributed on an "AS IS" BASIS,
28   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
29   * See the License for the specific language governing permissions and
30   * limitations under the License.
31   */
32  package io.netty.handler.codec.http2;
33  
34  import io.netty.buffer.ByteBuf;
35  import io.netty.handler.codec.http2.HpackUtil.IndexType;
36  import io.netty.handler.codec.http2.Http2HeadersEncoder.SensitivityDetector;
37  import io.netty.util.AsciiString;
38  import io.netty.util.CharsetUtil;
39  
40  import java.util.Map;
41  
42  import static io.netty.handler.codec.http2.HpackUtil.equalsConstantTime;
43  import static io.netty.handler.codec.http2.HpackUtil.equalsVariableTime;
44  import static io.netty.handler.codec.http2.Http2CodecUtil.DEFAULT_HEADER_TABLE_SIZE;
45  import static io.netty.handler.codec.http2.Http2CodecUtil.MAX_HEADER_LIST_SIZE;
46  import static io.netty.handler.codec.http2.Http2CodecUtil.MAX_HEADER_TABLE_SIZE;
47  import static io.netty.handler.codec.http2.Http2CodecUtil.MIN_HEADER_LIST_SIZE;
48  import static io.netty.handler.codec.http2.Http2CodecUtil.MIN_HEADER_TABLE_SIZE;
49  import static io.netty.handler.codec.http2.Http2CodecUtil.headerListSizeExceeded;
50  import static io.netty.handler.codec.http2.Http2Error.PROTOCOL_ERROR;
51  import static io.netty.handler.codec.http2.Http2Exception.connectionError;
52  import static io.netty.util.internal.MathUtil.findNextPositivePowerOfTwo;
53  import static java.lang.Math.max;
54  import static java.lang.Math.min;
55  
56  /**
57   * An HPACK encoder.
58   *
59   * <p>Implementation note:  This class is security sensitive, and depends on users correctly identifying their headers
60   * as security sensitive or not.  If a header is considered not sensitive, methods names "insensitive" are used which
61   * are fast, but don't provide any security guarantees.
62   */
63  final class HpackEncoder {
64      static final int NOT_FOUND = -1;
65      static final int HUFF_CODE_THRESHOLD = 512;
66      // a hash map of header fields keyed by header name
67      private final NameEntry[] nameEntries;
68  
69      // a hash map of header fields keyed by header name and value
70      private final NameValueEntry[] nameValueEntries;
71  
72      private final NameValueEntry head = new NameValueEntry(-1, AsciiString.EMPTY_STRING,
73        AsciiString.EMPTY_STRING, Integer.MAX_VALUE, null);
74  
75      private NameValueEntry latest = head;
76  
77      private final HpackHuffmanEncoder hpackHuffmanEncoder = new HpackHuffmanEncoder();
78      private final byte hashMask;
79      private final boolean ignoreMaxHeaderListSize;
80      private final int huffCodeThreshold;
81      private long size;
82      private long maxHeaderTableSize;
83      private long maxHeaderListSize;
84  
85      /**
86       * Creates a new encoder.
87       */
88      HpackEncoder() {
89          this(false);
90      }
91  
92      /**
93       * Creates a new encoder.
94       */
95      HpackEncoder(boolean ignoreMaxHeaderListSize) {
96          this(ignoreMaxHeaderListSize, 64, HUFF_CODE_THRESHOLD);
97      }
98  
99      /**
100      * Creates a new encoder.
101      */
102     HpackEncoder(boolean ignoreMaxHeaderListSize, int arraySizeHint, int huffCodeThreshold) {
103         this.ignoreMaxHeaderListSize = ignoreMaxHeaderListSize;
104         maxHeaderTableSize = DEFAULT_HEADER_TABLE_SIZE;
105         maxHeaderListSize = MAX_HEADER_LIST_SIZE;
106         // Enforce a bound of [2, 128] because hashMask is a byte. The max possible value of hashMask is one less
107         // than the length of this array, and we want the mask to be > 0. There is also simply an
108         // upper limit on how many entries can be safely held, as nameEntries will have duplicate
109         // entries every time a header name is used, and thus form a long collision chain.
110         nameEntries = new NameEntry[findNextPositivePowerOfTwo(max(2, min(arraySizeHint, 128)))];
111         nameValueEntries = new NameValueEntry[nameEntries.length];
112         hashMask = (byte) (nameEntries.length - 1);
113         this.huffCodeThreshold = huffCodeThreshold;
114     }
115 
116     /**
117      * Encode the header field into the header block.
118      * <p>
119      * <strong>The given {@link CharSequence}s must be immutable!</strong>
120      */
121     public void encodeHeaders(int streamId, ByteBuf out, Http2Headers headers, SensitivityDetector sensitivityDetector)
122       throws Http2Exception {
123         if (ignoreMaxHeaderListSize) {
124             encodeHeadersIgnoreMaxHeaderListSize(out, headers, sensitivityDetector);
125         } else {
126             encodeHeadersEnforceMaxHeaderListSize(streamId, out, headers, sensitivityDetector);
127         }
128     }
129 
130     private void encodeHeadersEnforceMaxHeaderListSize(int streamId, ByteBuf out, Http2Headers headers,
131                                                        SensitivityDetector sensitivityDetector)
132       throws Http2Exception {
133         long headerSize = 0;
134         // To ensure we stay consistent with our peer check the size is valid before we potentially modify HPACK state.
135         for (Map.Entry<CharSequence, CharSequence> header : headers) {
136             CharSequence name = header.getKey();
137             CharSequence value = header.getValue();
138             // OK to increment now and check for bounds after because this value is limited to unsigned int and will not
139             // overflow.
140             headerSize += HpackHeaderField.sizeOf(name, value);
141             if (headerSize > maxHeaderListSize) {
142                 headerListSizeExceeded(streamId, maxHeaderListSize, false);
143             }
144         }
145         encodeHeadersIgnoreMaxHeaderListSize(out, headers, sensitivityDetector);
146     }
147 
148     private void encodeHeadersIgnoreMaxHeaderListSize(ByteBuf out, Http2Headers headers,
149                                                       SensitivityDetector sensitivityDetector) {
150         for (Map.Entry<CharSequence, CharSequence> header : headers) {
151             CharSequence name = header.getKey();
152             CharSequence value = header.getValue();
153             encodeHeader(out, name, value, sensitivityDetector.isSensitive(name, value),
154               HpackHeaderField.sizeOf(name, value));
155         }
156     }
157 
158     /**
159      * Encode the header field into the header block.
160      * <p>
161      * <strong>The given {@link CharSequence}s must be immutable!</strong>
162      */
163     private void encodeHeader(ByteBuf out, CharSequence name, CharSequence value, boolean sensitive, long headerSize) {
164         // If the header value is sensitive then it must never be indexed
165         if (sensitive) {
166             int nameIndex = getNameIndex(name);
167             encodeLiteral(out, name, value, IndexType.NEVER, nameIndex);
168             return;
169         }
170 
171         // If the peer will only use the static table
172         if (maxHeaderTableSize == 0) {
173             int staticTableIndex = HpackStaticTable.getIndexInsensitive(name, value);
174             if (staticTableIndex == HpackStaticTable.NOT_FOUND) {
175                 int nameIndex = HpackStaticTable.getIndex(name);
176                 encodeLiteral(out, name, value, IndexType.NONE, nameIndex);
177             } else {
178                 encodeInteger(out, 0x80, 7, staticTableIndex);
179             }
180             return;
181         }
182 
183         // If the headerSize is greater than the max table size then it must be encoded literally
184         if (headerSize > maxHeaderTableSize) {
185             int nameIndex = getNameIndex(name);
186             encodeLiteral(out, name, value, IndexType.NONE, nameIndex);
187             return;
188         }
189 
190         int nameHash = AsciiString.hashCode(name);
191         int valueHash = AsciiString.hashCode(value);
192         NameValueEntry headerField = getEntryInsensitive(name, nameHash, value, valueHash);
193         if (headerField != null) {
194             // Section 6.1. Indexed Header Field Representation
195             encodeInteger(out, 0x80, 7, getIndexPlusOffset(headerField.counter));
196         } else {
197             int staticTableIndex = HpackStaticTable.getIndexInsensitive(name, value);
198             if (staticTableIndex != HpackStaticTable.NOT_FOUND) {
199                 // Section 6.1. Indexed Header Field Representation
200                 encodeInteger(out, 0x80, 7, staticTableIndex);
201             } else {
202                 ensureCapacity(headerSize);
203                 encodeAndAddEntries(out, name, nameHash, value, valueHash);
204                 size += headerSize;
205             }
206         }
207     }
208 
209     private void encodeAndAddEntries(ByteBuf out, CharSequence name, int nameHash, CharSequence value, int valueHash) {
210         int staticTableIndex = HpackStaticTable.getIndex(name);
211         int nextCounter = latestCounter() - 1;
212         if (staticTableIndex == HpackStaticTable.NOT_FOUND) {
213             NameEntry e = getEntry(name, nameHash);
214             if (e == null) {
215                 encodeLiteral(out, name, value, IndexType.INCREMENTAL, NOT_FOUND);
216                 addNameEntry(name, nameHash, nextCounter);
217                 addNameValueEntry(name, value, nameHash, valueHash, nextCounter);
218             } else {
219                 encodeLiteral(out, name, value, IndexType.INCREMENTAL, getIndexPlusOffset(e.counter));
220                 addNameValueEntry(e.name, value, nameHash, valueHash, nextCounter);
221 
222                 // The name entry should always point to the latest counter.
223                 e.counter = nextCounter;
224             }
225         } else {
226             encodeLiteral(out, name, value, IndexType.INCREMENTAL, staticTableIndex);
227             // use the name from the static table to optimize memory usage.
228             addNameValueEntry(
229               HpackStaticTable.getEntry(staticTableIndex).name, value, nameHash, valueHash, nextCounter);
230         }
231     }
232 
233     /**
234      * Set the maximum table size.
235      */
236     public void setMaxHeaderTableSize(ByteBuf out, long maxHeaderTableSize) throws Http2Exception {
237         if (maxHeaderTableSize < MIN_HEADER_TABLE_SIZE || maxHeaderTableSize > MAX_HEADER_TABLE_SIZE) {
238             throw connectionError(PROTOCOL_ERROR, "Header Table Size must be >= %d and <= %d but was %d",
239               MIN_HEADER_TABLE_SIZE, MAX_HEADER_TABLE_SIZE, maxHeaderTableSize);
240         }
241         // While the receiver may allow a larger table, it is important to cap the local impact of
242         // hash collisions and memory use. This allows the default 4 KiB table when using the
243         // default 64 arraySizeHint.
244         maxHeaderTableSize = Math.min(maxHeaderTableSize, nameEntries.length * 64);
245         if (this.maxHeaderTableSize == maxHeaderTableSize) {
246             return;
247         }
248         this.maxHeaderTableSize = maxHeaderTableSize;
249         ensureCapacity(0);
250         // Casting to integer is safe as we verified the maxHeaderTableSize is a valid unsigned int.
251         encodeInteger(out, 0x20, 5, maxHeaderTableSize);
252     }
253 
254     /**
255      * Return the maximum table size.
256      */
257     public long getMaxHeaderTableSize() {
258         return maxHeaderTableSize;
259     }
260 
261     public void setMaxHeaderListSize(long maxHeaderListSize) throws Http2Exception {
262         if (maxHeaderListSize < MIN_HEADER_LIST_SIZE || maxHeaderListSize > MAX_HEADER_LIST_SIZE) {
263             throw connectionError(PROTOCOL_ERROR, "Header List Size must be >= %d and <= %d but was %d",
264               MIN_HEADER_LIST_SIZE, MAX_HEADER_LIST_SIZE, maxHeaderListSize);
265         }
266         this.maxHeaderListSize = maxHeaderListSize;
267     }
268 
269     public long getMaxHeaderListSize() {
270         return maxHeaderListSize;
271     }
272 
273     /**
274      * Encode integer according to <a href="https://tools.ietf.org/html/rfc7541#section-5.1">Section 5.1</a>.
275      */
276     private static void encodeInteger(ByteBuf out, int mask, int n, int i) {
277         encodeInteger(out, mask, n, (long) i);
278     }
279 
280     /**
281      * Encode integer according to <a href="https://tools.ietf.org/html/rfc7541#section-5.1">Section 5.1</a>.
282      */
283     private static void encodeInteger(ByteBuf out, int mask, int n, long i) {
284         assert n >= 0 && n <= 8 : "N: " + n;
285         int nbits = 0xFF >>> 8 - n;
286         if (i < nbits) {
287             out.writeByte((int) (mask | i));
288         } else {
289             out.writeByte(mask | nbits);
290             long length = i - nbits;
291             for (; (length & ~0x7F) != 0; length >>>= 7) {
292                 out.writeByte((int) (length & 0x7F | 0x80));
293             }
294             out.writeByte((int) length);
295         }
296     }
297 
298     /**
299      * Encode string literal according to Section 5.2.
300      */
301     private void encodeStringLiteral(ByteBuf out, CharSequence string) {
302         int huffmanLength;
303         if (string.length() >= huffCodeThreshold
304           && (huffmanLength = hpackHuffmanEncoder.getEncodedLength(string)) < string.length()) {
305             encodeInteger(out, 0x80, 7, huffmanLength);
306             hpackHuffmanEncoder.encode(out, string);
307         } else {
308             encodeInteger(out, 0x00, 7, string.length());
309             if (string instanceof AsciiString) {
310                 // Fast-path
311                 AsciiString asciiString = (AsciiString) string;
312                 out.writeBytes(asciiString.array(), asciiString.arrayOffset(), asciiString.length());
313             } else {
314                 // Only ASCII is allowed in http2 headers, so it is fine to use this.
315                 // https://tools.ietf.org/html/rfc7540#section-8.1.2
316                 out.writeCharSequence(string, CharsetUtil.ISO_8859_1);
317             }
318         }
319     }
320 
321     /**
322      * Encode literal header field according to Section 6.2.
323      */
324     private void encodeLiteral(ByteBuf out, CharSequence name, CharSequence value, IndexType indexType,
325                                int nameIndex) {
326         boolean nameIndexValid = nameIndex != NOT_FOUND;
327         switch (indexType) {
328             case INCREMENTAL:
329                 encodeInteger(out, 0x40, 6, nameIndexValid ? nameIndex : 0);
330                 break;
331             case NONE:
332                 encodeInteger(out, 0x00, 4, nameIndexValid ? nameIndex : 0);
333                 break;
334             case NEVER:
335                 encodeInteger(out, 0x10, 4, nameIndexValid ? nameIndex : 0);
336                 break;
337             default:
338                 throw new Error("Unexpected index type: " + indexType);
339         }
340         if (!nameIndexValid) {
341             encodeStringLiteral(out, name);
342         }
343         encodeStringLiteral(out, value);
344     }
345 
346     private int getNameIndex(CharSequence name) {
347         int index = HpackStaticTable.getIndex(name);
348         if (index != HpackStaticTable.NOT_FOUND) {
349             return index;
350         }
351         NameEntry e = getEntry(name, AsciiString.hashCode(name));
352         return e == null ? NOT_FOUND : getIndexPlusOffset(e.counter);
353     }
354 
355     /**
356      * Ensure that the dynamic table has enough room to hold 'headerSize' more bytes. Removes the
357      * oldest entry from the dynamic table until sufficient space is available.
358      */
359     private void ensureCapacity(long headerSize) {
360         while (maxHeaderTableSize - size < headerSize) {
361             remove();
362         }
363     }
364 
365     /**
366      * Return the number of header fields in the dynamic table. Exposed for testing.
367      */
368     int length() {
369         return isEmpty() ? 0 : getIndex(head.after.counter);
370     }
371 
372     /**
373      * Return the size of the dynamic table. Exposed for testing.
374      */
375     long size() {
376         return size;
377     }
378 
379     /**
380      * Return the header field at the given index. Exposed for testing.
381      */
382     HpackHeaderField getHeaderField(int index) {
383         NameValueEntry entry = head;
384         while (index++ < length()) {
385             entry = entry.after;
386         }
387         return entry;
388     }
389 
390     /**
391      * Returns the header entry with the lowest index value for the header field. Returns null if
392      * header field is not in the dynamic table.
393      */
394     private NameValueEntry getEntryInsensitive(CharSequence name, int nameHash, CharSequence value, int valueHash) {
395         int h = hash(nameHash, valueHash);
396         for (NameValueEntry e = nameValueEntries[bucket(h)]; e != null; e = e.next) {
397             if (e.hash == h && equalsVariableTime(value, e.value) && equalsVariableTime(name, e.name)) {
398                 return e;
399             }
400         }
401         return null;
402     }
403 
404     /**
405      * Returns the lowest index value for the header field name in the dynamic table. Returns -1 if
406      * the header field name is not in the dynamic table.
407      */
408     private NameEntry getEntry(CharSequence name, int nameHash) {
409         for (NameEntry e = nameEntries[bucket(nameHash)]; e != null; e = e.next) {
410             if (e.hash == nameHash && equalsConstantTime(name, e.name) != 0) {
411                 return e;
412             }
413         }
414         return null;
415     }
416 
417     private int getIndexPlusOffset(int counter) {
418         return getIndex(counter) + HpackStaticTable.length;
419     }
420 
421     /**
422      * Compute the index into the dynamic table given the counter in the header entry.
423      */
424     private int getIndex(int counter) {
425         return counter - latestCounter() + 1;
426     }
427 
428     private int latestCounter() {
429         return latest.counter;
430     }
431 
432     private void addNameEntry(CharSequence name, int nameHash, int nextCounter) {
433         int bucket = bucket(nameHash);
434         nameEntries[bucket] = new NameEntry(nameHash, name, nextCounter, nameEntries[bucket]);
435     }
436 
437     private void addNameValueEntry(CharSequence name, CharSequence value,
438                                    int nameHash, int valueHash, int nextCounter) {
439         int hash = hash(nameHash, valueHash);
440         int bucket = bucket(hash);
441         NameValueEntry e = new NameValueEntry(hash, name, value, nextCounter, nameValueEntries[bucket]);
442         nameValueEntries[bucket] = e;
443         latest.after = e;
444         latest = e;
445     }
446 
447     /**
448      * Remove the oldest header field from the dynamic table.
449      */
450     private void remove() {
451         NameValueEntry eldest = head.after;
452         removeNameValueEntry(eldest);
453         removeNameEntryMatchingCounter(eldest.name, eldest.counter);
454         head.after = eldest.after;
455         eldest.unlink();
456         size -= eldest.size();
457         if (isEmpty()) {
458             latest = head;
459         }
460     }
461 
462     private boolean isEmpty() {
463         return size == 0;
464     }
465 
466     private void removeNameValueEntry(NameValueEntry eldest) {
467         int bucket = bucket(eldest.hash);
468         NameValueEntry e = nameValueEntries[bucket];
469         if (e == eldest) {
470             nameValueEntries[bucket] = eldest.next;
471         } else {
472             while (e.next != eldest) {
473                 e = e.next;
474             }
475             e.next = eldest.next;
476         }
477     }
478 
479     private void removeNameEntryMatchingCounter(CharSequence name, int counter) {
480         int hash = AsciiString.hashCode(name);
481         int bucket = bucket(hash);
482         NameEntry e = nameEntries[bucket];
483         if (e == null) {
484             return;
485         }
486         if (counter == e.counter) {
487             nameEntries[bucket] = e.next;
488             e.unlink();
489         } else {
490             NameEntry prev = e;
491             e = e.next;
492             while (e != null) {
493                 if (counter == e.counter) {
494                     prev.next = e.next;
495                     e.unlink();
496                     break;
497                 }
498                 prev = e;
499                 e = e.next;
500             }
501         }
502     }
503 
504     /**
505      * Returns the bucket of the hash table for the hash code h.
506      */
507     private int bucket(int h) {
508         return h & hashMask;
509     }
510 
511     private static int hash(int nameHash, int valueHash) {
512         return 31 * nameHash + valueHash;
513     }
514 
515     private static final class NameEntry {
516         NameEntry next;
517 
518         final CharSequence name;
519 
520         final int hash;
521 
522         // This is used to compute the index in the dynamic table.
523         int counter;
524 
525         NameEntry(int hash, CharSequence name, int counter, NameEntry next) {
526             this.hash = hash;
527             this.name = name;
528             this.counter = counter;
529             this.next = next;
530         }
531 
532         void unlink() {
533             next = null; // null references to prevent nepotism in generational GC.
534         }
535     }
536 
537     private static final class NameValueEntry extends HpackHeaderField {
538         // This field comprises the linked list used for implementing the eviction policy.
539         NameValueEntry after;
540 
541         NameValueEntry next;
542 
543         // hash of both name and value
544         final int hash;
545 
546         // This is used to compute the index in the dynamic table.
547         final int counter;
548 
549         NameValueEntry(int hash, CharSequence name, CharSequence value, int counter, NameValueEntry next) {
550             super(name, value);
551             this.next = next;
552             this.hash = hash;
553             this.counter = counter;
554         }
555 
556         void unlink() {
557             after = null; // null references to prevent nepotism in generational GC.
558             next = null;
559         }
560     }
561 }