[PATCH] improved MTU calculation for GlobalProtect ESP

[Daniel Lenski]

On Wed, Aug 16, 2017 at 8:03 PM, Daniel Lenski wrote:

> I believe the correct [ESP padding] algorithm is actually as follows:
>
> * From payload MTU, add 2 footer bytes, round *up* to a multiple of
>   the blocksize. Add the size of the MAC, IV, and other headers. That's
>   the size of the packet on the wire.
> * From wire packet MTU, subtract headers and MAC and IV, round *down*
>   to a multiple of blocksize, subtract TWO footer bytes, and that's the
>   largest payload you can carry.

Signed-off-by: Daniel Lenski <dlenski at gmail.com>
---
 gpst.c | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/gpst.c b/gpst.c
index 77d0d7b..87cfbc9 100644
--- a/gpst.c
+++ b/gpst.c
@@ -267,14 +267,17 @@ out:
 }
 
 
-/* XXX: Look at set_esp_algo() and tell me again what the biggest supported IV is? */
-#define ESP_OVERHEAD (4 /* SPI */ + 4 /* sequence number */ + \
-                      1 /* pad length */ + 1 /* next header */ + \
-                      16 /* max padding */ )
+#define ESP_HEADER_SIZE (4 /* SPI */ + 4 /* sequence number */)
+#define ESP_FOOTER_SIZE (1 /* pad length */ + 1 /* next header */)
 #define UDP_HEADER_SIZE 8
 #define IPV4_HEADER_SIZE 20
 #define IPV6_HEADER_SIZE 40
 
+/* Based on cstp.c's calculate_mtu().
+ *
+ * With HTTPS tunnel, there are 21 bytes of overhead beyond the
+ * TCP MSS: 5 bytes for TLS and 16 for GPST.
+ */
 static int calculate_mtu(struct openconnect_info *vpninfo)
 {
 	int mtu = vpninfo->reqmtu, base_mtu = vpninfo->basemtu;
@@ -296,9 +299,9 @@ static int calculate_mtu(struct openconnect_info *vpninfo)
 
 			if (!base_mtu) {
 				if (ti.tcpi_rcv_mss < ti.tcpi_snd_mss)
-					base_mtu = ti.tcpi_rcv_mss - 13;
+					base_mtu = ti.tcpi_rcv_mss - 21;
 				else
-					base_mtu = ti.tcpi_snd_mss - 13;
+					base_mtu = ti.tcpi_snd_mss - 21;
 			}
 		}
 	}
@@ -310,7 +313,7 @@ static int calculate_mtu(struct openconnect_info *vpninfo)
 		if (!getsockopt(vpninfo->ssl_fd, IPPROTO_TCP, TCP_MAXSEG,
 				&mss, &mss_size)) {
 			vpn_progress(vpninfo, PRG_DEBUG, _("TCP_MAXSEG %d\n"), mss);
-			base_mtu = mss - 13;
+			base_mtu = mss - 21;
 		}
 	}
 #endif
@@ -323,14 +326,18 @@ static int calculate_mtu(struct openconnect_info *vpninfo)
 		base_mtu = 1280;
 
 	if (!mtu) {
-		/* remove IP/UDP and ESP overhead from base MTU to calculate tunnel MTU */
-		mtu = ( base_mtu - UDP_HEADER_SIZE - ESP_OVERHEAD
-		        - (vpninfo->hmac_key_len ? : 20) /* biggest supported MAC (SHA1) */
+		/* remove ESP, UDP, IP headers from base (wire) MTU */
+		mtu = ( base_mtu - UDP_HEADER_SIZE - ESP_HEADER_SIZE
+		        - 12 /* both supported algos (SHA1 and MD5) have 96-bit MAC lengths (RFC2403 and RFC2404) */
 		        - (vpninfo->enc_key_len ? : 32) /* biggest supported IV (AES-256) */ );
 		if (vpninfo->peer_addr->sa_family == AF_INET6)
 			mtu -= IPV6_HEADER_SIZE;
 		else
 			mtu -= IPV4_HEADER_SIZE;
+		/* round down to a multiple of blocksize */
+		mtu -= mtu % (vpninfo->enc_key_len ? : 32);
+		/* subtract ESP footer, which is included in the payload before padding to the blocksize */
+		mtu -= ESP_FOOTER_SIZE;
 	}
 	return mtu;
 }
-- 
2.7.4