don't send server_name when literal IP

src/libraries/System.Net.Security/src/System.Net.Security.csproj

@@ -56,6 +56,13 @@
     <Compile Include="System\Security\Authentication\ExtendedProtection\PolicyEnforcement.cs" />
     <Compile Include="System\Security\Authentication\ExtendedProtection\ProtectionScenario.cs" />
     <Compile Include="System\Security\Authentication\ExtendedProtection\ServiceNameCollection.cs" />
+    <!-- IP parser -->
+    <Compile Include="$(CommonPath)System\Net\IPv4AddressHelper.Common.cs"
+             Link="System\Net\IPv4AddressHelper.Common.cs" />
+    <Compile Include="$(CommonPath)System\Net\IPv6AddressHelper.Common.cs"
+             Link="System\Net\IPv6AddressHelper.Common.cs" />
+    <Compile Include="$(CommonPath)System\Net\IPAddressParserStatics.cs"
+             Link="Common\System\Net\IPAddressParserStatics.cs" />
     <!-- Common sources -->
     <Compile Include="$(CommonPath)DisableRuntimeMarshalling.cs"
              Link="Common\DisableRuntimeMarshalling.cs" />

src/libraries/System.Net.Security/src/System/Net/Security/SslAuthenticationOptions.cs

@@ -3,13 +3,49 @@
 
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Runtime.InteropServices;
 using System.Security.Authentication;
 using System.Security.Cryptography.X509Certificates;
 
 namespace System.Net.Security
 {
     internal sealed class SslAuthenticationOptions
     {
+        // Simplified version of IPAddressParser.Parse to avoid allocations and dependencies.
+        // It purposely ignores scopeId as we don't really use so we do not need to map it to actual interface id.
+        private static unsafe bool IsValidAddress(ReadOnlySpan<char> ipSpan)
+        {
+            int end = ipSpan.Length;
+
+            if (ipSpan.Contains(':'))
+            {
+                // The address is parsed as IPv6 if and only if it contains a colon. This is valid because
+                // we don't support/parse a port specification at the end of an IPv4 address.
+                Span<ushort> numbers = stackalloc ushort[IPAddressParserStatics.IPv6AddressShorts];
+
+                fixed (char* ipStringPtr = &MemoryMarshal.GetReference(ipSpan))
+                {
+                    return IPv6AddressHelper.IsValidStrict(ipStringPtr, 0, ref end);
+                }
+            }
+            else if (char.IsDigit(ipSpan[0]))
+            {
+                long tmpAddr;
+
+                fixed (char* ipStringPtr = &MemoryMarshal.GetReference(ipSpan))
+                {
+                    tmpAddr = IPv4AddressHelper.ParseNonCanonical(ipStringPtr, 0, ref end, notImplicitFile: true);
+                }
+
+                if (tmpAddr != IPv4AddressHelper.Invalid && end == ipSpan.Length)
+                {
+                    return true;
+                }
+            }
+
+            return false;
+        }
+
         internal SslAuthenticationOptions()
         {
             TargetHost = string.Empty;
@@ -47,10 +83,16 @@ internal void UpdateOptions(SslClientAuthenticationOptions sslClientAuthenticati
             IsServer = false;
             RemoteCertRequired = true;
             CertificateContext = sslClientAuthenticationOptions.ClientCertificateContext;
-            // RFC 6066 section 3 says to exclude trailing dot from fully qualified DNS hostname
-            if (sslClientAuthenticationOptions.TargetHost != null)
+            if (!string.IsNullOrEmpty(sslClientAuthenticationOptions.TargetHost))
             {
+                // RFC 6066 section 3 says to exclude trailing dot from fully qualified DNS hostname
                 TargetHost = sslClientAuthenticationOptions.TargetHost.TrimEnd('.');
+
+                // RFC 6066 forbids IP literals
+                if (IsValidAddress(TargetHost))
+                {
+                    TargetHost = string.Empty;
+                }
             }
 
             // Client specific options.

src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamSniTest.cs

@@ -172,6 +172,31 @@ await server.AuthenticateAsServerAsync(options, cts.Token)
                 return true;
             });
         }
+        [Theory]
+        [InlineData("127.0.0.1")]
+        [InlineData("::1")]
+        [InlineData("2001:11:22::1")]
+        [InlineData("fe80::9c3a:b64d:6249:1de8%2")]
+        [InlineData("fe80::9c3a:b64d:6249:1de8")]
+        public async Task SslStream_IpLiteral_NotSend(string target)
+        {
+            (SslStream client, SslStream server) = TestHelper.GetConnectedSslStreams();
+            SslClientAuthenticationOptions clientOptions = new SslClientAuthenticationOptions()
+            {
+                    TargetHost = target,
+                    RemoteCertificateValidationCallback = (sender, certificate, chain, sslPolicyErrors) => true,
+            };
+            SslServerAuthenticationOptions serverOptions = new SslServerAuthenticationOptions()
+            {
+                ServerCertificate = Configuration.Certificates.GetServerCertificate(),
+            };
+
+            await TestConfiguration.WhenAllOrAnyFailedWithTimeout(
+                        client.AuthenticateAsClientAsync(clientOptions, default),
+                        server.AuthenticateAsServerAsync(serverOptions, default));
+
+            Assert.Equal(string.Empty, server.TargetHostName);
+        }
 
         private static Func<Task> WithAggregateExceptionUnwrapping(Func<Task> a)
         {

src/libraries/System.Net.Security/tests/UnitTests/System.Net.Security.Unit.Tests.csproj

@@ -84,6 +84,13 @@
              Link="Common\System\Threading\Tasks\TaskToApm.cs" />
     <Compile Include="$(CommonPath)System\Obsoletions.cs" 
              Link="Common\System\Obsoletions.cs" />
+    <!-- IP parser -->
+    <Compile Include="$(CommonPath)System\Net\IPv4AddressHelper.Common.cs"
+             Link="System\Net\IPv4AddressHelper.Common.cs" />
+    <Compile Include="$(CommonPath)System\Net\IPv6AddressHelper.Common.cs"
+             Link="System\Net\IPv6AddressHelper.Common.cs" />
+    <Compile Include="$(CommonPath)System\Net\IPAddressParserStatics.cs"
+             Link="Common\System\Net\IPAddressParserStatics.cs" />
     <!-- Logging -->
     <Compile Include="$(CommonPath)System\Net\Logging\NetEventSource.Common.cs"
              Link="ProductionCode\Common\System\Net\Logging\NetEventSource.Common.cs" />