Output/TLS: Allow logging of client/server handshake parameters - V4

doc/userguide/output/eve/eve-json-format.rst

@@ -1041,12 +1041,17 @@ If extended logging is enabled the following fields are also included:
 * "client_alpns": array of strings with ALPN values
 * "server_alpns": array of strings with ALPN values
 
-JA3 and JA4 must be enabled in the Suricata config file (set 'app-layer.protocols.tls.ja3-fingerprints'/'app-layer.protocols.tls.ja4-fingerprints' to 'yes').
+JA3 and JA4 fingerprints can be enabled in the Suricata config file (set 'app-layer.protocols.tls.ja3-fingerprints'/'app-layer.protocols.tls.ja4-fingerprints' to 'yes').
 
 In addition to this, custom logging also allows the following fields:
 
 * "certificate": The TLS certificate base64 encoded
 * "chain": The entire TLS certificate chain base64 encoded
+* "client_handshake": structure containing "version", "ciphers" ([u16]), "exts" ([u16]), "sig_algs" ([u16]),
+  for client hello supported cipher suites, extensions, and signature algorithms,
+  respectively, in the order that they're mentioned (ie. unsorted)
+* "server_handshake": structure containing "version", "chosen cipher", "exts" ([u16]), for server hello
+  in the order that they're mentioned (ie. unsorted)
 
 Examples
 ~~~~~~~~

etc/schema.json

@@ -6729,6 +6729,54 @@
                         "type": "string"
                     }
                 },
+		"client_handshake": {
+		    "type": "object",
+		    "properties": {
+			"version": {
+			    "type": "string"
+			},
+			"ciphers": {
+			    "description": "TLS client cipher(s)",
+			    "type": "array",
+			    "items": {
+				"type": "integer"
+			    }
+			},
+			"exts": {
+			    "description": "TLS client extension(s)",
+			    "type": "array",
+			    "items": {
+				"type": "integer"
+			    }
+			},
+			"sig_algs": {
+			    "description": "TLS client signature algorithm(s)",
+			    "type": "array",
+			    "items": {
+				"type": "integer"
+			    }
+			}
+		    }
+		},
+		"server_handshake": {
+		    "type": "object",
+		    "properties": {
+			"version": {
+			    "type": "string"
+			},
+			"cipher": {
+			    "description": "TLS server's chosen cipher",
+			    "type": "integer"
+			},
+			"exts": {
+			    "description": "TLS server extension(s)",
+			    "type": "array",
+			    "items": {
+				"type": "integer"
+			    }
+			}
+		    }
+		},
                 "server_alpns": {
                     "description": "TLS server ALPN field(s)",
                     "type": "array",

rust/src/ja4.rs

@@ -30,6 +30,8 @@ use tls_parser::{TlsCipherSuiteID, TlsExtensionType, TlsVersion};
 #[cfg(feature = "ja4")]
 use crate::jsonbuilder::HEX;
 
+pub const JA4_HEX_LEN: usize = 36;
+
 #[derive(Debug, PartialEq)]
 pub struct JA4 {
     tls_version: Option<TlsVersion>,
@@ -39,10 +41,6 @@ pub struct JA4 {
     domain: bool,
     alpn: [char; 2],
     quic: bool,
-    // Some extensions contribute to the total count component of the
-    // fingerprint, yet are not to be included in the SHA256 hash component.
-    // Let's track the count separately.
-    nof_exts: u16,
 }
 
 impl Default for JA4 {
@@ -65,7 +63,6 @@ impl JA4 {
             domain: false,
             alpn: ['0', '0'],
             quic: false,
-            nof_exts: 0,
         }
     }
     pub fn set_quic(&mut self) {}
@@ -113,7 +110,6 @@ impl JA4 {
             domain: false,
             alpn: ['0', '0'],
             quic: false,
-            nof_exts: 0,
         }
     }
 
@@ -170,14 +166,10 @@ impl JA4 {
         if JA4::is_grease(u16::from(ext)) {
             return;
         }
-        if ext != TlsExtensionType::ApplicationLayerProtocolNegotiation
-            && ext != TlsExtensionType::ServerName
-        {
-            self.extensions.push(ext);
-        } else if ext == TlsExtensionType::ServerName {
+        if ext == TlsExtensionType::ServerName {
             self.domain = true;
         }
-        self.nof_exts += 1;
+        self.extensions.push(ext);
     }
 
     pub fn add_signature_algorithm(&mut self, sigalgo: u16) {
@@ -188,14 +180,25 @@ impl JA4 {
     }
 
     pub fn get_hash(&self) -> String {
+        // All non-GREASE extensions are stored to produce a more verbose, complete output
+        // of extensions but we need to omit ALPN & SNI extensions from the JA4_a hash.
+        let mut exts = self
+            .extensions
+            .iter()
+            .filter(|&ext| {
+                *ext != TlsExtensionType::ApplicationLayerProtocolNegotiation
+                    && *ext != TlsExtensionType::ServerName
+            })
+            .collect::<Vec<&TlsExtensionType>>();
+
         // Calculate JA4_a
         let ja4_a = format!(
             "{proto}{version}{sni}{nof_c:02}{nof_e:02}{al1}{al2}",
             proto = if self.quic { "q" } else { "t" },
             version = JA4::version_to_ja4code(self.tls_version),
             sni = if self.domain { "d" } else { "i" },
             nof_c = min(99, self.ciphersuites.len()),
-            nof_e = min(99, self.nof_exts),
+            nof_e = min(99, self.extensions.len()),
             al1 = self.alpn[0],
             al2 = self.alpn[1]
         );
@@ -214,11 +217,10 @@ impl JA4 {
         ja4_b.truncate(12);
 
         // Calculate JA4_c
-        let mut sorted_exts = self.extensions.to_vec();
-        sorted_exts.sort_by(|a, b| u16::from(*a).cmp(&u16::from(*b)));
-        let sorted_extstrings: Vec<String> = sorted_exts
-            .iter()
-            .map(|v| format!("{:04x}", u16::from(*v)))
+        exts.sort_by(|&a, &b| u16::from(*a).cmp(&u16::from(*b)));
+        let sorted_extstrings: Vec<String> = exts
+            .into_iter()
+            .map(|&v| format!("{:04x}", u16::from(v)))
             .collect();
         let ja4_c1_raw = sorted_extstrings.join(",");
         let unsorted_sigalgostrings: Vec<String> = self
@@ -269,9 +271,37 @@ pub unsafe extern "C" fn SCJA4SetALPN(j: &mut JA4, proto: *const c_char, len: u1
 }
 
 #[no_mangle]
-pub unsafe extern "C" fn SCJA4GetHash(j: &mut JA4, out: &mut [u8; 36]) {
+pub unsafe extern "C" fn SCJA4GetVersion(j: &JA4) -> u16 {
+    u16::from(j.tls_version.unwrap_or(TlsVersion(0)))
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn SCJA4GetHash(j: &mut JA4, out: &mut [u8; JA4_HEX_LEN]) {
     let hash = j.get_hash();
-    out[0..36].copy_from_slice(hash.as_bytes());
+    out[0..JA4_HEX_LEN].copy_from_slice(hash.as_bytes());
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn SCJA4GetCiphers(j: &mut JA4, out: *mut usize) -> *const u16 {
+    *out = j.ciphersuites.len();
+    j.ciphersuites.as_ptr() as *const u16
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn SCJA4GetFirstCipher(j: &mut JA4) -> u16 {
+    j.ciphersuites.first().map(|&v| *v).unwrap_or(0)
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn SCJA4GetExtensions(j: &mut JA4, out: *mut usize) -> *const u16 {
+    *out = j.extensions.len();
+    j.extensions.as_ptr() as *const u16
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn SCJA4GetSigAlgs(j: &mut JA4, out: *mut usize) -> *const u16 {
+    *out = j.signature_algorithms.len();
+    j.signature_algorithms.as_ptr()
 }
 
 #[no_mangle]

src/Makefile.am

@@ -507,7 +507,6 @@ noinst_HEADERS = \
 	util-ioctl.h \
 	util-ip.h \
 	util-ja3.h \
-	util-ja4.h \
 	util-landlock.h \
 	util-logopenfile.h \
 	util-log-redis.h \

src/app-layer-ssl.c

@@ -695,7 +695,8 @@ static inline int TLSDecodeHSHelloVersion(SSLState *ssl_state,
     ssl_state->curr_connp->version = version;
 
     if (ssl_state->curr_connp->ja4 != NULL &&
-            ssl_state->current_flags & SSL_AL_FLAG_STATE_CLIENT_HELLO) {
+            ssl_state->current_flags &
+                    (SSL_AL_FLAG_STATE_CLIENT_HELLO | SSL_AL_FLAG_STATE_SERVER_HELLO)) {
         SCJA4SetTLSVersion(ssl_state->curr_connp->ja4, version);
     }
 
@@ -845,7 +846,7 @@ static inline int TLSDecodeHSHelloCipherSuites(SSLState *ssl_state,
     const bool enable_ja3 =
             SC_ATOMIC_GET(ssl_config.enable_ja3) && ssl_state->curr_connp->ja3_hash == NULL;
 
-    if (enable_ja3 || SC_ATOMIC_GET(ssl_config.enable_ja4)) {
+    if (enable_ja3 || ssl_state->curr_connp->ja4 != NULL) {
         JA3Buffer *ja3_cipher_suites = NULL;
 
         if (enable_ja3) {
@@ -870,7 +871,8 @@ static inline int TLSDecodeHSHelloCipherSuites(SSLState *ssl_state,
 
             if (TLSDecodeValueIsGREASE(cipher_suite) != 1) {
                 if (ssl_state->curr_connp->ja4 != NULL &&
-                        ssl_state->current_flags & SSL_AL_FLAG_STATE_CLIENT_HELLO) {
+                        ssl_state->current_flags &
+                                (SSL_AL_FLAG_STATE_CLIENT_HELLO | SSL_AL_FLAG_STATE_SERVER_HELLO)) {
                     SCJA4AddCipher(ssl_state->curr_connp->ja4, cipher_suite);
                 }
                 if (enable_ja3) {
@@ -1293,7 +1295,8 @@ static inline int TLSDecodeHSHelloExtensionALPN(
 
         /* Only record the first value for JA4 */
         if (ssl_state->curr_connp->ja4 != NULL &&
-                ssl_state->current_flags & SSL_AL_FLAG_STATE_CLIENT_HELLO) {
+                ssl_state->current_flags &
+                        (SSL_AL_FLAG_STATE_CLIENT_HELLO | SSL_AL_FLAG_STATE_SERVER_HELLO)) {
             if (alpn_processed_len == 1) {
                 SCJA4SetALPN(ssl_state->curr_connp->ja4, (const char *)input, protolen);
             }
@@ -1492,7 +1495,8 @@ static inline int TLSDecodeHSHelloExtensions(SSLState *ssl_state,
         }
 
         if (ssl_state->curr_connp->ja4 != NULL &&
-                ssl_state->current_flags & SSL_AL_FLAG_STATE_CLIENT_HELLO) {
+                ssl_state->current_flags &
+                        (SSL_AL_FLAG_STATE_CLIENT_HELLO | SSL_AL_FLAG_STATE_SERVER_HELLO)) {
             if (TLSDecodeValueIsGREASE(ext_type) != 1) {
                 SCJA4AddExtension(ssl_state->curr_connp->ja4, ext_type);
             }
@@ -1546,11 +1550,11 @@ static int TLSDecodeHandshakeHello(SSLState *ssl_state,
     int ret;
     uint32_t parsed = 0;
 
-    /* Ensure that we have a JA4 state defined by now if we have JA4 enabled,
-       we are in a client hello and we don't have such a state yet (to avoid
-       leaking memory in case this function is entered more than once). */
-    if (SC_ATOMIC_GET(ssl_config.enable_ja4) &&
-            ssl_state->current_flags & SSL_AL_FLAG_STATE_CLIENT_HELLO &&
+    /* Ensure that we have a JA4 state defined by now, we are in a client/server hello
+       and we don't have such a state yet (to avoid leaking memory in case this function
+       is entered more than once). */
+    if (ssl_state->current_flags &
+                    (SSL_AL_FLAG_STATE_CLIENT_HELLO | SSL_AL_FLAG_STATE_SERVER_HELLO) &&
             ssl_state->curr_connp->ja4 == NULL) {
         ssl_state->curr_connp->ja4 = SCJA4New();
     }
@@ -2894,6 +2898,8 @@ static void SSLStateFree(void *p)
 
     if (ssl_state->client_connp.ja4)
         SCJA4Free(ssl_state->client_connp.ja4);
+    if (ssl_state->server_connp.ja4)
+        SCJA4Free(ssl_state->server_connp.ja4);
     if (ssl_state->client_connp.ja3_str)
         Ja3BufferFree(&ssl_state->client_connp.ja3_str);
     if (ssl_state->client_connp.ja3_hash)

src/detect-ja4-hash.c

@@ -34,8 +34,6 @@
 #include "detect-engine-prefilter.h"
 #include "detect-ja4-hash.h"
 
-#include "util-ja4.h"
-
 #include "app-layer-ssl.h"
 
 #ifndef HAVE_JA4