Fix unbounded recursive handling of SSL/GSS in ProcessStartupPacket()

The handling of SSL and GSS negotiation messages in
ProcessStartupPacket() could cause a recursion of the backend,
ultimately crashing the server as the negotiation attempts were not
tracked across multiple calls processing startup packets.

A malicious client could therefore alternate rejected SSL and GSS
requests indefinitely, each adding a stack frame, until the backend
crashed with a stack overflow, taking down a server.

This commit addresses this issue by modifying ProcessStartupPacket() so
as processed negotiation attempts are tracked, preventing infinite
recursive attempts.  A TAP test is added to check this problem, where
multiple SSL and GSS negotiated attempts are stacked.

Reported-by: Calif.io in collaboration with Claude and Anthropic
Research
Author: Michael Paquier <michael@paquier.xyz>
Reviewed-by: Daniel Gustafsson <daniel@yesql.se>
Security: CVE-2026-6479
Backpatch-through: 14

src/backend/postmaster/postmaster.c

@@ -2013,6 +2013,7 @@ ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done)
 	ProtocolVersion proto;
 	MemoryContext oldcontext;
 
+retry:
 	pq_startmsgread();
 
 	/*
@@ -2142,7 +2143,16 @@ retry1:
 		 * another SSL negotiation request, and a GSS request should only
 		 * follow if SSL was rejected (client may negotiate in either order)
 		 */
-		return ProcessStartupPacket(port, true, SSLok == 'S');
+		ssl_done = true;
+		if (SSLok == 'S')
+		{
+			/*
+			 * We are done with SSL and negotiated correctly, so consider the
+			 * same for GSS.
+			 */
+			gss_done = true;
+		}
+		goto retry;
 	}
 	else if (proto == NEGOTIATE_GSS_CODE && !gss_done)
 	{
@@ -2186,7 +2196,16 @@ retry1:
 		 * another GSS negotiation request, and an SSL request should only
 		 * follow if GSS was rejected (client may negotiate in either order)
 		 */
-		return ProcessStartupPacket(port, GSSok == 'G', true);
+		gss_done = true;
+		if (GSSok == 'G')
+		{
+			/*
+			 * We are done with GSS and negotiated correctly, so consider the
+			 * same for SSL.
+			 */
+			ssl_done = true;
+		}
+		goto retry;
 	}
 
 	/* Could add additional special packet types here */

src/test/Makefile

@@ -12,7 +12,7 @@ subdir = src/test
 top_builddir = ../..
 include $(top_builddir)/src/Makefile.global
 
-SUBDIRS = perl regress isolation modules authentication recovery subscription
+SUBDIRS = perl postmaster regress isolation modules authentication recovery subscription
 
 ifeq ($(with_icu),yes)
 SUBDIRS += icu

src/test/postmaster/.gitignore

@@ -0,0 +1,2 @@
+# Generated by test suite
+/tmp_check/

src/test/postmaster/Makefile

@@ -0,0 +1,23 @@
+#-------------------------------------------------------------------------
+#
+# Makefile for src/test/postmaster
+#
+# Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
+# Portions Copyright (c) 1994, Regents of the University of California
+#
+# src/test/postmaster/Makefile
+#
+#-------------------------------------------------------------------------
+
+subdir = src/test/postmaster
+top_builddir = ../../..
+include $(top_builddir)/src/Makefile.global
+
+check:
+	$(prove_check)
+
+installcheck:
+	$(prove_installcheck)
+
+clean distclean maintainer-clean:
+	rm -rf tmp_check

src/test/postmaster/README

@@ -0,0 +1,27 @@
+src/test/postmaster/README
+
+Regression tests for postmaster
+===============================
+
+This directory contains a test suite for postmaster's handling of
+connections, connection limits, and startup/shutdown sequence.
+
+
+Running the tests
+=================
+
+NOTE: You must have given the --enable-tap-tests argument to configure.
+
+Run
+    make check
+or
+    make installcheck
+You can use "make installcheck" if you previously did "make install".
+In that case, the code in the installation tree is tested.  With
+"make check", a temporary installation tree is built from the current
+sources and then tested.
+
+Either way, this test initializes, starts, and stops a test Postgres
+cluster.
+
+See src/test/perl/README for more info about running these tests.

src/test/postmaster/t/004_negotiate.pl

@@ -0,0 +1,81 @@
+# Copyright (c) 2026, PostgreSQL Global Development Group
+
+# Test the negotiation of combined SSL and GSS requests.  This test
+# relies on both SSL and GSS requests to be rejected first, followed
+# by more requests.
+
+use strict;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+use Time::HiRes qw(usleep);
+
+my $node = PostgreSQL::Test::Cluster->new('main');
+$node->init;
+$node->append_conf('postgresql.conf', "log_min_messages = debug2");
+$node->append_conf('postgresql.conf',
+	"log_connections = 'on'");
+$node->start;
+
+if (!$node->raw_connect_works())
+{
+	plan skip_all => "this test requires working raw_connect()";
+}
+
+my $sock = $node->raw_connect();
+
+# SSLRequest: packet length followed by NEGOTIATE_SSL_CODE.
+my $ssl_request = pack("Nnn", 8, 1234, 5679);
+
+# GSSENCRequest: packet length followed by NEGOTIATE_GSS_CODE.
+my $gss_request = pack("Nnn", 8, 1234, 5680);
+
+# Send SSLRequest, reject or bypass.
+$sock->send($ssl_request);
+my $reply = "";
+$sock->recv($reply, 1);
+if ($reply ne 'N')
+{
+	$sock->close();
+	plan skip_all =>
+	  "server accepted SSL; test requires SSL to be rejected";
+}
+
+# Send GSSENCRequest, reject or bypass test.
+$sock->send($gss_request);
+$reply = "";
+$sock->recv($reply, 1);
+if ($reply ne 'N')
+{
+	$sock->close();
+	plan skip_all =>
+	  "server accepted GSS; test requires GSS to be rejected";
+}
+
+my $log_offset = -s $node->logfile;
+
+# Send a second SSLRequest, now that we know that both SSL and GSS have
+# been rejected for this connection.  We are done with both requests, so
+# extra requests will be rejected and fail with an invalid protocol
+# version, and the connection should be closed by the server.
+$sock->send($ssl_request);
+
+# Try to read a response, there should be nothing, and certainly not an
+# extra 'N' message indicating a rejection.
+$reply = "";
+my $bytes = $sock->recv($reply, 1024);
+isnt($reply, 'N',
+	"server does not re-enter SSL negotiation after SSL+GSS were both tried");
+
+$sock->close();
+$node->wait_for_log(qr/FATAL: .* unsupported frontend protocol 1234.5679/,
+					$log_offset);
+
+# Check extra connection with a simple query.
+my $result = $node->safe_psql('postgres', 'select 1;');
+is($result, '1', 'server able to accept connection');
+
+$node->stop;
+
+done_testing();