From 9e9288a53e51c35532aceee467d2de8724dd700e Mon Sep 17 00:00:00 2001
From: Gareth <email@garethdawson.xyz>
Date: Thu, 22 Jan 2026 10:04:22 +0000
Subject: [PATCH] Tempo: add custom grpc limits (#116013)

* Tempo: add custom grpc limits

* use var from grafana-plugin-sdk-go

* fix tests

* Update pkg/tsdb/tempo/grpc.go

Co-authored-by: Andres Martinez Gotor <andres.martinez@grafana.com>

* Update tests

---------

Co-authored-by: Andres Martinez Gotor <andres.martinez@grafana.com>
---
 .../pluginsintegration/pluginconfig/config.go  |  3 +++
 .../pluginsintegration/pluginconfig/request.go |  2 ++
 .../pluginconfig/request_test.go               | 18 ++++++++++++++++++
 pkg/tsdb/tempo/grpc.go                         | 17 ++++++++++++++++-
 4 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/pkg/services/pluginsintegration/pluginconfig/config.go b/pkg/services/pluginsintegration/pluginconfig/config.go
index 288f204cd12..e283acf03ea 100644
--- a/pkg/services/pluginsintegration/pluginconfig/config.go
+++ b/pkg/services/pluginsintegration/pluginconfig/config.go
@@ -77,6 +77,8 @@ type PluginInstanceCfg struct {
 
 	SigV4AuthEnabled    bool
 	SigV4VerboseLogging bool
+
+	LiveClientQueueMaxSize int
 }
 
 // ProvidePluginInstanceConfig returns a new PluginInstanceCfg.
@@ -124,6 +126,7 @@ func ProvidePluginInstanceConfig(cfg *setting.Cfg, settingProvider setting.Provi
 		ResponseLimit:                       cfg.ResponseLimit,
 		SigV4AuthEnabled:                    cfg.SigV4AuthEnabled,
 		SigV4VerboseLogging:                 cfg.SigV4VerboseLogging,
+		LiveClientQueueMaxSize:              cfg.LiveClientQueueMaxSize,
 	}, nil
 }
 
diff --git a/pkg/services/pluginsintegration/pluginconfig/request.go b/pkg/services/pluginsintegration/pluginconfig/request.go
index a6861c7acec..132e1556632 100644
--- a/pkg/services/pluginsintegration/pluginconfig/request.go
+++ b/pkg/services/pluginsintegration/pluginconfig/request.go
@@ -198,5 +198,7 @@ func (s *RequestConfigProvider) PluginRequestConfig(ctx context.Context, pluginI
 		m[backend.AppClientSecret] = externalService.ClientSecret
 	}
 
+	m[backend.LiveClientQueueMaxSize] = strconv.Itoa(s.cfg.LiveClientQueueMaxSize)
+
 	return m
 }
diff --git a/pkg/services/pluginsintegration/pluginconfig/request_test.go b/pkg/services/pluginsintegration/pluginconfig/request_test.go
index d9720045e01..b42de3980b7 100644
--- a/pkg/services/pluginsintegration/pluginconfig/request_test.go
+++ b/pkg/services/pluginsintegration/pluginconfig/request_test.go
@@ -22,6 +22,7 @@ func TestRequestConfigProvider_PluginRequestConfig_Defaults(t *testing.T) {
 
 	p := NewRequestConfigProvider(pCfg, &fakeSSOSettingsProvider{})
 	require.Equal(t, map[string]string{
+		"GF_LIVE_CLIENT_QUEUE_MAX_SIZE":            "0",
 		"GF_SQL_MAX_OPEN_CONNS_DEFAULT":            "0",
 		"GF_SQL_MAX_IDLE_CONNS_DEFAULT":            "0",
 		"GF_SQL_MAX_CONN_LIFETIME_SECONDS_DEFAULT": "0",
@@ -219,6 +220,7 @@ func TestRequestConfigProvider_PluginRequestConfig_SQL(t *testing.T) {
 
 		p := NewRequestConfigProvider(pCfg, &fakeSSOSettingsProvider{})
 		require.Equal(t, map[string]string{
+			"GF_LIVE_CLIENT_QUEUE_MAX_SIZE":            "0",
 			"GF_SQL_MAX_OPEN_CONNS_DEFAULT":            "0",
 			"GF_SQL_MAX_IDLE_CONNS_DEFAULT":            "0",
 			"GF_SQL_MAX_CONN_LIFETIME_SECONDS_DEFAULT": "0",
@@ -259,6 +261,22 @@ func TestRequestConfigProvider_PluginRequestConfig_concurrentQueryCount(t *testi
 	})
 }
 
+func TestRequestConfigProvider_PluginRequestConfig_liveClientQueueMaxSize(t *testing.T) {
+	t.Run("Sets the live client queue max size only for Tempo", func(t *testing.T) {
+		cfg := setting.NewCfg()
+		cfg.LiveClientQueueMaxSize = 123
+
+		pCfg, err := ProvidePluginInstanceConfig(cfg, setting.ProvideProvider(cfg), featuremgmt.WithFeatures())
+		require.NoError(t, err)
+
+		p := NewRequestConfigProvider(pCfg, &fakeSSOSettingsProvider{})
+
+		require.Subset(t, p.PluginRequestConfig(context.Background(), "tempo", nil), map[string]string{
+			"GF_LIVE_CLIENT_QUEUE_MAX_SIZE": "123",
+		})
+	})
+}
+
 func TestRequestConfigProvider_PluginRequestConfig_azureAuthEnabled(t *testing.T) {
 	t.Run("Uses the configured azureAuthEnabled", func(t *testing.T) {
 		cfg := &PluginInstanceCfg{
diff --git a/pkg/tsdb/tempo/grpc.go b/pkg/tsdb/tempo/grpc.go
index ac1ff74d424..a6e18be958c 100644
--- a/pkg/tsdb/tempo/grpc.go
+++ b/pkg/tsdb/tempo/grpc.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net"
 	"net/url"
+	"strconv"
 	"strings"
 
 	"google.golang.org/grpc/metadata"
@@ -88,7 +89,21 @@ func getDialOpts(ctx context.Context, settings backend.DataSourceInstanceSetting
 
 	var dialOps []grpc.DialOption
 
-	dialOps = append(dialOps, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(100*1024*1024)))
+	// Default max gRPC receive size is 4MB. Tempo responses can exceed this, so we should increase it.
+	// Prefer `GF_LIVE_CLIENT_QUEUE_MAX_SIZE` (set by Grafana for the Tempo plugin) when it's present and valid.
+	const defaultMaxCallRecvMsgSizeBytes = 4 * 1024 * 1024
+	maxCallRecvMsgSizeBytes := defaultMaxCallRecvMsgSizeBytes
+
+	if v := backend.GrafanaConfigFromContext(ctx).Get(backend.LiveClientQueueMaxSize); v != "" {
+		parsed, err := strconv.Atoi(v)
+		if err != nil || parsed <= 0 {
+			logger.Debug("Invalid GF_LIVE_CLIENT_QUEUE_MAX_SIZE; using default gRPC max receive size", "value", v, "default", defaultMaxCallRecvMsgSizeBytes, "error", err)
+		} else {
+			maxCallRecvMsgSizeBytes = parsed
+		}
+	}
+
+	dialOps = append(dialOps, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxCallRecvMsgSizeBytes)))
 	dialOps = append(dialOps, grpc.WithChainStreamInterceptor(CustomHeadersStreamInterceptor(opts)))
 	if settings.BasicAuthEnabled {
 		// If basic authentication is enabled, it uses TLS transport credentials and sets the basic authentication header for each RPC call.