prometheus · Mentigen · Apr 5, 2026 · Apr 9, 2026 · Apr 9, 2026 · Apr 9, 2026
@@ -205,6 +205,8 @@ type flagConfig struct {
 	enableAutoReload   bool
 	autoReloadInterval model.Duration
 
+	expandRelabelEnv bool
+
 	maxprocsEnable bool
 	memlimitEnable bool
 	memlimitRatio  float64
@@ -333,6 +335,9 @@ func (c *flagConfig) setFeatureListOptions(logger *slog.Logger) error {
 			case "fast-startup":
 				c.tsdb.EnableFastStartup = true
 				logger.Info("Experimental fast startup is enabled.")
+			case "expand-relabel-env-vars":
+				c.expandRelabelEnv = true
+				logger.Info("Environment variable expansion in relabel_configs enabled.")
 			default:
 				logger.Warn("Unknown option for --enable-feature", "option", o)
 			}
@@ -608,7 +613,7 @@ func main() {
 	a.Flag("scrape.discovery-reload-interval", "Interval used by scrape manager to throttle target groups updates.").
 		Hidden().Default("5s").SetValue(&cfg.scrape.DiscoveryReloadInterval)
 
-	a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, extra-scrape-metrics, auto-gomaxprocs, created-timestamp-zero-ingestion, st-storage, concurrent-rule-eval, delayed-compaction, old-ui, otlp-deltatocumulative, promql-duration-expr, use-uncached-io, promql-extended-range-selectors, promql-binop-fill-modifiers, xor2-encoding. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details.").
+	a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, extra-scrape-metrics, auto-gomaxprocs, created-timestamp-zero-ingestion, st-storage, concurrent-rule-eval, delayed-compaction, old-ui, otlp-deltatocumulative, promql-duration-expr, use-uncached-io, promql-extended-range-selectors, promql-binop-fill-modifiers, xor2-encoding, expand-relabel-env-vars. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details.").
 		Default("").StringsVar(&cfg.featureList)
 
 	a.Flag("agent", "Run Prometheus in 'Agent mode'.").BoolVar(&agentMode)
@@ -692,7 +697,7 @@ func main() {
 
 	// Throw error for invalid config before starting other components.
 	var cfgFile *config.Config
-	if cfgFile, err = config.LoadFile(cfg.configFile, agentMode, promslog.NewNopLogger()); err != nil {
+	if cfgFile, err = config.LoadFile(cfg.configFile, agentMode, promslog.NewNopLogger(), cfg.expandRelabelEnv); err != nil {
 		absPath, pathErr := filepath.Abs(cfg.configFile)
 		if pathErr != nil {
 			absPath = cfg.configFile
@@ -1318,7 +1323,7 @@ func main() {
 				for {
 					select {
 					case <-hup:
-						if err := reloadConfig(cfg.configFile, cfg.tsdb.EnableExemplarStorage, logger, noStepSubqueryInterval, callback, reloaders...); err != nil {
+						if err := reloadConfig(cfg.configFile, cfg.tsdb.EnableExemplarStorage, cfg.expandRelabelEnv, logger, noStepSubqueryInterval, callback, reloaders...); err != nil {
 							logger.Error("Error reloading config", "err", err)
 						} else if cfg.enableAutoReload {
 							checksum, err = config.GenerateChecksum(cfg.configFile)
@@ -1327,7 +1332,7 @@ func main() {
 							}
 						}
 					case rc := <-webHandler.Reload():
-						if err := reloadConfig(cfg.configFile, cfg.tsdb.EnableExemplarStorage, logger, noStepSubqueryInterval, callback, reloaders...); err != nil {
+						if err := reloadConfig(cfg.configFile, cfg.tsdb.EnableExemplarStorage, cfg.expandRelabelEnv, logger, noStepSubqueryInterval, callback, reloaders...); err != nil {
 							logger.Error("Error reloading config", "err", err)
 							rc <- err
 						} else {
@@ -1352,7 +1357,7 @@ func main() {
 						}
 						logger.Info("Configuration file change detected, reloading the configuration.")
 
-						if err := reloadConfig(cfg.configFile, cfg.tsdb.EnableExemplarStorage, logger, noStepSubqueryInterval, callback, reloaders...); err != nil {
+						if err := reloadConfig(cfg.configFile, cfg.tsdb.EnableExemplarStorage, cfg.expandRelabelEnv, logger, noStepSubqueryInterval, callback, reloaders...); err != nil {
 							logger.Error("Error reloading config", "err", err)
 						} else {
 							checksum = currentChecksum
@@ -1384,7 +1389,7 @@ func main() {
 					return nil
 				}
 
-				if err := reloadConfig(cfg.configFile, cfg.tsdb.EnableExemplarStorage, logger, noStepSubqueryInterval, func(bool) {}, reloaders...); err != nil {
+				if err := reloadConfig(cfg.configFile, cfg.tsdb.EnableExemplarStorage, cfg.expandRelabelEnv, logger, noStepSubqueryInterval, func(bool) {}, reloaders...); err != nil {
 					return fmt.Errorf("error loading config from %q: %w", cfg.configFile, err)
 				}
 
@@ -1626,7 +1631,7 @@ type reloader struct {
 	reloader func(*config.Config) error
 }
 
-func reloadConfig(filename string, enableExemplarStorage bool, logger *slog.Logger, noStepSubqueryInterval *safePromQLNoStepSubqueryInterval, callback func(bool), rls ...reloader) (err error) {
+func reloadConfig(filename string, enableExemplarStorage, expandRelabelEnv bool, logger *slog.Logger, noStepSubqueryInterval *safePromQLNoStepSubqueryInterval, callback func(bool), rls ...reloader) (err error) {
 	start := time.Now()
 	timingsLogger := logger
 	logger.Info("Loading configuration file", "filename", filename)
@@ -1642,7 +1647,7 @@ func reloadConfig(filename string, enableExemplarStorage bool, logger *slog.Logg
 		}
 	}()
 
-	conf, err := config.LoadFile(filename, agentMode, logger)
+	conf, err := config.LoadFile(filename, agentMode, logger, expandRelabelEnv)
 	if err != nil {
 		return fmt.Errorf("couldn't load configuration (--config.file=%q): %w", filename, err)
 	}

@@ -666,7 +666,7 @@ func checkFileExists(fn string) error {
 func checkConfig(agentMode bool, filename string, checkSyntaxOnly bool) ([]string, []*config.ScrapeConfig, error) {
 	fmt.Println("Checking", filename)
 
-	cfg, err := config.LoadFile(filename, agentMode, logger)
+	cfg, err := config.LoadFile(filename, agentMode, logger, false)
 	if err != nil {
 		return nil, nil, err
 	}

@@ -41,7 +41,7 @@ type sdCheckResult struct {
 func CheckSD(sdConfigFiles, sdJobName string, sdTimeout time.Duration, _ prometheus.Registerer) int {
 	logger := promslog.New(&promslog.Config{})
 
-	cfg, err := config.LoadFile(sdConfigFiles, false, logger)
+	cfg, err := config.LoadFile(sdConfigFiles, false, logger, false)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, "Cannot load config", err)
 		return failureExitCode

@@ -71,7 +71,98 @@ var (
 )
 
 // Load parses the YAML input s into a Config.
-func Load(s string, logger *slog.Logger) (*Config, error) {
+// expandEnvVars expands environment variables in a string using os.Expand.
+// Variables are referenced as $VAR or ${VAR}. Escaping is done with $$.
+// Empty environment variables are logged as warnings and expanded to empty string.
+func expandEnvVars(s string, logger *slog.Logger) string {
+	return os.Expand(s, func(key string) string {
+		if key == "$" {
+			return "$"
+		}
+		if v := os.Getenv(key); v != "" {
+			return v
+		}
+		logger.Warn("Empty environment variable", "name", key)
+		return ""
+	})
+}
+
+// expandRelabelConfigs expands environment variables in replacement and target_label fields.
+// Note: $1, $2, etc. are regex capture group references and are not expanded.
+func expandRelabelConfigs(relabelConfigs []*relabel.Config, logger *slog.Logger) {
+	for _, relabelConfig := range relabelConfigs {
+		if relabelConfig == nil {
+			continue
+		}
+		if relabelConfig.Replacement != "" && containsEnvVar(relabelConfig.Replacement) {
+			newReplacement := expandEnvVars(relabelConfig.Replacement, logger)
+			if newReplacement != relabelConfig.Replacement {
+				logger.Debug("Relabel replacement expanded", "input", relabelConfig.Replacement, "output", newReplacement)
+				relabelConfig.Replacement = newReplacement
+			}
+		}
+		if relabelConfig.TargetLabel != "" && containsEnvVar(relabelConfig.TargetLabel) {
+			newTargetLabel := expandEnvVars(relabelConfig.TargetLabel, logger)
+			if newTargetLabel != relabelConfig.TargetLabel {
+				logger.Debug("Relabel target_label expanded", "input", relabelConfig.TargetLabel, "output", newTargetLabel)
+				relabelConfig.TargetLabel = newTargetLabel
+			}
+		}
+	}
+}
+
+// containsEnvVar checks if a string likely contains environment variables (not just regex references like $1).
+func containsEnvVar(s string) bool {
+	i := 0
+	for i < len(s) {
+		if s[i] == '$' {
+			if i+1 < len(s) {
+				next := s[i+1]
+				switch {
+				case next == '{':
+					// ${VAR} format - check what's inside braces
+					j := i + 2
+					for j < len(s) && s[j] != '}' {
+						j++
+					}
+					if j < len(s) && j > i+2 {
+						varName := s[i+2 : j]
+						// If not all digits, it's likely an env var
+						if !isAllDigits(varName) {
+							return true
+						}
+					}
+					i = j + 1
+				case next == '$':
+					// $$ escape
+					i += 2
+				case (next >= 'A' && next <= 'Z') || (next >= 'a' && next <= 'z') || next == '_':
+					// $VAR_NAME format - likely an env var
+					return true
+				default:
+					i++
+				}
+			} else {
+				i++
+			}
+		} else {
+			i++
+		}
+	}
+	return false
+}
+
+// isAllDigits checks if a string contains only digits.
+func isAllDigits(s string) bool {
+	for _, c := range s {
+		if c < '0' || c > '9' {
+			return false
+		}
+	}
+	return s != ""
+}
+
+func Load(s string, logger *slog.Logger, expandRelabelEnv bool) (*Config, error) {
 	cfg := &Config{}
 	// If the entire config body is empty the UnmarshalYAML method is
 	// never called. We thus have to set the DefaultConfig at the entry
@@ -92,16 +183,7 @@ func Load(s string, logger *slog.Logger) (*Config, error) {
 
 	b := labels.NewScratchBuilder(0)
 	cfg.GlobalConfig.ExternalLabels.Range(func(v labels.Label) {
-		newV := os.Expand(v.Value, func(s string) string {
-			if s == "$" {
-				return "$"
-			}
-			if v := os.Getenv(s); v != "" {
-				return v
-			}
-			logger.Warn("Empty environment variable", "name", s)
-			return ""
-		})
+		newV := expandEnvVars(v.Value, logger)
 		if newV != v.Value {
 			logger.Debug("External label replaced", "label", v.Name, "input", v.Value, "output", newV)
 		}
@@ -112,6 +194,27 @@ func Load(s string, logger *slog.Logger) (*Config, error) {
 		cfg.GlobalConfig.ExternalLabels = b.Labels()
 	}
 
+	if expandRelabelEnv {
+		// Expand environment variables in relabel_configs (opt-in via
+		// --enable-feature=expand-relabel-env-vars).
+		for _, scrapeConfig := range cfg.ScrapeConfigs {
+			expandRelabelConfigs(scrapeConfig.RelabelConfigs, logger)
+			expandRelabelConfigs(scrapeConfig.MetricRelabelConfigs, logger)
+		}
+
+		// Expand environment variables in alerting relabel configs.
+		expandRelabelConfigs(cfg.AlertingConfig.AlertRelabelConfigs, logger)
+		for _, amConfig := range cfg.AlertingConfig.AlertmanagerConfigs {
+			expandRelabelConfigs(amConfig.RelabelConfigs, logger)
+			expandRelabelConfigs(amConfig.AlertRelabelConfigs, logger)
+		}
+
+		// Expand environment variables in remote write relabel configs.
+		for _, remoteWriteConfig := range cfg.RemoteWriteConfigs {
+			expandRelabelConfigs(remoteWriteConfig.WriteRelabelConfigs, logger)
+		}
+	}
+
 	switch cfg.OTLPConfig.TranslationStrategy {
 	case otlptranslator.UnderscoreEscapingWithSuffixes, otlptranslator.UnderscoreEscapingWithoutSuffixes:
 	case "":
@@ -128,12 +231,12 @@ func Load(s string, logger *slog.Logger) (*Config, error) {
 
 // LoadFile parses and validates the given YAML file into a read-only Config.
 // Callers should never write to or shallow copy the returned Config.
-func LoadFile(filename string, agentMode bool, logger *slog.Logger) (*Config, error) {
+func LoadFile(filename string, agentMode bool, logger *slog.Logger, expandRelabelEnv bool) (*Config, error) {
 	content, err := os.ReadFile(filename)
 	if err != nil {
 		return nil, err
 	}
-	cfg, err := Load(string(content), logger)
+	cfg, err := Load(string(content), logger, expandRelabelEnv)
 	if err != nil {
 		return nil, fmt.Errorf("parsing YAML file %s: %w", filename, err)
 	}