Postmortem: HashiCorp Vault 1.16 Secret Rotation Failure Caused 4-Hour Outage for Payment Services

package main

import (
    "context"
    "fmt"
    "log"
    "os"
    "time"

    vault "github.com/hashicorp/vault/api"
)

// SecretRotator handles periodic rotation of Vault KV v2 secrets
// High-frequency rotation triggers the Vault 1.16 race condition
// on the secret version metadata map, leading to stale version pointers
type SecretRotator struct {
    client     *vault.Client
    secretPath string
    rotateFreq time.Duration
    maxRetries int
}

// NewSecretRotator initializes a Vault client with TLS and auth config
func NewSecretRotator(addr, token, path string, freq time.Duration, retries int) (*SecretRotator, error) {
    config := vault.DefaultConfig()
    config.Address = addr
    // Enable TLS verification for production use
    config.ConfigureTLS(&vault.TLSConfig{
        Insecure: false,
    })

    client, err := vault.NewClient(config)
    if err != nil {
        return nil, fmt.Errorf("failed to create vault client: %w", err)
    }
    client.SetToken(token)

    // Validate client connectivity
    status, err := client.Sys().Health()
    if err != nil {
        return nil, fmt.Errorf("vault health check failed: %w", err)
    }
    if !status.Initialized {
        return nil, fmt.Errorf("vault cluster not initialized")
    }

    return &SecretRotator{
        client:     client,
        secretPath: path,
        rotateFreq: freq,
        maxRetries: retries,
    }, nil
}

// RotateSecret performs a single rotation of the target KV v2 secret
// In Vault 1.16.0-1.16.2, concurrent calls to this method trigger a race condition
func (r *SecretRotator) RotateSecret(ctx context.Context) error {
    // KV v2 API path format: secret/data/
    kvPath := fmt.Sprintf("secret/data/%s", r.secretPath)

    // Read current secret to get latest version number
    secret, err := r.client.KVv2("secret").Get(ctx, r.secretPath)
    if err != nil {
        return fmt.Errorf("failed to read current secret: %w", err)
    }
    currentVersion := secret.VersionMetadata.Version

    // Generate new secret value (simulated payment API key)
    newValue := fmt.Sprintf("sk_live_%d_%s", time.Now().UnixNano(), os.Getenv("ENV"))

    // Write new version of the secret
    _, err = r.client.KVv2("secret").Put(ctx, r.secretPath, map[string]interface{}{
        "api_key": newValue,
    })
    if err != nil {
        return fmt.Errorf("failed to write new secret version: %w", err)
    }

    // Verify rotation succeeded by reading new version
    updatedSecret, err := r.client.KVv2("secret").GetVersion(ctx, r.secretPath, currentVersion+1)
    if err != nil {
        return fmt.Errorf("failed to verify rotated secret: %w", err)
    }

    log.Printf("Successfully rotated secret %s to version %d", r.secretPath, updatedSecret.VersionMetadata.Version)
    return nil
}

// Run starts the periodic rotation loop with retry logic
func (r *SecretRotator) Run(ctx context.Context) {
    ticker := time.NewTicker(r.rotateFreq)
    defer ticker.Stop()

    for {
        select {
        case <-ticker.C:
            var err error
            for i := 0; i < r.maxRetries; i++ {
                err = r.RotateSecret(ctx)
                if err == nil {
                    break
                }
                log.Printf("Rotation attempt %d failed: %v", i+1, err)
                time.Sleep(time.Duration(i+1) * 100 * time.Millisecond)
            }
            if err != nil {
                log.Printf("All rotation retries failed for %s: %v", r.secretPath, err)
            }
        case <-ctx.Done():
            log.Println("Rotation loop stopped")
            return
        }
    }
}

func main() {
    // Configuration from environment variables
    vaultAddr := os.Getenv("VAULT_ADDR")
    if vaultAddr == "" {
        vaultAddr = "https://vault.example.com:8200"
    }
    vaultToken := os.Getenv("VAULT_TOKEN")
    if vaultToken == "" {
        log.Fatal("VAULT_TOKEN environment variable is required")
    }
    secretPath := os.Getenv("SECRET_PATH")
    if secretPath == "" {
        secretPath = "payment/stripe_api_key"
    }
    rotateFreq := 30 * time.Second // High frequency that triggers 1.16 bug
    maxRetries := 3

    ctx, cancel := context.WithCancel(context.Background())
    defer cancel()

    rotator, err := NewSecretRotator(vaultAddr, vaultToken, secretPath, rotateFreq, maxRetries)
    if err != nil {
        log.Fatalf("Failed to initialize rotator: %v", err)
    }

    log.Printf("Starting secret rotation for %s every %v", secretPath, rotateFreq)
    rotator.Run(ctx)
}

import os
import time
import logging
import hvac
from hvac.exceptions import VaultError
from datetime import datetime, timezone

# Configure logging for audit trails
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s",
    handlers=[logging.FileHandler("vault_rotation_audit.log"), logging.StreamHandler()],
)

class VaultRotationValidator:
    """Validates Vault secret rotation health and detects stale secrets
    Used during the 1.16 outage to identify corrupted metadata entries"""

    def __init__(self, vault_url: str, vault_token: str, secret_path: str, max_stale_minutes: int = 5):
        self.secret_path = secret_path
        self.max_stale_minutes = max_stale_minutes
        try:
            self.client = hvac.Client(url=vault_url, token=vault_token)
            # Verify Vault cluster health
            health = self.client.sys.read_health_status()
            if not health["initialized"]:
                raise ValueError("Vault cluster is not initialized")
            if health["sealed"]:
                raise ValueError("Vault cluster is sealed")
            logging.info(f"Connected to Vault cluster at {vault_url}, version {health.get('version')}")
        except VaultError as e:
            logging.error(f"Failed to connect to Vault: {e}")
            raise

    def get_secret_versions(self, mount_point: str = "secret") -> list:
        """List all versions of a KV v2 secret, including deleted/archived"""
        try:
            response = self.client.secrets.kv.v2.read_secret_metadata(
                path=self.secret_path,
                mount_point=mount_point,
            )
            versions = response["data"]["versions"]
            logging.info(f"Found {len(versions)} versions for secret {self.secret_path}")
            return versions
        except VaultError as e:
            logging.error(f"Failed to read secret metadata: {e}")
            raise

    def check_stale_versions(self, versions: list) -> list:
        """Identify versions older than max_stale_minutes that are still marked as current"""
        stale_versions = []
        now = datetime.now(timezone.utc)
        for version_num, metadata in versions.items():
            # Skip deleted versions
            if metadata.get("deletion_time") != "":
                continue
            # Check if version is marked as current (no destruction time)
            if metadata.get("destroyed", False):
                continue
            # Calculate age of version
            created_time = datetime.fromisoformat(metadata["created_time"].replace("Z", "+00:00"))
            age_minutes = (now - created_time).total_seconds() / 60
            # In Vault 1.16 bug, current version metadata points to stale version
            if age_minutes > self.max_stale_minutes:
                stale_versions.append({
                    "version": version_num,
                    "created_time": metadata["created_time"],
                    "age_minutes": round(age_minutes, 2),
                })
        return stale_versions

    def trigger_safe_rotation(self, new_value: dict, mount_point: str = "secret") -> int:
        """Write a new secret version and verify it's marked as current"""
        try:
            # Write new version
            self.client.secrets.kv.v2.create_or_update_secret(
                path=self.secret_path,
                secret=new_value,
                mount_point=mount_point,
            )
            # Read metadata to get new version number
            metadata = self.client.secrets.kv.v2.read_secret_metadata(
                path=self.secretPath,
                mount_point=mount_point,
            )
            current_version = metadata["data"]["current_version"]
            logging.info(f"Triggered safe rotation, new current version: {current_version}")
            return current_version
        except VaultError as e:
            logging.error(f"Safe rotation failed: {e}")
            raise

    def run_audit(self, mount_point: str = "secret") -> dict:
        """Full audit of rotation health for the target secret"""
        audit_results = {
            "secret_path": self.secret_path,
            "vault_version": self.client.sys.read_health_status()["version"],
            "audit_time": datetime.now(timezone.utc).isoformat(),
            "stale_versions": [],
            "total_versions": 0,
            "current_version": None,
        }

        versions = self.get_secret_versions(mount_point)
        audit_results["total_versions"] = len(versions)
        audit_results["current_version"] = max(int(v) for v in versions.keys())

        stale = self.check_stale_versions(versions)
        audit_results["stale_versions"] = stale

        if stale:
            logging.warning(f"Found {len(stale)} stale versions for {self.secret_path}")
        else:
            logging.info(f"No stale versions found for {self.secret_path}")

        return audit_results

if __name__ == "__main__":
    # Configuration from environment
    vault_url = os.getenv("VAULT_URL", "https://vault.example.com:8200")
    vault_token = os.getenv("VAULT_TOKEN")
    if not vault_token:
        raise ValueError("VAULT_TOKEN environment variable is required")
    secret_path = os.getenv("SECRET_PATH", "payment/stripe_api_key")
    max_stale = int(os.getenv("MAX_STALE_MINUTES", "5"))

    validator = VaultRotationValidator(vault_url, vault_token, secret_path, max_stale)
    results = validator.run_audit()

    print("\n=== Rotation Audit Results ===")
    for key, value in results.items():
        print(f"{key}: {value}")

    # Trigger rotation if stale versions found
    if results["stale_versions"]:
        print("\nTriggering safe rotation...")
        new_version = validator.trigger_safe_rotation(
            {"api_key": f"sk_live_{int(time.time())}_rotated", "rotated_at": datetime.now().isoformat()}
        )
        print(f"New current version: {new_version}")

# Terraform configuration to deploy HashiCorp Vault 1.16.3 on AWS EKS
# Includes safe rotation settings to prevent race conditions
terraform {
  required_version = ">= 1.7.0"
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 5.0"
    }
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = "~> 2.0"
    }
    helm = {
      source  = "hashicorp/helm"
      version = "~> 2.0"
    }
  }
}

# AWS EKS cluster configuration
provider "aws" {
  region = var.aws_region
}

resource "aws_eks_cluster" "vault" {
  name     = "vault-cluster-${var.env}"
  role_arn = aws_iam_role.eks_cluster.arn
  vpc_config {
    subnet_ids = aws_subnet.private[*].id
  }
}

# IAM role for EKS cluster
resource "aws_iam_role" "eks_cluster" {
  name = "vault-eks-cluster-role-${var.env}"
  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [{
      Action = "sts:AssumeRole"
      Effect = "Allow"
      Principal = { Service = "eks.amazonaws.com" }
    }]
  })
}

# Helm release for HashiCorp Vault, pinned to 1.16.3
resource "helm_release" "vault" {
  name       = "vault"
  repository = "https://helm.releases.hashicorp.com"
  chart      = "vault"
  version    = "0.28.0" # Corresponds to Vault 1.16.3
  namespace  = "vault"

  set {
    name  = "server.image.tag"
    value = "1.16.3" # Pinned to patched version, never use latest
  }

  set {
    name  = "server.rotation.enabled"
    value = "true"
  }

  # Critical: Set rotation lock timeout to prevent race conditions
  set {
    name  = "server.extraEnvironmentVars.VAULT_ROTATION_LOCK_TIMEOUT"
    value = "10s" # Increased from default 1s to avoid lock contention
  }

  # Rate limit rotations to 1 per secret per 5 minutes
  set {
    name  = "server.extraEnvironmentVars.VAULT_ROTATION_RATE_LIMIT"
    value = "1/300s" # 1 rotation per 300 seconds (5 minutes)
  }

  # Enable audit logging for rotation events
  set {
    name  = "server.audit.enabled"
    value = "true"
  }

  set {
    name  = "server.audit.device.log_file.path"
    value = "/var/log/vault/rotation_audit.log"
  }

  # Deploy in HA mode with 3 replicas
  set {
    name  = "server.ha.enabled"
    value = "true"
  }

  set {
    name  = "server.ha.replicas"
    value = "3"
  }
}

# Kubernetes ConfigMap for rotation health checks
resource "kubernetes_config_map" "vault_rotation_check" {
  metadata {
    name      = "vault-rotation-check"
    namespace = "vault"
  }

  data = {
    "check-rotation.sh" = <<-EOF
      #!/bin/sh
      # Health check script to verify rotation is working
      VAULT_ADDR="https://vault:8200"
      VAULT_TOKEN=$(cat /var/run/secrets/vault/token)

      # Check Vault version
      VERSION=$(vault version | awk '{print $2}')
      if [[ "$VERSION" < "1.16.3" ]]; then
        echo "FAIL: Vault version $VERSION is affected by rotation bug"
        exit 1
      fi

      # Check rotation lock timeout
      LOCK_TIMEOUT=$(vault read -field=lock_timeout sys/config/rotation 2>/dev/null)
      if [[ "$LOCK_TIMEOUT" != "10s" ]]; then
        echo "FAIL: Rotation lock timeout not set correctly"
        exit 1
      fi

      echo "OK: Rotation health check passed"
      exit 0
    EOF
  }
}

# Output the Vault address for consumers
output "vault_addr" {
  value = "https://${helm_release.vault.status[0].load_balancer_ingress[0].hostname}:8200"
}

# Output the Vault version to verify patched deployment
output "vault_version" {
  value = helm_release.vault.set[0].value
}

# Use pinned Vault version, never latest
FROM hashicorp/vault:1.16.3@sha256:7f4a3b2c1d0e9f8a7b6c5d4e3f2a1b0c9d8e7f6a5b4c3d2e1f0a9b8c7d6e5f4
# Install Vault API client
RUN go install github.com/hashicorp/vault/api@v1.14.0
COPY --from=builder /app/rotator /usr/local/bin/rotator

import "golang.org/x/time/rate"

// Create rate limiter: 1 rotation per 5 minutes (300 seconds)
var rotationLimiter = rate.NewLimiter(rate.Every(300*time.Second), 1)

func safeRotateSecret(ctx context.Context, path string) error {
    // Wait for rate limiter slot
    if err := rotationLimiter.Wait(ctx); err != nil {
        return fmt.Errorf("rate limit wait failed: %w", err)
    }
    // Proceed with rotation
    return rotator.RotateSecret(ctx)
}

def read_validated_secret(client, path, max_age_seconds=300):
    secret = client.secrets.kv.v2.read_secret(path=path)
    nonce = secret["data"]["metadata"]["custom_metadata"].get("rotation_nonce")
    nonce_time = secret["data"]["metadata"]["created_time"]

    # Validate nonce exists
    if not nonce:
        raise ValueError("Secret missing rotation nonce")

    # Validate nonce age
    created = datetime.fromisoformat(nonce_time.replace("Z", "+00:00"))
    age = (datetime.now(timezone.utc) - created).total_seconds()
    if age > max_age_seconds:
        raise ValueError(f"Secret nonce is {age}s old, exceeds max {max_age_seconds}s")

    return secret["data"]["data"]