veeamm/modules/nixos/redis-cluster.nix

# Redis Cluster NixOS Configuration Module
# Optimized for high throughput and lowest latency
# Security disabled for maximum performance
{
  config,
  pkgs,
  lib,
  ...
}:

with lib;

let
  cfg = config.services.redisCluster;
  
  # Generate Redis instances for cluster
  generateRedisInstances = masters: replicasPerMaster: basePort:
    let
      # Create master instances
      masterInstances = listToAttrs (map (i: {
        name = "master-${toString i}";
        value = {
          enable = true;
          port = basePort + i;
          bind = "0.0.0.0";  # Bind to all interfaces for cluster communication
          
          # Cluster configuration
          extraParams = [
            "--cluster-enabled" "yes"
            "--cluster-config-file" "nodes-${toString (basePort + i)}.conf"
            "--cluster-node-timeout" "5000"
            "--cluster-announce-ip" cfg.announceIp
            "--cluster-announce-port" (toString (basePort + i))
          ];
          
          # Performance optimizations
          save = [];  # Disable RDB for maximum performance
          appendOnly = false;  # Disable AOF for maximum performance
          
          # High-performance settings
          settings = {
            # Memory and performance optimizations
            maxclients = mkForce 65000;
            timeout = mkForce 0;  # Never timeout connections
            tcp-keepalive = mkForce 60;
            tcp-backlog = mkForce 511;
            
            # Disable slow operations for performance
            slowlog-log-slower-than = mkForce (-1);  # Disable slow log
            
            # Memory optimization
            maxmemory-policy = mkForce "noeviction";  # Don't evict keys
            
            # Network optimizations
            # Note: tcp-nodelay is automatically handled by Redis as a socket option
            
            # Hash optimizations for speed
            hash-max-ziplist-entries = mkForce 512;
            hash-max-ziplist-value = mkForce 64;
            list-max-ziplist-size = mkForce (-2);
            set-max-intset-entries = mkForce 512;
            zset-max-ziplist-entries = mkForce 128;
            zset-max-ziplist-value = mkForce 64;
            
            # Cluster optimizations
            cluster-require-full-coverage = mkForce "no";  # Allow partial cluster operation
            cluster-allow-reads-when-down = mkForce "yes";
          };
        };
      }) (range 0 (masters - 1)));
      
      # Create replica instances
      replicaInstances = listToAttrs (flatten (map (masterIdx: 
        map (replicaIdx: {
          name = "replica-${toString masterIdx}-${toString replicaIdx}";
          value = {
            enable = true;
            port = basePort + masters + (masterIdx * replicasPerMaster) + replicaIdx;
            bind = "0.0.0.0";
            
            # Cluster configuration
            extraParams = [
              "--cluster-enabled" "yes"
              "--cluster-config-file" "nodes-${toString (basePort + masters + (masterIdx * replicasPerMaster) + replicaIdx)}.conf"
              "--cluster-node-timeout" "5000"
              "--cluster-announce-ip" cfg.announceIp
              "--cluster-announce-port" (toString (basePort + masters + (masterIdx * replicasPerMaster) + replicaIdx))
            ];
            
            # Performance optimizations (same as masters)
            save = [];
            appendOnly = false;
            
            settings = {
              maxclients = mkForce 65000;
              timeout = mkForce 0;
              tcp-keepalive = mkForce 60;
              tcp-backlog = mkForce 511;
              slowlog-log-slower-than = mkForce (-1);
              maxmemory-policy = mkForce "noeviction";
              # Note: tcp-nodelay is automatically handled by Redis as a socket option
              hash-max-ziplist-entries = mkForce 512;
              hash-max-ziplist-value = mkForce 64;
              list-max-ziplist-size = mkForce (-2);
              set-max-intset-entries = mkForce 512;
              zset-max-ziplist-entries = mkForce 128;
              zset-max-ziplist-value = mkForce 64;
              cluster-require-full-coverage = mkForce "no";
              cluster-allow-reads-when-down = mkForce "yes";
              
              # Replica-specific settings
              replica-read-only = mkForce "no";  # Allow writes to replicas in cluster mode
            };
          };
        }) (range 0 (replicasPerMaster - 1))
      ) (range 0 (masters - 1))));
      
    in masterInstances // replicaInstances;

in

{
  options.services.redisCluster = {
    enable = mkEnableOption "Redis Cluster";
    
    masters = mkOption {
      type = types.int;
      default = 3;
      description = "Number of master nodes in the cluster (minimum 3)";
    };
    
    replicasPerMaster = mkOption {
      type = types.int;
      default = 1;
      description = "Number of replica nodes per master";
    };
    
    basePort = mkOption {
      type = types.int;
      default = 7000;
      description = "Base port number for Redis cluster nodes";
    };
    
    announceIp = mkOption {
      type = types.str;
      default = "127.0.0.1";
      description = "IP address to announce to other cluster nodes";
    };
    
    openFirewall = mkOption {
      type = types.bool;
      default = false;
      description = "Open firewall ports for cluster communication";
    };
    
    createCluster = mkOption {
      type = types.bool;
      default = true;
      description = "Automatically create and initialize the cluster";
    };
  };

  config = mkIf cfg.enable {
    
    assertions = [
      {
        assertion = cfg.masters >= 3;
        message = "Redis cluster requires at least 3 master nodes";
      }
    ];

    # Configure Redis instances
    services.redis = {
      # Global Redis optimizations
      vmOverCommit = true;
      package = pkgs.redis;
      
      servers = generateRedisInstances cfg.masters cfg.replicasPerMaster cfg.basePort;
    };

    # System-wide performance optimizations
    boot.kernel.sysctl = {
      # Network optimizations
      "net.core.somaxconn" = mkForce "65535";
      "net.core.netdev_max_backlog" = mkForce "5000";
      "net.ipv4.tcp_max_syn_backlog" = mkForce "65535";
      "net.ipv4.tcp_fin_timeout" = mkForce "30";
      "net.ipv4.tcp_keepalive_time" = mkForce "1200";
      "net.ipv4.tcp_keepalive_intvl" = mkForce "15";
      "net.ipv4.tcp_keepalive_probes" = mkForce "5";
      
      # Memory optimizations
      "vm.swappiness" = mkForce "1";
      "vm.overcommit_memory" = mkForce "1";
      "vm.dirty_background_ratio" = mkForce "5";
      "vm.dirty_ratio" = mkForce "10";
      
      # File system optimizations
      "fs.file-max" = mkForce "2097152";
    };

    # System-wide ulimit settings for Redis
    security.pam.loginLimits = [
      {
        domain = "*";
        type = "soft";
        item = "nofile";
        value = "1048576";
      }
      {
        domain = "*";
        type = "hard";
        item = "nofile";
        value = "1048576";
      }
    ];

    # Service-level optimizations for all Redis instances
    systemd.services = 
      let
        redisServices = map (i: "redis-master-${toString i}") (range 0 (cfg.masters - 1)) ++
                       flatten (map (masterIdx: 
                         map (replicaIdx: "redis-replica-${toString masterIdx}-${toString replicaIdx}") 
                             (range 0 (cfg.replicasPerMaster - 1))
                       ) (range 0 (cfg.masters - 1)));
        
        serviceConfig = {
          # Resource limits - set much higher for Redis cluster
          LimitNOFILE = "1048576";
          LimitNPROC = "65535";
          LimitMEMLOCK = "infinity";
          
          # CPU and scheduling optimizations
          Nice = "-10";
          IOSchedulingClass = "1";
          IOSchedulingPriority = "4";
          
          # Memory optimizations
          OOMScoreAdjust = "-900";
        };
        
        # Redis service optimizations
        redisServiceConfigs = listToAttrs (map (serviceName: {
          name = serviceName;
          value = { inherit serviceConfig; };
        }) redisServices);
        
        # Cluster initialization service
        clusterInitService = mkIf cfg.createCluster {
          redis-cluster-init = {
            description = "Initialize Redis Cluster";
            after = map (i: "redis-master-${toString i}.service") (range 0 (cfg.masters - 1));
            wants = map (i: "redis-master-${toString i}.service") (range 0 (cfg.masters - 1));
            wantedBy = [ "multi-user.target" ];
            
            serviceConfig = {
              Type = "oneshot";
              RemainAfterExit = true;
              ExecStart = let
                masterNodes = concatStringsSep " " (map (i: 
                  "${cfg.announceIp}:${toString (cfg.basePort + i)}"
                ) (range 0 (cfg.masters - 1)));
              in "${pkgs.redis}/bin/redis-cli --cluster create ${masterNodes} --cluster-replicas ${toString cfg.replicasPerMaster} --cluster-yes";
              
              # Wait for Redis instances to be ready
              ExecStartPre = "${pkgs.bash}/bin/bash -c 'for i in {1..30}; do ${pkgs.redis}/bin/redis-cli -p ${toString cfg.basePort} ping && break || sleep 2; done'";
            };
          };
        };
        
      in redisServiceConfigs // clusterInitService;

    # Firewall configuration
    networking.firewall = mkIf cfg.openFirewall {
      allowedTCPPorts = 
        # Redis cluster ports
        (map (i: cfg.basePort + i) (range 0 (cfg.masters - 1))) ++
        # Replica ports
        (map (i: cfg.basePort + cfg.masters + i) (range 0 (cfg.masters * cfg.replicasPerMaster - 1))) ++
        # Cluster bus ports (node port + 10000)
        (map (i: cfg.basePort + i + 10000) (range 0 (cfg.masters - 1))) ++
        (map (i: cfg.basePort + cfg.masters + i + 10000) (range 0 (cfg.masters * cfg.replicasPerMaster - 1)));
    };

    # Cluster management utilities
    environment.systemPackages = [ 
      pkgs.redis 
      (pkgs.writeShellScriptBin "redis-cluster-rebuild" ''
        exec /etc/redis-cluster-rebuild.sh "$@"
      '')
    ];
    
    # Create helper scripts
    environment.etc."redis-cluster-info.sh" = {
      text = ''
        #!/bin/bash
        echo "Redis Cluster Information:"
        echo "========================="
        echo "Masters: ${toString cfg.masters}"
        echo "Replicas per master: ${toString cfg.replicasPerMaster}"
        echo "Base port: ${toString cfg.basePort}"
        echo "Announce IP: ${cfg.announceIp}"
        echo ""
        echo "Master nodes:"
        ${concatStringsSep "\n" (map (i: 
          "echo \"  Master ${toString i}: ${cfg.announceIp}:${toString (cfg.basePort + i)}\""
        ) (range 0 (cfg.masters - 1)))}
        echo ""
        echo "Cluster status:"
        ${pkgs.redis}/bin/redis-cli -p ${toString cfg.basePort} cluster nodes
      '';
      mode = "0755";
    };

    environment.etc."redis-cluster-rebuild.sh" = {
      text = ''
        #!/usr/bin/env bash
        set -e

        # Configuration from NixOS
        MASTERS=${toString cfg.masters}
        REPLICAS_PER_MASTER=${toString cfg.replicasPerMaster}
        BASE_PORT=${toString cfg.basePort}
        ANNOUNCE_IP="${cfg.announceIp}"

        echo "Redis Cluster Rebuild Script"
        echo "============================"
        echo "Masters: $MASTERS"
        echo "Replicas per master: $REPLICAS_PER_MASTER"
        echo "Base port: $BASE_PORT"
        echo "Announce IP: $ANNOUNCE_IP"
        echo ""

        # Function to stop all Redis services
        stop_redis_services() {
          echo "Stopping Redis services..."
          
          # Stop cluster init service first
          systemctl stop redis-cluster-init.service 2>/dev/null || true
          
          # Stop all master services
          for i in $(seq 0 $((MASTERS - 1))); do
            echo "  Stopping redis-master-$i..."
            systemctl stop redis-master-$i.service 2>/dev/null || true
          done
          
          # Stop all replica services
          for master_idx in $(seq 0 $((MASTERS - 1))); do
            for replica_idx in $(seq 0 $((REPLICAS_PER_MASTER - 1))); do
              echo "  Stopping redis-replica-$master_idx-$replica_idx..."
              systemctl stop redis-replica-$master_idx-$replica_idx.service 2>/dev/null || true
            done
          done
          
          echo "All Redis services stopped."
        }

        # Function to clean Redis data directories
        clean_redis_data() {
          echo "Cleaning Redis data directories..."
          
          # Remove main Redis data directory
          if [ -d "/var/lib/redis" ]; then
            echo "  Removing /var/lib/redis..."
            rm -rf /var/lib/redis
          fi
          
          # Remove any cluster configuration files from working directories
          for i in $(seq 0 $((MASTERS - 1))); do
            local port=$((BASE_PORT + i))
            rm -f /var/lib/redis*/nodes-$port.conf 2>/dev/null || true
            rm -f /tmp/nodes-$port.conf 2>/dev/null || true
            rm -f ./nodes-$port.conf 2>/dev/null || true
          done
          
          # Remove replica cluster config files
          for master_idx in $(seq 0 $((MASTERS - 1))); do
            for replica_idx in $(seq 0 $((REPLICAS_PER_MASTER - 1))); do
              local port=$((BASE_PORT + MASTERS + master_idx * REPLICAS_PER_MASTER + replica_idx))
              rm -f /var/lib/redis*/nodes-$port.conf 2>/dev/null || true
              rm -f /tmp/nodes-$port.conf 2>/dev/null || true
              rm -f ./nodes-$port.conf 2>/dev/null || true
            done
          done
          
          echo "Redis data directories cleaned."
        }

        # Function to start Redis services
        start_redis_services() {
          echo "Starting Redis services..."
          
          # Start all master services
          for i in $(seq 0 $((MASTERS - 1))); do
            echo "  Starting redis-master-$i..."
            systemctl start redis-master-$i.service
          done
          
          # Start all replica services
          for master_idx in $(seq 0 $((MASTERS - 1))); do
            for replica_idx in $(seq 0 $((REPLICAS_PER_MASTER - 1))); do
              echo "  Starting redis-replica-$master_idx-$replica_idx..."
              systemctl start redis-replica-$master_idx-$replica_idx.service
            done
          done
          
          echo "All Redis services started."
        }

        # Function to wait for Redis services to be ready
        wait_for_redis() {
          echo "Waiting for Redis services to be ready..."
          
          # Wait for masters
          for i in $(seq 0 $((MASTERS - 1))); do
            local port=$((BASE_PORT + i))
            echo "  Waiting for master on port $port..."
            for attempt in $(seq 1 30); do
              if ${pkgs.redis}/bin/redis-cli -p $port ping >/dev/null 2>&1; then
                echo "    Master on port $port is ready."
                break
              fi
              if [ $attempt -eq 30 ]; then
                echo "    ERROR: Master on port $port failed to start!"
                exit 1
              fi
              sleep 1
            done
          done
          
          # Wait for replicas
          for master_idx in $(seq 0 $((MASTERS - 1))); do
            for replica_idx in $(seq 0 $((REPLICAS_PER_MASTER - 1))); do
              local port=$((BASE_PORT + MASTERS + master_idx * REPLICAS_PER_MASTER + replica_idx))
              echo "  Waiting for replica on port $port..."
              for attempt in $(seq 1 30); do
                if ${pkgs.redis}/bin/redis-cli -p $port ping >/dev/null 2>&1; then
                  echo "    Replica on port $port is ready."
                  break
                fi
                if [ $attempt -eq 30 ]; then
                  echo "    ERROR: Replica on port $port failed to start!"
                  exit 1
                fi
                sleep 1
              done
            done
          done
          
          echo "All Redis services are ready."
        }

        # Function to create the cluster
        create_cluster() {
          echo "Creating Redis cluster..."
          
          # Build master nodes list
          MASTER_NODES=""
          for i in $(seq 0 $((MASTERS - 1))); do
            local port=$((BASE_PORT + i))
            MASTER_NODES="$MASTER_NODES $ANNOUNCE_IP:$port"
          done
          
          echo "  Master nodes:$MASTER_NODES"
          echo "  Creating cluster with $REPLICAS_PER_MASTER replicas per master..."
          
          ${pkgs.redis}/bin/redis-cli --cluster create $MASTER_NODES --cluster-replicas $REPLICAS_PER_MASTER --cluster-yes
          
          echo "Redis cluster created successfully!"
        }

        # Function to show cluster status
        show_cluster_status() {
          echo ""
          echo "Cluster Status:"
          echo "==============="
          ${pkgs.redis}/bin/redis-cli -p $BASE_PORT cluster nodes
          echo ""
          ${pkgs.redis}/bin/redis-cli -p $BASE_PORT cluster info
        }

        # Main execution
        case "''${1:-rebuild}" in
          "stop")
            stop_redis_services
            ;;
          "clean")
            clean_redis_data
            ;;
          "start")
            start_redis_services
            wait_for_redis
            ;;
          "create")
            create_cluster
            show_cluster_status
            ;;
          "rebuild"|"")
            echo "Full rebuild process starting..."
            stop_redis_services
            sleep 2
            clean_redis_data
            start_redis_services
            wait_for_redis
            sleep 3
            create_cluster
            show_cluster_status
            echo ""
            echo "Redis cluster rebuild completed successfully!"
            ;;
          "status")
            show_cluster_status
            ;;
          *)
            echo "Usage: $0 [stop|clean|start|create|rebuild|status]"
            echo ""
            echo "Commands:"
            echo "  stop     - Stop all Redis services"
            echo "  clean    - Remove Redis data directories"
            echo "  start    - Start all Redis services"
            echo "  create   - Create the cluster (services must be running)"
            echo "  rebuild  - Full rebuild (stop, clean, start, create) [default]"
            echo "  status   - Show cluster status"
            exit 1
            ;;
        esac
      '';
      mode = "0755";
    };
  };
}