veeamm/modules/nixos/redis-cluster.nix

# Redis Cluster NixOS Configuration Module
# Optimized for high throughput and lowest latency
# Security disabled for maximum performance
{
  config,
  pkgs,
  lib,
  ...
}:

with lib;

let
  cfg = config.services.redisCluster;
  
  # Generate Redis instances for cluster
  generateRedisInstances = masters: replicasPerMaster: basePort:
    let
      # Create master instances
      masterInstances = listToAttrs (map (i: {
        name = "master-${toString i}";
        value = {
          enable = true;
          port = basePort + i;
          bind = "0.0.0.0";  # Bind to all interfaces for cluster communication
          
          # Cluster configuration
          extraParams = [
            "--cluster-enabled" "yes"
            "--cluster-config-file" "nodes-${toString (basePort + i)}.conf"
            "--cluster-node-timeout" "5000"
            "--cluster-announce-ip" cfg.announceIp
            "--cluster-announce-port" (toString (basePort + i))
          ];
          
          # Performance optimizations
          save = [];  # Disable RDB for maximum performance
          appendOnly = false;  # Disable AOF for maximum performance
          
          # High-performance settings
          settings = {
            # Memory and performance optimizations
            maxclients = 65000;
            timeout = 0;  # Never timeout connections
            tcp-keepalive = 60;
            tcp-backlog = 511;
            
            # Disable slow operations for performance
            slowlog-log-slower-than = -1;  # Disable slow log
            
            # Memory optimization
            maxmemory-policy = "noeviction";  # Don't evict keys
            
            # Network optimizations
            "tcp-nodelay" = "yes";
            
            # Hash optimizations for speed
            hash-max-ziplist-entries = 512;
            hash-max-ziplist-value = 64;
            list-max-ziplist-size = -2;
            set-max-intset-entries = 512;
            zset-max-ziplist-entries = 128;
            zset-max-ziplist-value = 64;
            
            # Disable key expiration notifications for performance
            notify-keyspace-events = "";
            
            # Cluster optimizations
            cluster-require-full-coverage = "no";  # Allow partial cluster operation
            cluster-allow-reads-when-down = "yes";
          };
        };
      }) (range 0 (masters - 1)));
      
      # Create replica instances
      replicaInstances = listToAttrs (flatten (map (masterIdx: 
        map (replicaIdx: {
          name = "replica-${toString masterIdx}-${toString replicaIdx}";
          value = {
            enable = true;
            port = basePort + masters + (masterIdx * replicasPerMaster) + replicaIdx;
            bind = "0.0.0.0";
            
            # Cluster configuration
            extraParams = [
              "--cluster-enabled" "yes"
              "--cluster-config-file" "nodes-${toString (basePort + masters + (masterIdx * replicasPerMaster) + replicaIdx)}.conf"
              "--cluster-node-timeout" "5000"
              "--cluster-announce-ip" cfg.announceIp
              "--cluster-announce-port" (toString (basePort + masters + (masterIdx * replicasPerMaster) + replicaIdx))
            ];
            
            # Performance optimizations (same as masters)
            save = [];
            appendOnly = false;
            
            settings = {
              maxclients = 65000;
              timeout = 0;
              tcp-keepalive = 60;
              tcp-backlog = 511;
              slowlog-log-slower-than = -1;
              maxmemory-policy = "noeviction";
              "tcp-nodelay" = "yes";
              hash-max-ziplist-entries = 512;
              hash-max-ziplist-value = 64;
              list-max-ziplist-size = -2;
              set-max-intset-entries = 512;
              zset-max-ziplist-entries = 128;
              zset-max-ziplist-value = 64;
              notify-keyspace-events = "";
              cluster-require-full-coverage = "no";
              cluster-allow-reads-when-down = "yes";
              
              # Replica-specific settings
              replica-read-only = "no";  # Allow writes to replicas in cluster mode
            };
          };
        }) (range 0 (replicasPerMaster - 1))
      ) (range 0 (masters - 1))));
      
    in masterInstances // replicaInstances;

in

{
  options.services.redisCluster = {
    enable = mkEnableOption "Redis Cluster";
    
    masters = mkOption {
      type = types.int;
      default = 3;
      description = "Number of master nodes in the cluster (minimum 3)";
    };
    
    replicasPerMaster = mkOption {
      type = types.int;
      default = 1;
      description = "Number of replica nodes per master";
    };
    
    basePort = mkOption {
      type = types.int;
      default = 7000;
      description = "Base port number for Redis cluster nodes";
    };
    
    announceIp = mkOption {
      type = types.str;
      default = "127.0.0.1";
      description = "IP address to announce to other cluster nodes";
    };
    
    openFirewall = mkOption {
      type = types.bool;
      default = false;
      description = "Open firewall ports for cluster communication";
    };
    
    createCluster = mkOption {
      type = types.bool;
      default = true;
      description = "Automatically create and initialize the cluster";
    };
  };

  config = mkIf cfg.enable {
    
    assertions = [
      {
        assertion = cfg.masters >= 3;
        message = "Redis cluster requires at least 3 master nodes";
      }
    ];

    # Configure Redis instances
    services.redis = {
      # Global Redis optimizations
      vmOverCommit = true;
      package = pkgs.redis;
      
      servers = generateRedisInstances cfg.masters cfg.replicasPerMaster cfg.basePort;
    };

    # System-wide performance optimizations
    boot.kernel.sysctl = {
      # Network optimizations
      "net.core.somaxconn" = "65535";
      "net.core.netdev_max_backlog" = "5000";
      "net.ipv4.tcp_max_syn_backlog" = "65535";
      "net.ipv4.tcp_fin_timeout" = "30";
      "net.ipv4.tcp_keepalive_time" = "1200";
      "net.ipv4.tcp_keepalive_intvl" = "15";
      "net.ipv4.tcp_keepalive_probes" = "5";
      
      # Memory optimizations
      "vm.swappiness" = "1";
      "vm.overcommit_memory" = "1";
      "vm.dirty_background_ratio" = "5";
      "vm.dirty_ratio" = "10";
      
      # File system optimizations
      "fs.file-max" = "2097152";
    };

    # Service-level optimizations for all Redis instances
    systemd.services = 
      let
        redisServices = map (i: "redis-master-${toString i}") (range 0 (cfg.masters - 1)) ++
                       flatten (map (masterIdx: 
                         map (replicaIdx: "redis-replica-${toString masterIdx}-${toString replicaIdx}") 
                             (range 0 (cfg.replicasPerMaster - 1))
                       ) (range 0 (cfg.masters - 1)));
        
        serviceConfig = {
          # Resource limits
          LimitNOFILE = "65535";
          LimitNPROC = "65535";
          LimitMEMLOCK = "infinity";
          
          # CPU and scheduling optimizations
          Nice = "-10";
          IOSchedulingClass = "1";
          IOSchedulingPriority = "4";
          
          # Memory optimizations
          OOMScoreAdjust = "-900";
        };
        
      in listToAttrs (map (serviceName: {
        name = serviceName;
        value = { inherit serviceConfig; };
      }) redisServices);

    # Firewall configuration
    networking.firewall = mkIf cfg.openFirewall {
      allowedTCPPorts = 
        # Redis cluster ports
        (map (i: cfg.basePort + i) (range 0 (cfg.masters - 1))) ++
        # Replica ports
        (map (i: cfg.basePort + cfg.masters + i) (range 0 (cfg.masters * cfg.replicasPerMaster - 1))) ++
        # Cluster bus ports (node port + 10000)
        (map (i: cfg.basePort + i + 10000) (range 0 (cfg.masters - 1))) ++
        (map (i: cfg.basePort + cfg.masters + i + 10000) (range 0 (cfg.masters * cfg.replicasPerMaster - 1)));
    };

    # Cluster initialization script
    systemd.services.redis-cluster-init = mkIf cfg.createCluster {
      description = "Initialize Redis Cluster";
      after = map (i: "redis-master-${toString i}.service") (range 0 (cfg.masters - 1));
      wants = map (i: "redis-master-${toString i}.service") (range 0 (cfg.masters - 1));
      wantedBy = [ "multi-user.target" ];
      
      serviceConfig = {
        Type = "oneshot";
        RemainAfterExit = true;
        ExecStart = let
          masterNodes = concatStringsSep " " (map (i: 
            "${cfg.announceIp}:${toString (cfg.basePort + i)}"
          ) (range 0 (cfg.masters - 1)));
        in "${pkgs.redis}/bin/redis-cli --cluster create ${masterNodes} --cluster-replicas ${toString cfg.replicasPerMaster} --cluster-yes";
        
        # Wait for Redis instances to be ready
        ExecStartPre = "${pkgs.bash}/bin/bash -c 'for i in {1..30}; do ${pkgs.redis}/bin/redis-cli -p ${toString cfg.basePort} ping && break || sleep 2; done'";
      };
    };

    # Cluster management utilities
    environment.systemPackages = [ pkgs.redis ];
    
    # Create helper scripts
    environment.etc."redis-cluster-info.sh" = {
      text = ''
        #!/bin/bash
        echo "Redis Cluster Information:"
        echo "========================="
        echo "Masters: ${toString cfg.masters}"
        echo "Replicas per master: ${toString cfg.replicasPerMaster}"
        echo "Base port: ${toString cfg.basePort}"
        echo "Announce IP: ${cfg.announceIp}"
        echo ""
        echo "Master nodes:"
        ${concatStringsSep "\n" (map (i: 
          "echo \"  Master ${toString i}: ${cfg.announceIp}:${toString (cfg.basePort + i)}\""
        ) (range 0 (cfg.masters - 1)))}
        echo ""
        echo "Cluster status:"
        ${pkgs.redis}/bin/redis-cli -p ${toString cfg.basePort} cluster nodes
      '';
      mode = "0755";
    };
  };
}