From e3f8a84ff54b03f5ee42ac944ba8f11adb3965c2 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 24 Jul 2025 15:55:54 +0800 Subject: [PATCH] redis --- flake.lock | 8 +- modules/nixos/default.nix | 1 + modules/nixos/redis-cluster.nix | 293 ++++++++++++++++++++++++++++++++ nixos/configuration.nix | 11 ++ 4 files changed, 309 insertions(+), 4 deletions(-) create mode 100644 modules/nixos/redis-cluster.nix diff --git a/flake.lock b/flake.lock index 532134e..7a6e362 100644 --- a/flake.lock +++ b/flake.lock @@ -7,16 +7,16 @@ ] }, "locked": { - "lastModified": 1726989464, - "narHash": "sha256-Vl+WVTJwutXkimwGprnEtXc/s/s8sMuXzqXaspIGlwM=", + "lastModified": 1752780124, + "narHash": "sha256-5dn97vIYxn6VozKePOQSDxVCsrl38nDdMJXx86KIJH0=", "owner": "nix-community", "repo": "home-manager", - "rev": "2f23fa308a7c067e52dfcc30a0758f47043ec176", + "rev": "c718918222bdb104397762dea67e6b397a7927fe", "type": "github" }, "original": { "owner": "nix-community", - "ref": "release-24.05", + "ref": "release-25.05", "repo": "home-manager", "type": "github" } diff --git a/modules/nixos/default.nix b/modules/nixos/default.nix index 5e75405..9d4b76e 100644 --- a/modules/nixos/default.nix +++ b/modules/nixos/default.nix @@ -6,4 +6,5 @@ incus = import ./incus.nix; orbstack = import ./orbstack.nix; power-user-defaults = import ./power-user-defaults.nix; + redis-cluster = import ./redis-cluster.nix; } diff --git a/modules/nixos/redis-cluster.nix b/modules/nixos/redis-cluster.nix new file mode 100644 index 0000000..5d45cd9 --- /dev/null +++ b/modules/nixos/redis-cluster.nix @@ -0,0 +1,293 @@ +# Redis Cluster NixOS Configuration Module +# Optimized for high throughput and lowest latency +# Security disabled for maximum performance +{ + config, + pkgs, + lib, + ... +}: + +with lib; + +let + cfg = config.services.redisCluster; + + # Generate Redis instances for cluster + generateRedisInstances = masters: replicasPerMaster: basePort: + let + # Create master instances + masterInstances = listToAttrs (map (i: { + name = "master-${toString i}"; + value = { + enable = true; + port = basePort + i; + bind = "0.0.0.0"; # Bind to all interfaces for cluster communication + + # Cluster configuration + extraParams = [ + "--cluster-enabled" "yes" + "--cluster-config-file" "nodes-${toString (basePort + i)}.conf" + "--cluster-node-timeout" "5000" + "--cluster-announce-ip" cfg.announceIp + "--cluster-announce-port" (toString (basePort + i)) + ]; + + # Performance optimizations + save = []; # Disable RDB for maximum performance + appendOnly = false; # Disable AOF for maximum performance + + # High-performance settings + settings = { + # Memory and performance optimizations + maxclients = 65000; + timeout = 0; # Never timeout connections + tcp-keepalive = 60; + tcp-backlog = 511; + + # Disable slow operations for performance + slowlog-log-slower-than = -1; # Disable slow log + + # Memory optimization + maxmemory-policy = "noeviction"; # Don't evict keys + + # Network optimizations + "tcp-nodelay" = "yes"; + + # Hash optimizations for speed + hash-max-ziplist-entries = 512; + hash-max-ziplist-value = 64; + list-max-ziplist-size = -2; + set-max-intset-entries = 512; + zset-max-ziplist-entries = 128; + zset-max-ziplist-value = 64; + + # Disable key expiration notifications for performance + notify-keyspace-events = ""; + + # Cluster optimizations + cluster-require-full-coverage = "no"; # Allow partial cluster operation + cluster-allow-reads-when-down = "yes"; + }; + }; + }) (range 0 (masters - 1))); + + # Create replica instances + replicaInstances = listToAttrs (flatten (map (masterIdx: + map (replicaIdx: { + name = "replica-${toString masterIdx}-${toString replicaIdx}"; + value = { + enable = true; + port = basePort + masters + (masterIdx * replicasPerMaster) + replicaIdx; + bind = "0.0.0.0"; + + # Cluster configuration + extraParams = [ + "--cluster-enabled" "yes" + "--cluster-config-file" "nodes-${toString (basePort + masters + (masterIdx * replicasPerMaster) + replicaIdx)}.conf" + "--cluster-node-timeout" "5000" + "--cluster-announce-ip" cfg.announceIp + "--cluster-announce-port" (toString (basePort + masters + (masterIdx * replicasPerMaster) + replicaIdx)) + ]; + + # Performance optimizations (same as masters) + save = []; + appendOnly = false; + + settings = { + maxclients = 65000; + timeout = 0; + tcp-keepalive = 60; + tcp-backlog = 511; + slowlog-log-slower-than = -1; + maxmemory-policy = "noeviction"; + "tcp-nodelay" = "yes"; + hash-max-ziplist-entries = 512; + hash-max-ziplist-value = 64; + list-max-ziplist-size = -2; + set-max-intset-entries = 512; + zset-max-ziplist-entries = 128; + zset-max-ziplist-value = 64; + notify-keyspace-events = ""; + cluster-require-full-coverage = "no"; + cluster-allow-reads-when-down = "yes"; + + # Replica-specific settings + replica-read-only = "no"; # Allow writes to replicas in cluster mode + }; + }; + }) (range 0 (replicasPerMaster - 1)) + ) (range 0 (masters - 1)))); + + in masterInstances // replicaInstances; + +in + +{ + options.services.redisCluster = { + enable = mkEnableOption "Redis Cluster"; + + masters = mkOption { + type = types.int; + default = 3; + description = "Number of master nodes in the cluster (minimum 3)"; + }; + + replicasPerMaster = mkOption { + type = types.int; + default = 1; + description = "Number of replica nodes per master"; + }; + + basePort = mkOption { + type = types.int; + default = 7000; + description = "Base port number for Redis cluster nodes"; + }; + + announceIp = mkOption { + type = types.str; + default = "127.0.0.1"; + description = "IP address to announce to other cluster nodes"; + }; + + openFirewall = mkOption { + type = types.bool; + default = false; + description = "Open firewall ports for cluster communication"; + }; + + createCluster = mkOption { + type = types.bool; + default = true; + description = "Automatically create and initialize the cluster"; + }; + }; + + config = mkIf cfg.enable { + + assertions = [ + { + assertion = cfg.masters >= 3; + message = "Redis cluster requires at least 3 master nodes"; + } + ]; + + # Configure Redis instances + services.redis = { + # Global Redis optimizations + vmOverCommit = true; + package = pkgs.redis; + + servers = generateRedisInstances cfg.masters cfg.replicasPerMaster cfg.basePort; + }; + + # System-wide performance optimizations + boot.kernel.sysctl = { + # Network optimizations + "net.core.somaxconn" = "65535"; + "net.core.netdev_max_backlog" = "5000"; + "net.ipv4.tcp_max_syn_backlog" = "65535"; + "net.ipv4.tcp_fin_timeout" = "30"; + "net.ipv4.tcp_keepalive_time" = "1200"; + "net.ipv4.tcp_keepalive_intvl" = "15"; + "net.ipv4.tcp_keepalive_probes" = "5"; + + # Memory optimizations + "vm.swappiness" = "1"; + "vm.overcommit_memory" = "1"; + "vm.dirty_background_ratio" = "5"; + "vm.dirty_ratio" = "10"; + + # File system optimizations + "fs.file-max" = "2097152"; + }; + + # Service-level optimizations for all Redis instances + systemd.services = + let + redisServices = map (i: "redis-master-${toString i}") (range 0 (cfg.masters - 1)) ++ + flatten (map (masterIdx: + map (replicaIdx: "redis-replica-${toString masterIdx}-${toString replicaIdx}") + (range 0 (cfg.replicasPerMaster - 1)) + ) (range 0 (cfg.masters - 1))); + + serviceConfig = { + # Resource limits + LimitNOFILE = "65535"; + LimitNPROC = "65535"; + LimitMEMLOCK = "infinity"; + + # CPU and scheduling optimizations + Nice = "-10"; + IOSchedulingClass = "1"; + IOSchedulingPriority = "4"; + + # Memory optimizations + OOMScoreAdjust = "-900"; + }; + + in listToAttrs (map (serviceName: { + name = serviceName; + value = { inherit serviceConfig; }; + }) redisServices); + + # Firewall configuration + networking.firewall = mkIf cfg.openFirewall { + allowedTCPPorts = + # Redis cluster ports + (map (i: cfg.basePort + i) (range 0 (cfg.masters - 1))) ++ + # Replica ports + (map (i: cfg.basePort + cfg.masters + i) (range 0 (cfg.masters * cfg.replicasPerMaster - 1))) ++ + # Cluster bus ports (node port + 10000) + (map (i: cfg.basePort + i + 10000) (range 0 (cfg.masters - 1))) ++ + (map (i: cfg.basePort + cfg.masters + i + 10000) (range 0 (cfg.masters * cfg.replicasPerMaster - 1))); + }; + + # Cluster initialization script + systemd.services.redis-cluster-init = mkIf cfg.createCluster { + description = "Initialize Redis Cluster"; + after = map (i: "redis-master-${toString i}.service") (range 0 (cfg.masters - 1)); + wants = map (i: "redis-master-${toString i}.service") (range 0 (cfg.masters - 1)); + wantedBy = [ "multi-user.target" ]; + + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = let + masterNodes = concatStringsSep " " (map (i: + "${cfg.announceIp}:${toString (cfg.basePort + i)}" + ) (range 0 (cfg.masters - 1))); + in "${pkgs.redis}/bin/redis-cli --cluster create ${masterNodes} --cluster-replicas ${toString cfg.replicasPerMaster} --cluster-yes"; + + # Wait for Redis instances to be ready + ExecStartPre = "${pkgs.bash}/bin/bash -c 'for i in {1..30}; do ${pkgs.redis}/bin/redis-cli -p ${toString cfg.basePort} ping && break || sleep 2; done'"; + }; + }; + + # Cluster management utilities + environment.systemPackages = [ pkgs.redis ]; + + # Create helper scripts + environment.etc."redis-cluster-info.sh" = { + text = '' + #!/bin/bash + echo "Redis Cluster Information:" + echo "=========================" + echo "Masters: ${toString cfg.masters}" + echo "Replicas per master: ${toString cfg.replicasPerMaster}" + echo "Base port: ${toString cfg.basePort}" + echo "Announce IP: ${cfg.announceIp}" + echo "" + echo "Master nodes:" + ${concatStringsSep "\n" (map (i: + "echo \" Master ${toString i}: ${cfg.announceIp}:${toString (cfg.basePort + i)}\"" + ) (range 0 (cfg.masters - 1)))} + echo "" + echo "Cluster status:" + ${pkgs.redis}/bin/redis-cli -p ${toString cfg.basePort} cluster nodes + ''; + mode = "0755"; + }; + }; +} \ No newline at end of file diff --git a/nixos/configuration.nix b/nixos/configuration.nix index 2547dd4..31d209c 100644 --- a/nixos/configuration.nix +++ b/nixos/configuration.nix @@ -21,6 +21,7 @@ outputs.nixosModules.important-defaults outputs.nixosModules.incus outputs.nixosModules.orbstack + outputs.nixosModules.redis-cluster # outputs.nixosModules.power-user-defaults ]; @@ -127,6 +128,16 @@ ports = [ 2222 ]; }; + # Redis Cluster configuration + services.redisCluster = { + enable = true; + masters = 8; # 8 master nodes + replicasPerMaster = 1; # 1 replicas each = 16 total instances + basePort = 7000; # Masters: 7000-7007, Replicas: 7008-7015 + announceIp = "0.0.0.0"; # Bind to all interfaces + openFirewall = true; + }; + # System packages environment.systemPackages = with pkgs; [ neovim