nix-server/modules/services/metrics.nix

{ config, lib, pkgs, options, ... }:

with lib;
let
  cfg = config.modules.services.metrics;
in {
  options.modules.services.metrics = {
    enable = mkEnableOption "enable grafana with prometheus, alloy, and loki";
    domain = mkOption {
      type = types.str;
      default = "grafana.reidlab.pink";
    };
    grafanaSocket = mkOption {
      type = types.str;
      default = "/run/grafana/grafana.sock";
    };
    alloyPort = mkOption {
      type = types.int;
      default = 12345;
    };
    lokiPort = mkOption {
      type = types.int;
      default = 3100;
    };
    prometheusPort = mkOption {
      type = types.int;
      default = 9090;
    };
    prometheusExporterPortStart = mkOption {
      type = types.int;
      default = 9100;
    };
  };

  config = mkIf cfg.enable {
    services.grafana = {
      enable = true;

      settings = {
        server = {
          domain = cfg.domain;
          protocol = "socket";
          socket = cfg.grafanaSocket;
        };
        database.type = "sqlite3"; # TODO: use postgres?
        # use default secret key
        # TODO: maybe don't do that idk
        security.secret_key = "SW2YcwTIb9zpOOhoPsMm";
      };

      provision.datasources.settings = {
        prune = true; # delete datasources if not in provision file
        datasources = [
          {
            name = "Prometheus";
            type = "prometheus";
            url = "http://127.0.0.1:${toString cfg.prometheusPort}";
            uid = "prometheus";
            isDefault = true;
          }
          {
            name = "Loki";
            type = "loki";
            url = "http://127.0.0.1:${toString cfg.lokiPort}";
            uid = "loki";
          }
        ];
      };
    };

    services.prometheus = let
      mkPort = offset: cfg.prometheusExporterPortStart + offset;

      ports = {
        node = mkPort 0;
        nginx = mkPort 1;
        smartctl = mkPort 2;
      };
    in {
      enable = true;

      port = cfg.prometheusPort;

      exporters = {
        node = {
          enable = true;
          enabledCollectors = [ "systemd" ];
          port = ports.node;
        };
        nginx = {
          enable = true;
          port = ports.nginx;
        };
        smartctl = {
          enable = true;
          port = ports.smartctl;
        };
      };

      scrapeConfigs = let
        commonLabels = {
          hostname = config.networking.hostName;
        };
      in [
        {
          job_name = "node";
          static_configs = [{
            targets = [ "127.0.0.1:${toString ports.node}" ];
            labels = commonLabels;
          }];
        }
        {
          job_name = "nginx";
          static_configs = [{
            targets = [ "127.0.0.1:${toString ports.nginx}" ];
            labels = commonLabels;
          }];
        }
        {
          job_name = "smartctl";
          static_configs = [{
            targets = [ "127.0.0.1:${toString ports.smartctl}" ];
            labels = commonLabels;
          }];
        }
      ];
    };

    services.loki = {
      enable = true;

      configuration = let
        dataDir = config.services.loki.dataDir;
      in {
        auth_enabled = false;

        server.http_listen_port = cfg.lokiPort;

        ingester = {
          lifecycler = {
            address = "127.0.0.1";
            ring = {
              kvstore.store = "inmemory";
              replication_factor = 1;
            };
            final_sleep = "0s";
          };
          chunk_idle_period = "5m";
          chunk_retain_period = "30s";
        };

        schema_config = {
          configs = [
            {
              from = "2023-12-08";
              store = "boltdb-shipper";
              object_store = "filesystem";
              schema = "v11";
              index.prefix = "index_";
              index.period = "24h";
            }
            {
              from = "2024-08-24";
              store = "tsdb";
              object_store = "filesystem";
              schema = "v13";
              index.prefix = "index_";
              index.period = "24h";
            }
          ];
        };

        storage_config = {
          boltdb_shipper = {
            active_index_directory = "${dataDir}/boltdb-shipper-active";
            cache_location = "${dataDir}/boltdb-shipper-cache";
            cache_ttl = "24h";
          };
          tsdb_shipper = {
            active_index_directory = "${dataDir}/tsdb-shipper-active";
            cache_location = "${dataDir}/tsdb-shipper-cache";
            cache_ttl = "24h";
          };
          filesystem.directory = "${dataDir}/chunks";
        };

        limits_config = {
          reject_old_samples = true;
          reject_old_samples_max_age = "168h";
          max_query_lookback = "0s";
        };

        table_manager = {
          retention_deletes_enabled = false;
          retention_period = "0s";
        };

        compactor = {
          working_directory = "${dataDir}/compactor";
          compactor_ring.kvstore.store = "inmemory";
        };
      };
    };

    services.alloy = {
      enable = true;

      extraFlags = [
        "--server.http.listen-addr=127.0.0.1:${toString cfg.alloyPort}"
        "--disable-reporting" # disable telemetry
      ];
      configPath = pkgs.writeText "config.alloy" ''
        loki.relabel "journal" {
          forward_to = []
          rule {
            source_labels = ["__journal__systemd_unit"]
            target_label = "unit"
          }
          rule {
            source_labels = ["__journal__hostname"]
            target_label = "hostname"
          }
        }

        loki.source.journal "journal" {
          forward_to = [loki.write.default.receiver]
          max_age = "12h"

          labels = {job = "systemd-journal"}
          relabel_rules = loki.relabel.journal.rules
        }

        loki.write "default" {
          endpoint {
            url = "http://localhost:${toString cfg.lokiPort}/loki/api/v1/push"
          }
        }
      '';
    };

    services.nginx.statusPage = true;
    services.nginx.virtualHosts."${cfg.domain}" = {
      forceSSL = true;
      enableACME = true;
      locations."/" = {
        proxyPass = "http://unix:${cfg.grafanaSocket}";
        proxyWebsockets = true;
      };
      locations."= /robots.txt" = {
        extraConfig = ''
          add_header Content-Type text/plain;
          return 200 "User-agent: *\nDisallow: /\n";
        '';
      };
    };
  };
}