nix-server/modules/services/metrics.nix

{ config, lib, pkgs, options, ... }:

with lib;
let
  cfg = config.modules.services.metrics;
in {
  options.modules.services.metrics = {
    enable = mkEnableOption "enable grafana with loki, prometheus, and promtail";
    domain = mkOption {
      type = types.str;
      default = "grafana.reidlab.pink";
    };
    orgId = mkOption {
      type = types.int;
      default = 1;
    };
    grafanaPort = mkOption {
      type = types.int;
      default = 3000;
    };
    promtailPort = mkOption {
      type = types.int;
      default = 3001;
    };
    lokiPort = mkOption {
      type = types.int;
      default = 3002;
    };
    prometheusPort = mkOption {
      type = types.int;
      default = 9090;
    };
    prometheusExporterPortStart = mkOption {
      type = types.int;
      default = 9100;
    };
  };

  config = mkIf cfg.enable {
    services.grafana = {
      enable = true;

      settings = {
        server = {
          domain = cfg.domain;
          http_port = cfg.grafanaPort;
          http_addr = "127.0.0.1";
        };
      };

      provision.datasources.settings = {
        datasources = [
          {
            orgId = cfg.orgId;
            name = "Prometheus";
            type = "prometheus";
            url = "http://127.0.0.1:${toString cfg.prometheusPort}";
            uid = "prometheus";
            isDefault = true;
          }
          {
            orgId = cfg.orgId;
            name = "Loki";
            type = "loki";
            url = "http://127.0.0.1:${toString cfg.lokiPort}";
            uid = "loki";
          }
        ];
        deleteDatasources = [
          {
            orgId = cfg.orgId;
            name = "Prometheus";
          }
          {
            orgId = cfg.orgId;
            name = "Loki";
          }
        ];
      };
    };

    services.prometheus = let
      mkPort = offset: cfg.prometheusExporterPortStart + offset;

      ports = {
        node = mkPort 0;
        nginx = mkPort 1;
      };
    in {
      enable = true;

      port = cfg.prometheusPort;

      exporters = {
        node = {
          enable = true;
          enabledCollectors = [ "systemd" ];
          port = ports.node;
        };
        nginx = {
          enable = true;
          port = ports.nginx;
        };
      };

      scrapeConfigs = [
        {
          job_name = "node";
          static_configs = [{
            targets = [ "127.0.0.1:${toString ports.node}" ];
          }];
        }
        {
          job_name = "nginx";
          static_configs = [{
            targets = [ "127.0.0.1:${toString ports.nginx}" ];
          }];
        }
      ];
    };

    services.loki = {
      enable = true;

      configuration = let
        dataDir = config.services.loki.dataDir;
      in {
        auth_enabled = false;

        server.http_listen_port = cfg.lokiPort;

        ingester = {
          lifecycler = {
            address = "127.0.0.1";
            ring = {
              kvstore.store = "inmemory";
              replication_factor = 1;
            };
            final_sleep = "0s";
          };
          chunk_idle_period = "5m";
          chunk_retain_period = "30s";
        };

        schema_config = {
          configs = [
            {
              from = "2023-12-08";
              store = "boltdb-shipper";
              object_store = "filesystem";
              schema = "v11";
              index.prefix = "index_";
              index.period = "24h";
            }
            {
              from = "2024-08-24";
              store = "tsdb";
              object_store = "filesystem";
              schema = "v13";
              index.prefix = "index_";
              index.period = "24h";
            }
          ];
        };

        storage_config = {
          boltdb_shipper = {
            active_index_directory = "${dataDir}/boltdb-shipper-active";
            cache_location = "${dataDir}/boltdb-shipper-cache";
            cache_ttl = "24h";
          };
          tsdb_shipper = {
            active_index_directory = "${dataDir}/tsdb-shipper-active";
            cache_location = "${dataDir}/tsdb-shipper-cache";
            cache_ttl = "24h";
          };
          filesystem.directory = "${dataDir}/chunks";
        };

        limits_config = {
          reject_old_samples = true;
          reject_old_samples_max_age = "168h";
          max_query_lookback = "0s";
        };

        table_manager = {
          retention_deletes_enabled = false;
          retention_period = "0s";
        };

        compactor = {
          working_directory = "${dataDir}/compactor";
          compactor_ring.kvstore.store = "inmemory";
        };
      };
    };

    services.promtail = {
      enable = true;

      configuration = {
        server = {
          http_listen_port = cfg.promtailPort;
          grpc_listen_port = 0;
        };

        positions.filename = "/tmp/positions.yaml";

        client.url = "http://127.0.0.1:${toString cfg.lokiPort}/loki/api/v1/push";

        scrape_configs = [
          {
            job_name = "journal";
            journal = {
              max_age = "12h";
              labels = {
                job = "systemd-journal";
                host = "${config.networking.hostName}";
              };
            };
            relabel_configs = [
              {
                source_labels = [ "__journal__systemd_unit" ];
                target_label = "unit";
              }
            ];
          }
        ];
      };
    };

    services.nginx.statusPage = true;
    services.nginx.virtualHosts."${cfg.domain}" = {
      forceSSL = true;
      enableACME = true;
      locations."/" = {
        proxyPass = "http://127.0.0.1:${toString cfg.grafanaPort}";
        proxyWebsockets = true;
      };
      locations."= /robots.txt" = {
        extraConfig = ''
          add_header Content-Type text/plain;
          return 200 "User-agent: *\nDisallow: /\n";
        '';
      };
    };
  };
}