{ config, lib, pkgs, options, ... }: with lib; let cfg = config.modules.services.metrics; in { options.modules.services.metrics = { enable = mkEnableOption "enable grafana with loki, prometheus, and promtail"; domain = mkOption { type = types.str; default = "grafana.reidlab.pink"; }; orgId = mkOption { type = types.int; default = 1; }; grafanaPort = mkOption { type = types.int; default = 3000; }; promtailPort = mkOption { type = types.int; default = 3001; }; lokiPort = mkOption { type = types.int; default = 3002; }; prometheusPort = mkOption { type = types.int; default = 9090; }; prometheusExporterPortStart = mkOption { type = types.int; default = 9100; }; }; config = mkIf cfg.enable { services.grafana = { enable = true; settings = { server = { domain = cfg.domain; http_port = cfg.grafanaPort; http_addr = "127.0.0.1"; }; }; provision.datasources.settings = { datasources = [ { orgId = cfg.orgId; name = "Prometheus"; type = "prometheus"; url = "http://127.0.0.1:${toString cfg.prometheusPort}"; uid = "prometheus"; isDefault = true; } { orgId = cfg.orgId; name = "Loki"; type = "loki"; url = "http://127.0.0.1:${toString cfg.lokiPort}"; uid = "loki"; } ]; deleteDatasources = [ { orgId = cfg.orgId; name = "Prometheus"; } { orgId = cfg.orgId; name = "Loki"; } ]; }; }; services.prometheus = let mkPort = offset: cfg.prometheusExporterPortStart + offset; ports = { node = mkPort 0; nginx = mkPort 1; }; in { enable = true; port = cfg.prometheusPort; exporters = { node = { enable = true; enabledCollectors = [ "systemd" ]; port = ports.node; }; nginx = { enable = true; port = ports.nginx; }; }; scrapeConfigs = [ { job_name = "node"; static_configs = [{ targets = [ "127.0.0.1:${toString ports.node}" ]; }]; } { job_name = "nginx"; static_configs = [{ targets = [ "127.0.0.1:${toString ports.nginx}" ]; }]; } ]; }; services.loki = { enable = true; configuration = let dataDir = config.services.loki.dataDir; in { auth_enabled = false; server.http_listen_port = cfg.lokiPort; ingester = { lifecycler = { address = "127.0.0.1"; ring = { kvstore.store = "inmemory"; replication_factor = 1; }; final_sleep = "0s"; }; chunk_idle_period = "5m"; chunk_retain_period = "30s"; }; schema_config = { configs = [ { from = "2023-12-08"; store = "boltdb-shipper"; object_store = "filesystem"; schema = "v11"; index.prefix = "index_"; index.period = "24h"; } { from = "2024-08-24"; store = "tsdb"; object_store = "filesystem"; schema = "v13"; index.prefix = "index_"; index.period = "24h"; } ]; }; storage_config = { boltdb_shipper = { active_index_directory = "${dataDir}/boltdb-shipper-active"; cache_location = "${dataDir}/boltdb-shipper-cache"; cache_ttl = "24h"; }; tsdb_shipper = { active_index_directory = "${dataDir}/tsdb-shipper-active"; cache_location = "${dataDir}/tsdb-shipper-cache"; cache_ttl = "24h"; }; filesystem.directory = "${dataDir}/chunks"; }; limits_config = { reject_old_samples = true; reject_old_samples_max_age = "168h"; max_query_lookback = "0s"; }; table_manager = { retention_deletes_enabled = false; retention_period = "0s"; }; compactor = { working_directory = "${dataDir}/compactor"; compactor_ring.kvstore.store = "inmemory"; }; }; }; services.promtail = { enable = true; configuration = { server = { http_listen_port = cfg.promtailPort; grpc_listen_port = 0; }; positions.filename = "/tmp/positions.yaml"; client.url = "http://127.0.0.1:${toString cfg.lokiPort}/loki/api/v1/push"; scrape_configs = [ { job_name = "journal"; journal = { max_age = "12h"; labels = { job = "systemd-journal"; host = "${config.networking.hostName}"; }; }; relabel_configs = [ { source_labels = [ "__journal__systemd_unit" ]; target_label = "unit"; } ]; } ]; }; }; services.nginx.statusPage = true; services.nginx.virtualHosts."${cfg.domain}" = { forceSSL = true; enableACME = true; locations."/" = { proxyPass = "http://127.0.0.1:${toString cfg.grafanaPort}"; proxyWebsockets = true; }; locations."= /robots.txt" = { extraConfig = '' add_header Content-Type text/plain; return 200 "User-agent: *\nDisallow: /\n"; ''; }; }; }; }