{ config, lib, pkgs, options, ... }: with lib; let cfg = config.modules.services.metrics; in { options.modules.services.metrics = { enable = mkEnableOption "enable grafana with prometheus, alloy, and loki"; domain = mkOption { type = types.str; default = "grafana.reidlab.pink"; }; grafanaSocket = mkOption { type = types.str; default = "/run/grafana/grafana.sock"; }; alloyPort = mkOption { type = types.int; default = 12345; }; lokiPort = mkOption { type = types.int; default = 3100; }; prometheusPort = mkOption { type = types.int; default = 9090; }; prometheusExporterPortStart = mkOption { type = types.int; default = 9100; }; }; config = mkIf cfg.enable { services.grafana = { enable = true; settings = { server = { domain = cfg.domain; protocol = "socket"; socket = cfg.grafanaSocket; }; database.type = "sqlite3"; # TODO: use postgres? # use default secret key # TODO: maybe don't do that idk security.secret_key = "SW2YcwTIb9zpOOhoPsMm"; }; provision.datasources.settings = { prune = true; # delete datasources if not in provision file datasources = [ { name = "Prometheus"; type = "prometheus"; url = "http://127.0.0.1:${toString cfg.prometheusPort}"; uid = "prometheus"; isDefault = true; } { name = "Loki"; type = "loki"; url = "http://127.0.0.1:${toString cfg.lokiPort}"; uid = "loki"; } ]; }; }; services.prometheus = let mkPort = offset: cfg.prometheusExporterPortStart + offset; ports = { node = mkPort 0; nginx = mkPort 1; smartctl = mkPort 2; }; in { enable = true; port = cfg.prometheusPort; exporters = { node = { enable = true; enabledCollectors = [ "systemd" ]; port = ports.node; }; nginx = { enable = true; port = ports.nginx; }; smartctl = { enable = true; port = ports.smartctl; }; }; scrapeConfigs = let commonLabels = { hostname = config.networking.hostName; }; in [ { job_name = "node"; static_configs = [{ targets = [ "127.0.0.1:${toString ports.node}" ]; labels = commonLabels; }]; } { job_name = "nginx"; static_configs = [{ targets = [ "127.0.0.1:${toString ports.nginx}" ]; labels = commonLabels; }]; } { job_name = "smartctl"; static_configs = [{ targets = [ "127.0.0.1:${toString ports.smartctl}" ]; labels = commonLabels; }]; } ]; }; services.loki = { enable = true; configuration = let dataDir = config.services.loki.dataDir; in { auth_enabled = false; server.http_listen_port = cfg.lokiPort; ingester = { lifecycler = { address = "127.0.0.1"; ring = { kvstore.store = "inmemory"; replication_factor = 1; }; final_sleep = "0s"; }; chunk_idle_period = "5m"; chunk_retain_period = "30s"; }; schema_config = { configs = [ { from = "2023-12-08"; store = "boltdb-shipper"; object_store = "filesystem"; schema = "v11"; index.prefix = "index_"; index.period = "24h"; } { from = "2024-08-24"; store = "tsdb"; object_store = "filesystem"; schema = "v13"; index.prefix = "index_"; index.period = "24h"; } ]; }; storage_config = { boltdb_shipper = { active_index_directory = "${dataDir}/boltdb-shipper-active"; cache_location = "${dataDir}/boltdb-shipper-cache"; cache_ttl = "24h"; }; tsdb_shipper = { active_index_directory = "${dataDir}/tsdb-shipper-active"; cache_location = "${dataDir}/tsdb-shipper-cache"; cache_ttl = "24h"; }; filesystem.directory = "${dataDir}/chunks"; }; limits_config = { reject_old_samples = true; reject_old_samples_max_age = "168h"; max_query_lookback = "0s"; }; table_manager = { retention_deletes_enabled = false; retention_period = "0s"; }; compactor = { working_directory = "${dataDir}/compactor"; compactor_ring.kvstore.store = "inmemory"; }; }; }; services.alloy = { enable = true; extraFlags = [ "--server.http.listen-addr=127.0.0.1:${toString cfg.alloyPort}" "--disable-reporting" # disable telemetry ]; configPath = pkgs.writeText "config.alloy" '' loki.relabel "journal" { forward_to = [] rule { source_labels = ["__journal__systemd_unit"] target_label = "unit" } rule { source_labels = ["__journal__hostname"] target_label = "hostname" } } loki.source.journal "journal" { forward_to = [loki.write.default.receiver] max_age = "12h" labels = {job = "systemd-journal"} relabel_rules = loki.relabel.journal.rules } loki.write "default" { endpoint { url = "http://localhost:${toString cfg.lokiPort}/loki/api/v1/push" } } ''; }; # fix unix sockets systemd.services.nginx.serviceConfig.SupplementaryGroups = [ "grafana" ]; services.nginx.statusPage = true; services.nginx.virtualHosts."${cfg.domain}" = { forceSSL = true; enableACME = true; locations."/" = { proxyPass = "http://unix:${cfg.grafanaSocket}"; proxyWebsockets = true; }; locations."= /robots.txt" = { extraConfig = '' add_header Content-Type text/plain; return 200 "User-agent: *\nDisallow: /\n"; ''; }; }; }; }