From c6bdaab7b346731084cd9463d6b2d33676edb546 Mon Sep 17 00:00:00 2001 From: reidlab Date: Fri, 10 Jan 2025 18:24:03 -0800 Subject: [PATCH] nixify loki config --- hosts/nixos-server-reid/webapps/default.nix | 2 - modules/services/loki-local-config.yml | 63 ----- modules/services/matomo.nix | 44 ++-- modules/services/metrics.nix | 273 +++++++++++++++----- modules/services/promtail.yml | 20 -- 5 files changed, 227 insertions(+), 175 deletions(-) delete mode 100644 modules/services/loki-local-config.yml delete mode 100644 modules/services/promtail.yml diff --git a/hosts/nixos-server-reid/webapps/default.nix b/hosts/nixos-server-reid/webapps/default.nix index 235e1a1..3e2adc0 100755 --- a/hosts/nixos-server-reid/webapps/default.nix +++ b/hosts/nixos-server-reid/webapps/default.nix @@ -6,7 +6,6 @@ in { config = { modules = { services = { - # you should probably keep this on # actually enables nginx, configures acme, # gzip, optimization, proxy, ssl config, @@ -30,7 +29,6 @@ in { metrics = { enable = true; domain = "grafana.reidlab.pink"; - port = 2342; }; }; }; diff --git a/modules/services/loki-local-config.yml b/modules/services/loki-local-config.yml deleted file mode 100644 index dff98e0..0000000 --- a/modules/services/loki-local-config.yml +++ /dev/null @@ -1,63 +0,0 @@ -auth_enabled: false - -server: - http_listen_port: 3100 - -ingester: - lifecycler: - address: 0.0.0.0 - ring: - kvstore: - store: inmemory - replication_factor: 1 - final_sleep: 0s - chunk_idle_period: 1h # Any chunk not receiving new logs in this time will be flushed - max_chunk_age: 1h # All chunks will be flushed when they hit this age, default is 1h - chunk_target_size: 1048576 # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first - chunk_retain_period: 30s # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m) - -schema_config: - configs: - - from: 2023-12-08 - store: boltdb-shipper - object_store: filesystem - schema: v11 - index: - prefix: index_ - period: 24h - - from: 2024-08-24 - store: tsdb - object_store: filesystem - schema: v13 - index: - prefix: index_ - period: 24h - -storage_config: - tsdb_shipper: - active_index_directory: /var/lib/loki/tsdb-shipper-active - cache_location: /var/lib/loki/tsdb-shipper-cache - cache_ttl: 24h # Can be increased for faster performance over longer query periods, uses more disk space - boltdb_shipper: - active_index_directory: /var/lib/loki/boltdb-shipper-active - cache_location: /var/lib/loki/boltdb-shipper-cache - cache_ttl: 24h # Can be increased for faster performance over longer query periods, uses more disk space - filesystem: - directory: /var/lib/loki/chunks - -limits_config: - reject_old_samples: true - reject_old_samples_max_age: 168h - max_query_lookback: 0s - # remove this and boltdb entries when we are complete - allow_structured_metadata: false - -table_manager: - retention_deletes_enabled: false - retention_period: 0s - -compactor: - working_directory: /var/lib/loki - compactor_ring: - kvstore: - store: inmemory diff --git a/modules/services/matomo.nix b/modules/services/matomo.nix index 2a8fd49..17f60cf 100644 --- a/modules/services/matomo.nix +++ b/modules/services/matomo.nix @@ -16,32 +16,30 @@ in { }; config = mkIf cfg.enable { - services = { - matomo = { - enable = true; - package = pkgs.matomo-beta; + services.matomo = { + enable = true; + package = pkgs.matomo-beta; - periodicArchiveProcessing = true; - hostname = cfg.domain; - nginx = { - serverAliases = [ cfg.domain ]; - enableACME = true; - }; + periodicArchiveProcessing = true; + hostname = cfg.domain; + nginx = { + serverAliases = [ cfg.domain ]; + enableACME = true; }; + }; - # matomo doesn't automatically create the database - # just. make sure its named matomo - mysql = { - ensureDatabases = [ "matomo" ]; - ensureUsers = [ - { - name = "matomo"; - ensurePermissions = { - "matomo.*" = "ALL PRIVILEGES"; - }; - } - ]; - }; + # matomo doesn't automatically create the database + # just. make sure its named matomo + services.mysql = { + ensureDatabases = [ "matomo" ]; + ensureUsers = [ + { + name = "matomo"; + ensurePermissions = { + "matomo.*" = "ALL PRIVILEGES"; + }; + } + ]; }; }; } diff --git a/modules/services/metrics.nix b/modules/services/metrics.nix index d750d96..826a1e9 100644 --- a/modules/services/metrics.nix +++ b/modules/services/metrics.nix @@ -10,91 +10,230 @@ in { type = types.str; default = "grafana.reidlab.pink"; }; - port = mkOption { + orgId = mkOption { type = types.int; - default = 2342; + default = 1; + }; + grafanaPort = mkOption { + type = types.int; + default = 3000; + }; + promtailPort = mkOption { + type = types.int; + default = 3001; + }; + lokiPort = mkOption { + type = types.int; + default = 3002; + }; + prometheusPort = mkOption { + type = types.int; + default = 9090; + }; + prometheusExporterPortStart = mkOption { + type = types.int; + default = 9100; }; }; config = mkIf cfg.enable { - systemd.services.promtail = { - description = "promtail, an agent for loki"; - wantedBy = [ "multi-user.target" ]; + services.grafana = { + enable = true; - serviceConfig = { - ExecStart = '' - ${pkgs.grafana-loki}/bin/promtail --config.file ${./promtail.yml} - ''; + settings = { + server = { + domain = cfg.domain; + http_port = cfg.grafanaPort; + http_addr = "127.0.0.1"; + }; + }; + + provision.datasources.settings.datasources = [ + { + orgId = cfg.orgId; + name = "Prometheus"; + type = "prometheus"; + url = "http://127.0.0.1/${toString cfg.prometheusPort}"; + uid = "prometheus"; + isDefault = true; + } + { + orgId = cfg.orgId; + name = "Loki"; + type = "loki"; + url = "http://127.0.0.1/${toString cfg.lokiPort}"; + uid = "loki"; + } + ]; + }; + + services.prometheus = let + mkPort = offset: cfg.prometheusExporterPortStart + offset; + + ports = { + node = mkPort 0; + nginx = mkPort 1; + }; + in { + enable = true; + + port = cfg.prometheusPort; + + exporters = { + node = { + enable = true; + enabledCollectors = [ "systemd" ]; + port = ports.node; + }; + nginx = { + enable = true; + port = ports.nginx; + }; + }; + + scrapeConfigs = [ + { + job_name = "node"; + static_configs = [{ + targets = [ "127.0.0.1:${toString ports.node}" ]; + }]; + } + { + job_name = "nginx"; + static_configs = [{ + targets = [ "127.0.0.1:${toString ports.nginx}" ]; + }]; + } + ]; + }; + + services.loki = { + enable = true; + + configuration = let + dataDir = config.services.loki.dataDir; + in { + auth_enabled = false; + + server.http_listen_port = cfg.lokiPort; + + ingester = { + lifecycler = { + address = "127.0.0.1"; + ring = { + kvstore.store = "inmemory"; + replication_factor = 1; + }; + final_sleep = "0s"; + }; + chunk_idle_period = "5m"; + chunk_retain_period = "30s"; + }; + + schema_config = { + configs = [ + { + from = "2023-12-08"; + store = "boltdb-shipper"; + object_store = "filesystem"; + schema = "v11"; + index.prefix = "index_"; + index.period = "24h"; + } + { + from = "2024-08-24"; + store = "tsdb"; + object_store = "filesystem"; + schema = "v13"; + index.prefix = "index_"; + index.period = "24h"; + } + ]; + }; + + storage_config = { + boltdb_shipper = { + active_index_directory = "${dataDir}/boltdb-shipper-active"; + cache_location = "${dataDir}/boltdb-shipper-cache"; + cache_ttl = "24h"; + }; + tsdb_shipper = { + active_index_directory = "${dataDir}/tsdb-shipper-active"; + cache_location = "${dataDir}/tsdb-shipper-cache"; + cache_ttl = "24h"; + }; + filesystem.directory = "${dataDir}/chunks"; + }; + + limits_config = { + reject_old_samples = true; + reject_old_samples_max_age = "168h"; + max_query_lookback = "0s"; + }; + + chunk_store_config = { + max_look_back_period = "0s"; + }; + + table_manager = { + retention_deletes_enabled = false; + retention_period = "0s"; + }; + + compactor = { + working_directory = "${dataDir}/compactor"; + compactor_ring.kvstore.store = "inmemory"; + }; }; }; - services = { - grafana = { - enable = true; - settings = { - server = { - domain = cfg.domain; - http_port = cfg.port; - http_addr = "127.0.0.1"; - }; - }; - }; + services.promtail = { + enable = true; - prometheus = let - ports = { - base = 9001; - node = 9002; - nginx = 9003; - }; - in { - enable = true; - port = ports.base; - - exporters = { - node = { - enable = true; - enabledCollectors = [ "systemd" ]; - port = ports.node; - }; - nginx = { - enable = true; - port = ports.nginx; - }; + configuration = { + server = { + http_listen_port = cfg.promtailPort; + grpc_listen_port = 0; }; - scrapeConfigs = [ + positions.filename = "/tmp/positions.yaml"; + + client.url = "http://127.0.0.1:${toString cfg.lokiPort}/loki/api/v1/push"; + + scrape_configs = [ { - job_name = "nixos-server-reid"; - static_configs = [{ - targets = [ - "127.0.0.1:${toString ports.node}" - "127.0.0.1:${toString ports.nginx}" - ]; - }]; + job_name = "journal"; + journal = { + max_age = "12h"; + labels = { + job = "systemd-journal"; + host = "${config.networking.hostName}"; + }; + }; + relabel_configs = [ + { + source_labels = [ "__journal__systemd_unit" ]; + target_label = "unit"; + } + ]; } ]; }; + }; - loki = { - enable = true; - configFile = ./loki-local-config.yml; + services.nginx.statusPage = true; + services.nginx.virtualHosts."${cfg.domain}" = { + forceSSL = true; + enableACME = true; + locations."/" = { + proxyPass = "http://127.0.0.1:${toString cfg.grafanaPort}"; + proxyWebsockets = true; }; - - nginx.statusPage = true; - - nginx.virtualHosts."${cfg.domain}" = { - forceSSL = true; - enableACME = true; - locations."/" = { - proxyPass = "http://127.0.0.1:${toString cfg.port}"; - proxyWebsockets = true; - }; - locations."= /robots.txt" ={ - extraConfig = '' - add_header Content-Type text/plain; - return 200 "User-agent: *\nDisallow: /\n"; - ''; - }; + locations."= /robots.txt" = { + extraConfig = '' + add_header Content-Type text/plain; + return 200 "User-agent: *\nDisallow: /\n"; + ''; }; }; }; diff --git a/modules/services/promtail.yml b/modules/services/promtail.yml deleted file mode 100644 index 00eb943..0000000 --- a/modules/services/promtail.yml +++ /dev/null @@ -1,20 +0,0 @@ -server: - http_listen_port: 28183 - grpc_listen_port: 0 - -positions: - filename: /tmp/positions.yaml - -clients: - - url: http://127.0.0.1:3100/loki/api/v1/push - -scrape_configs: - - job_name: journal - journal: - max_age: 12h - labels: - job: systemd-journal - host: nixos-server-reid - relabel_configs: - - source_labels: ["__journal__systemd_unit"] - target_label: "unit"