nixify loki config

hosts/nixos-server-reid/webapps/default.nix

@@ -6,7 +6,6 @@ in {
   config = {
     modules = {
       services = {
-
         # you should probably keep this on
         # actually enables nginx, configures acme,
         # gzip, optimization, proxy, ssl config,
@@ -30,7 +29,6 @@ in {
         metrics = {
           enable = true;
           domain = "grafana.reidlab.pink";
-          port = 2342;
         };
       };
     };

modules/services/loki-local-config.yml

@@ -1,63 +0,0 @@
-auth_enabled: false
-
-server:
-  http_listen_port: 3100
-
-ingester:
-  lifecycler:
-    address: 0.0.0.0
-    ring:
-      kvstore:
-        store: inmemory
-      replication_factor: 1
-    final_sleep: 0s
-  chunk_idle_period: 1h       # Any chunk not receiving new logs in this time will be flushed
-  max_chunk_age: 1h           # All chunks will be flushed when they hit this age, default is 1h
-  chunk_target_size: 1048576  # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first
-  chunk_retain_period: 30s    # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)
-
-schema_config:
-  configs:
-    - from: 2023-12-08
-      store: boltdb-shipper
-      object_store: filesystem
-      schema: v11
-      index:
-        prefix: index_
-        period: 24h
-    - from: 2024-08-24
-      store: tsdb
-      object_store: filesystem
-      schema: v13
-      index:
-        prefix: index_
-        period: 24h
-
-storage_config:
-  tsdb_shipper:
-    active_index_directory: /var/lib/loki/tsdb-shipper-active
-    cache_location: /var/lib/loki/tsdb-shipper-cache
-    cache_ttl: 24h # Can be increased for faster performance over longer query periods, uses more disk space
-  boltdb_shipper:
-    active_index_directory: /var/lib/loki/boltdb-shipper-active
-    cache_location: /var/lib/loki/boltdb-shipper-cache
-    cache_ttl: 24h # Can be increased for faster performance over longer query periods, uses more disk space
-  filesystem:
-    directory: /var/lib/loki/chunks
-
-limits_config:
-  reject_old_samples: true
-  reject_old_samples_max_age: 168h
-  max_query_lookback: 0s
-  # remove this and boltdb entries when we are complete
-  allow_structured_metadata: false
-
-table_manager:
-  retention_deletes_enabled: false
-  retention_period: 0s
-
-compactor:
-  working_directory: /var/lib/loki
-  compactor_ring:
-    kvstore:
-      store: inmemory

modules/services/matomo.nix

@@ -16,32 +16,30 @@ in {
   };
 
   config = mkIf cfg.enable {
-    services = {
+    services.matomo = {
-      matomo = {
+      enable = true;
-        enable = true;
+      package = pkgs.matomo-beta;
-        package = pkgs.matomo-beta;
 
-        periodicArchiveProcessing = true;
+      periodicArchiveProcessing = true;
-        hostname = cfg.domain;
+      hostname = cfg.domain;
-        nginx = {
+      nginx = {
-          serverAliases = [ cfg.domain ];
+        serverAliases = [ cfg.domain ];
-          enableACME = true;
+        enableACME = true;
-        };
       };
+    };
 
-      # matomo doesn't automatically create the database
+    # matomo doesn't automatically create the database
-      # just. make sure its named matomo
+    # just. make sure its named matomo
-      mysql = {
+    services.mysql = {
-        ensureDatabases = [ "matomo" ];
+      ensureDatabases = [ "matomo" ];
-        ensureUsers = [
+      ensureUsers = [
-          {
+        {
-            name = "matomo";
+          name = "matomo";
-            ensurePermissions = {
+          ensurePermissions = {
-              "matomo.*" = "ALL PRIVILEGES";
+            "matomo.*" = "ALL PRIVILEGES";
-            };
+          };
-          }
+        }
-        ];
+      ];
-      };
     };
   };
 }

modules/services/metrics.nix

@@ -10,91 +10,238 @@ in {
       type = types.str;
       default = "grafana.reidlab.pink";
     };
-    port = mkOption {
+    orgId = mkOption {
       type = types.int;
-      default = 2342;
+      default = 1;
+    };
+    grafanaPort = mkOption {
+      type = types.int;
+      default = 3000;
+    };
+    promtailPort = mkOption {
+      type = types.int;
+      default = 3001;
+    };
+    lokiPort = mkOption {
+      type = types.int;
+      default = 3002;
+    };
+    prometheusPort = mkOption {
+      type = types.int;
+      default = 9090;
+    };
+    prometheusExporterPortStart = mkOption {
+      type = types.int;
+      default = 9100;
     };
   };
 
   config = mkIf cfg.enable {
-    systemd.services.promtail = {
+    services.grafana = {
-      description = "promtail, an agent for loki";
+      enable = true;
-      wantedBy = [ "multi-user.target" ];
 
-      serviceConfig = {
+      settings = {
-        ExecStart = ''
+        server = {
-          ${pkgs.grafana-loki}/bin/promtail --config.file ${./promtail.yml}
+          domain = cfg.domain;
-        '';
+          http_port = cfg.grafanaPort;
-      };
+          http_addr = "127.0.0.1";
-    };
-    services = {
-      grafana = {
-        enable = true;
-
-        settings = {
-          server = {
-            domain = cfg.domain;
-            http_port = cfg.port;
-            http_addr = "127.0.0.1";
-          };
         };
       };
 
-      prometheus = let
+      provision.datasources.settings = {
-        ports = {
+        datasources = [
-          base = 9001;
-          node = 9002;
-          nginx = 9003;
-        };
-      in {
-        enable = true;
-        port = ports.base;
-
-        exporters = {
-          node = {
-            enable = true;
-            enabledCollectors = [ "systemd" ];
-            port = ports.node;
-          };
-          nginx = {
-            enable = true;
-            port = ports.nginx;
-          };
-        };
-
-        scrapeConfigs = [
           {
-            job_name = "nixos-server-reid";
+            orgId = cfg.orgId;
-            static_configs = [{
+            name = "Prometheus";
-              targets = [
+            type = "prometheus";
-                "127.0.0.1:${toString ports.node}"
+            url = "http://127.0.0.1:${toString cfg.prometheusPort}";
-                "127.0.0.1:${toString ports.nginx}"
+            uid = "prometheus";
-              ];
+            isDefault = true;
-            }];
+          }
+          {
+            orgId = cfg.orgId;
+            name = "Loki";
+            type = "loki";
+            url = "http://127.0.0.1:${toString cfg.lokiPort}";
+            uid = "loki";
+          }
+        ];
+        deleteDatasources = [
+          {
+            orgId = cfg.orgId;
+            name = "Prometheus";
+          }
+          {
+            orgId = cfg.orgId;
+            name = "Loki";
           }
         ];
       };
+    };
 
-      loki = {
+    services.prometheus = let
-        enable = true;
+      mkPort = offset: cfg.prometheusExporterPortStart + offset;
-        configFile = ./loki-local-config.yml;
+
+      ports = {
+        node = mkPort 0;
+        nginx = mkPort 1;
+      };
+    in {
+      enable = true;
+
+      port = cfg.prometheusPort;
+
+      exporters = {
+        node = {
+          enable = true;
+          enabledCollectors = [ "systemd" ];
+          port = ports.node;
+        };
+        nginx = {
+          enable = true;
+          port = ports.nginx;
+        };
       };
 
-      nginx.statusPage = true;
+      scrapeConfigs = [
+        {
+          job_name = "node";
+          static_configs = [{
+            targets = [ "127.0.0.1:${toString ports.node}" ];
+          }];
+        }
+        {
+          job_name = "nginx";
+          static_configs = [{
+            targets = [ "127.0.0.1:${toString ports.nginx}" ];
+          }];
+        }
+      ];
+    };
 
-      nginx.virtualHosts."${cfg.domain}" = {
+    services.loki = {
-        forceSSL = true;
+      enable = true;
-        enableACME = true;
+
-        locations."/" = {
+      configuration = let
-          proxyPass = "http://127.0.0.1:${toString cfg.port}";
+        dataDir = config.services.loki.dataDir;
-          proxyWebsockets = true;
+      in {
+        auth_enabled = false;
+
+        server.http_listen_port = cfg.lokiPort;
+
+        ingester = {
+          lifecycler = {
+            address = "127.0.0.1";
+            ring = {
+              kvstore.store = "inmemory";
+              replication_factor = 1;
+            };
+            final_sleep = "0s";
+          };
+          chunk_idle_period = "5m";
+          chunk_retain_period = "30s";
         };
-        locations."= /robots.txt" ={
+
-          extraConfig = ''
+        schema_config = {
-            add_header Content-Type text/plain;
+          configs = [
-            return 200 "User-agent: *\nDisallow: /\n";
+            {
-          '';
+              from = "2023-12-08";
+              store = "boltdb-shipper";
+              object_store = "filesystem";
+              schema = "v11";
+              index.prefix = "index_";
+              index.period = "24h";
+            }
+            {
+              from = "2024-08-24";
+              store = "tsdb";
+              object_store = "filesystem";
+              schema = "v13";
+              index.prefix = "index_";
+              index.period = "24h";
+            }
+          ];
         };
+
+        storage_config = {
+          boltdb_shipper = {
+            active_index_directory = "${dataDir}/boltdb-shipper-active";
+            cache_location = "${dataDir}/boltdb-shipper-cache";
+            cache_ttl = "24h";
+          };
+          tsdb_shipper = {
+            active_index_directory = "${dataDir}/tsdb-shipper-active";
+            cache_location = "${dataDir}/tsdb-shipper-cache";
+            cache_ttl = "24h";
+          };
+          filesystem.directory = "${dataDir}/chunks";
+        };
+
+        limits_config = {
+          reject_old_samples = true;
+          reject_old_samples_max_age = "168h";
+          max_query_lookback = "0s";
+        };
+
+        table_manager = {
+          retention_deletes_enabled = false;
+          retention_period = "0s";
+        };
+
+        compactor = {
+          working_directory = "${dataDir}/compactor";
+          compactor_ring.kvstore.store = "inmemory";
+        };
+      };
+    };
+
+    services.promtail = {
+      enable = true;
+
+      configuration = {
+        server = {
+          http_listen_port = cfg.promtailPort;
+          grpc_listen_port = 0;
+        };
+
+        positions.filename = "/tmp/positions.yaml";
+
+        client.url = "http://127.0.0.1:${toString cfg.lokiPort}/loki/api/v1/push";
+
+        scrape_configs = [
+          {
+            job_name = "journal";
+            journal = {
+              max_age = "12h";
+              labels = {
+                job = "systemd-journal";
+                host = "${config.networking.hostName}";
+              };
+            };
+            relabel_configs = [
+              {
+                source_labels = [ "__journal__systemd_unit" ];
+                target_label = "unit";
+              }
+            ];
+          }
+        ];
+      };
+    };
+
+    services.nginx.statusPage = true;
+    services.nginx.virtualHosts."${cfg.domain}" = {
+      forceSSL = true;
+      enableACME = true;
+      locations."/" = {
+        proxyPass = "http://127.0.0.1:${toString cfg.grafanaPort}";
+        proxyWebsockets = true;
+      };
+      locations."= /robots.txt" = {
+        extraConfig = ''
+          add_header Content-Type text/plain;
+          return 200 "User-agent: *\nDisallow: /\n";
+        '';
       };
     };
   };

modules/services/promtail.yml

@@ -1,20 +0,0 @@
-server:
-  http_listen_port: 28183
-  grpc_listen_port: 0
-
-positions:
-  filename: /tmp/positions.yaml
-
-clients:
-  - url: http://127.0.0.1:3100/loki/api/v1/push
-
-scrape_configs:
-  - job_name: journal
-    journal:
-      max_age: 12h
-      labels:
-        job: systemd-journal
-        host: nixos-server-reid
-    relabel_configs:
-      - source_labels: ["__journal__systemd_unit"]
-        target_label: "unit"