commit f9684532bd8a617bd1e6f2cc6f15a18781d46526
Author: thomasabishop <tactonbishop@gmail.com>
Date:   Sun Mar 9 13:37:47 2025 +0000

    services: set up grafana

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..665da45
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+.env
+.env.*
diff --git a/services/grafana/README.md b/services/grafana/README.md
new file mode 100644
index 0000000..db55f6f
--- /dev/null
+++ b/services/grafana/README.md
@@ -0,0 +1,25 @@
+## Local development
+
+```sh
+docker-compose-up -d
+```
+
+### Prometheus
+
+The tool that serves as the data source for the Grafana representation layer.
+Adds timestamps to enable series data. Includes query language (PromQL) that
+Grafana uses to retrieve and transform data. Also provides alerting.
+
+> In a typical implementation, Prometheus collects the raw metrics data from various systems, and Grafana connects to Prometheus as a data source to create dashboards with visualizations of those metrics.
+
+### Node exporter
+
+Prometheus has different plug-ins called "exporters". These expose metrics from
+different processes and services and present them in a format that Prometheus
+can scrape. Node Exporter is one of these - it exposes hardware and OS metrics
+(such as CPU use and load, memory, disk capacity etc).
+
+### Loki
+
+Loki is for storing and displaying logs. This is part of Grafana. Promtail (also
+part of Grafana) is for collecting the logs.
diff --git a/services/grafana/docker-compose.yml b/services/grafana/docker-compose.yml
new file mode 100644
index 0000000..7a199b7
--- /dev/null
+++ b/services/grafana/docker-compose.yml
@@ -0,0 +1,48 @@
+services:
+  prometheus:
+    image: prom/prometheus:latest
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+      - prometheus_data:/prometheus
+    restart: unless-stopped
+
+  node-exporter:
+    image: prom/node-exporter:latest
+    restart: unless-stopped
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+    command:
+      - "--path.procfs=/host/proc"
+      - "--path.rootfs=/rootfs"
+      - "--path.sysfs=/host/sys"
+
+  loki:
+    image: grafana/loki:latest
+    restart: unless-stopped
+    volumes:
+      - loki_data:/loki
+
+  promtail:
+    image: grafana/promtail:latest
+    restart: unless-stopped
+    volumes:
+      - ./promtail/promtail-config.yml:/etc/promtail/config.yml
+      - /var/log:/var/log
+
+  grafana:
+    image: grafana/grafana:latest
+    restart: unless-stopped
+    volumes:
+      - grafana_data:/var/lib/grafana
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD}
+      - GF_USERS_ALLOW_SIGN_UP=false
+    ports:
+      - "3000:3000" # For local testing; remove in production with nginx
+
+volumes:
+  prometheus_data:
+  loki_data:
+  grafana_data:
diff --git a/services/grafana/prometheus/prometheus.yml b/services/grafana/prometheus/prometheus.yml
new file mode 100644
index 0000000..ff630f5
--- /dev/null
+++ b/services/grafana/prometheus/prometheus.yml
@@ -0,0 +1,11 @@
+global:
+  scrape_interval: 15s
+
+scrape_configs:
+  - job_name: 'prometheus'
+    static_configs:
+      - targets: ['localhost:9090']
+
+  - job_name: 'node'
+    static_configs:
+      - targets: ['node-exporter:9100']
diff --git a/services/grafana/promtail/promtail-config.yml b/services/grafana/promtail/promtail-config.yml
new file mode 100644
index 0000000..93a74d6
--- /dev/null
+++ b/services/grafana/promtail/promtail-config.yml
@@ -0,0 +1,17 @@
+server:
+  http_listen_port: 9080
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: system
+    static_configs:
+      - targets:
+          - localhost
+        labels:
+          job: varlogs
+          __path__: /var/log/*log