services: set up grafana

.gitignore

@@ -0,0 +1,2 @@
+.env
+.env.*

services/grafana/README.md

@@ -0,0 +1,25 @@
+## Local development
+
+```sh
+docker-compose-up -d
+```
+
+### Prometheus
+
+The tool that serves as the data source for the Grafana representation layer.
+Adds timestamps to enable series data. Includes query language (PromQL) that
+Grafana uses to retrieve and transform data. Also provides alerting.
+
+> In a typical implementation, Prometheus collects the raw metrics data from various systems, and Grafana connects to Prometheus as a data source to create dashboards with visualizations of those metrics.
+
+### Node exporter
+
+Prometheus has different plug-ins called "exporters". These expose metrics from
+different processes and services and present them in a format that Prometheus
+can scrape. Node Exporter is one of these - it exposes hardware and OS metrics
+(such as CPU use and load, memory, disk capacity etc).
+
+### Loki
+
+Loki is for storing and displaying logs. This is part of Grafana. Promtail (also
+part of Grafana) is for collecting the logs.

services/grafana/docker-compose.yml

@@ -0,0 +1,48 @@
+services:
+  prometheus:
+    image: prom/prometheus:latest
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+      - prometheus_data:/prometheus
+    restart: unless-stopped
+
+  node-exporter:
+    image: prom/node-exporter:latest
+    restart: unless-stopped
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+    command:
+      - "--path.procfs=/host/proc"
+      - "--path.rootfs=/rootfs"
+      - "--path.sysfs=/host/sys"
+
+  loki:
+    image: grafana/loki:latest
+    restart: unless-stopped
+    volumes:
+      - loki_data:/loki
+
+  promtail:
+    image: grafana/promtail:latest
+    restart: unless-stopped
+    volumes:
+      - ./promtail/promtail-config.yml:/etc/promtail/config.yml
+      - /var/log:/var/log
+
+  grafana:
+    image: grafana/grafana:latest
+    restart: unless-stopped
+    volumes:
+      - grafana_data:/var/lib/grafana
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD}
+      - GF_USERS_ALLOW_SIGN_UP=false
+    ports:
+      - "3000:3000" # For local testing; remove in production with nginx
+
+volumes:
+  prometheus_data:
+  loki_data:
+  grafana_data:

services/grafana/prometheus/prometheus.yml

@@ -0,0 +1,11 @@
+global:
+  scrape_interval: 15s
+
+scrape_configs:
+  - job_name: 'prometheus'
+    static_configs:
+      - targets: ['localhost:9090']
+
+  - job_name: 'node'
+    static_configs:
+      - targets: ['node-exporter:9100']

services/grafana/promtail/promtail-config.yml

@@ -0,0 +1,17 @@
+server:
+  http_listen_port: 9080
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: system
+    static_configs:
+      - targets:
+          - localhost
+        labels:
+          job: varlogs
+          __path__: /var/log/*log