Add some existing scripts for setting up prometheus

.gitignore

@@ -1 +1,3 @@
-docker-compose.yml
+/docker-compose.yml
+/monitoring/prometheus.yml
+.*.uptodate

monitoring/Dockerfile.prometheus

@@ -0,0 +1,2 @@
+FROM prom/prometheus
+COPY prometheus.yml /etc/prometheus/prometheus.yml

monitoring/Makefile

@@ -0,0 +1,11 @@
+
+
+all: .prometheus.uptodate .nginx.uptodate
+.PHONY: all
+
+.prometheus.uptodate: Dockerfile.prometheus prometheus.yml
+	docker build -t wubloader_prometheus:latest -f Dockerfile.prometheus .
+	touch "$@"
+
+prometheus.yml: prometheus.jsonnet
+	./generate-config

monitoring/docker-compose.yml

@@ -0,0 +1,18 @@
+version: "3"
+services:
+  prometheus:
+    image: "prometheus:latest"
+    command:
+    - "--storage.tsdb.retention=30d"
+    restart: "on-failure"
+    volumes:
+    - "/storage/wubloader-metrics/prometheus:/prometheus"
+    ports:
+    - "9090:9090"
+  grafana:
+    image: "grafana/grafana"
+    restart: "on-failure"
+    volumes:
+    - "/storage/wubloader-metrics/grafana:/var/lib/grafana"
+    ports:
+    - "1338:3000"

monitoring/generate-config

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -eu
+
+# We generate first, and capture the output, to avoid overwriting the file on error.
+# To avoid jsonnet needing to exist locally, we run it in a container.
+output=$(docker run --rm -i sparkprime/jsonnet - < prometheus.jsonnet)
+echo "$output" > prometheus.yml

monitoring/prometheus.jsonnet

@@ -0,0 +1,32 @@
+local hosts = [
+  "toodles.videostrike.team:1337",
+  "http://136.24.9.73:20088",
+  "wubloader.codegunner.com",
+];
+local services = [
+  "restreamer",
+  "downloader",
+  "backfiller",
+  "cutter",
+  "thrimshim",
+  "sheetsync",
+];
+
+{
+  global: {
+    evaluation_interval: "15s",
+    scrape_interval: "15s",
+  },
+  scrape_configs: [
+    {job_name: "prometheus", static_configs: [{targets: ["localhost:9090"]}]},
+  ] + [
+    {
+      job_name: service,
+      metrics_path: "/metrics/%s" % service,
+      static_configs: [{
+        targets: hosts,
+      }],
+    }
+    for service in services
+  ],
+}