pubbot: Add more detail to logging

Wrap raw pubnub messages in a metadata object containing:
- time
- our hostname
- our pid
- any inferred information

Plus make a message-less log write on startup so we know where there might be gaps.

Also do more error handling so that we don't completely lose messages on error.

zulip_bots/zulip_bots/pubbot.py

@@ -1,6 +1,8 @@
 
 import json
 import logging
+import os
+import socket
 import time
 
 from .config import get_config
@@ -49,7 +51,7 @@ def get_giveaway():
 	return g
 
 
-def main(conf_file, message_log_file):
+def main(conf_file, message_log_file, name=socket.gethostname()):
 	"""Config:
 		zulip_url
 		zulip_email
@@ -59,15 +61,41 @@ def main(conf_file, message_log_file):
 
 	config = get_config(conf_file)
 	client = Client(config["zulip_url"], config["zulip_email"], config["zulip_api_key"])
+
 	message_log = open(message_log_file, "a")
+	def write_log(log):
+		message_log.write(json.dumps(log) + '\n')
+
+	write_log({
+		"type": "startup",
+		"host": name,
+		"pid": os.getpid(),
+		"time": time.time(),
+	})
 
 	prizes = {}
 	total = None
 	for msg in stream():
-		message_log.write(json.dumps(msg) + '\n')
+		log = {
+			"type": "unknown",
+			"host": name,
+			"pid": os.getpid(),
+			"time": time.time(),
+			"message": msg,
+		}
+
+		try:
+			try:
+				message_time = float(msg["p"]["t"]) / 10000000
+			except (KeyError, ValueError):
+				message_time = None
+
+			log["message_time"] = message_time
 
 			if msg["c"] == "db_total":
+				log["type"] == "total"
 				increase = None if total is None else msg["d"] - total
+				log["increase"] = increase
 				increase_str = "" if increase is None else " (+${:.2f})".format(msg["d"] - total)
 				giveaway = get_giveaway()
 				entries_str = ""
@@ -75,6 +103,8 @@ def main(conf_file, message_log_file):
 					amount = giveaway["amount"]
 					if (increase + 0.005) % amount < 0.01:
 						entries = int((increase + 0.005) / amount)
+						log["giveaway_amount"] = amount
+						log["giveaway_entries"] = entries
 						entries_str = " ({} entries of ${:.2f})".format(entries, amount)
 				logging.info("New donation total: {}{}{}".format(msg["d"], increase_str, entries_str))
 				client.send_to_stream("bot-spam", "Donation Firehose", "Donation total is now ${:.2f}{}{}".format(msg["d"], increase_str, entries_str))
@@ -83,15 +113,19 @@ def main(conf_file, message_log_file):
 				total = msg["d"]
 
 			elif msg["c"] == "db_vue" and msg["d"].get("channel").startswith("prize:"):
+				log["type"] = "prize"
 				d = msg["d"]
 				prize_id = msg["d"]["channel"].split(":")[1]
 				data = d["data"]
 				logging.info(f"Prize update for {prize_id}: {data}")
 				if prize_id not in prizes:
 					prizes[prize_id] = get_prize(prize_id)
+				log["prize_id"] = prize_id
 				if "bidder" in data and "bid" in data:
+					log["bidder"] = data["bidder"]
+					log["bid"] = data["bid"]
 					client.send_to_stream("bot-spam", "Bids", "At <time:{time}>, {bidder} bid ${bid:.2f} for [{title}](https://desertbus.org/prize/{prize_id})".format(
-					time = float(msg["p"]["t"]) / 10000000,
+						time = message_time,
 						bidder = data["bidder"],
 						bid = data["bid"],
 						title = prizes[prize_id]["title"],
@@ -100,6 +134,11 @@ def main(conf_file, message_log_file):
 
 			else:
 				logging.warning("Unknown message: {}".format(msg))
+		except Exception:
+			logging.exception(f"Failed to handle message {msg}")
+			log["type"] = "error"
+
+		write_log(log)
 
 
 if __name__ == '__main__':