downloader: Track timestamp of latest segment

This gives us a "stream delay" metric.

Prom doesn't have any native way to check the current value of a metric,
in order to take max(). It only offers increment and set.

We reach into some internals to do this in a hacky way,
but the cleaner way would be to track the value ourselves and have a prom callback
that gets the value.

Sigh, I hate this prom library. I might write my own that's less dumb.

downloader/downloader/main.py

@@ -29,6 +29,12 @@ segments_downloaded = prom.Counter(
 	["partial", "stream", "variant"],
 )
 
+latest_segment = prom.Gauge(
+	"latest_segment",
+	"Timestamp of the time of the newest segment fully downloaded",
+	["stream", "variant"],
+)
+
 
 class TimedOutError(Exception):
 	pass
@@ -542,6 +548,11 @@ class SegmentGetter(object):
 			self.logger.debug("Saving completed segment {} as {}".format(temp_path, full_path))
 			common.rename(temp_path, full_path)
 			segments_downloaded.labels(partial="False", stream=self.channel, variant=self.stream).inc()
+			# Prom doesn't provide a way to compare value to gauge's existing value,
+			# we need to reach into internals
+			stat = latest_segment.labels(stream=self.channel, variant=self.stream)
+			timestamp = (self.date - datetime.datetime(1970, 1, 1)).total_seconds()
+			stat.set(max(stat._value.get(), timestamp)) # NOTE: not thread-safe but is gevent-safe
 
 
 @argh.arg('channels', nargs="+", help="Twitch channels to watch")

downloader/setup.py

@@ -9,6 +9,7 @@ setup(
 		"python-dateutil",
 		"gevent",
 		"monotonic",
+		"prometheus-client==0.7.1", # locked version as we rely on internals
 		"requests",
 		"wubloader-common",
 	],