From dc620087d8496456eba69c2e5041a40898c6d371 Mon Sep 17 00:00:00 2001 From: Mike Lang Date: Fri, 28 Jun 2019 13:41:47 -0700 Subject: [PATCH] downloader: Track timestamp of latest segment This gives us a "stream delay" metric. Prom doesn't have any native way to check the current value of a metric, in order to take max(). It only offers increment and set. We reach into some internals to do this in a hacky way, but the cleaner way would be to track the value ourselves and have a prom callback that gets the value. Sigh, I hate this prom library. I might write my own that's less dumb. --- downloader/downloader/main.py | 11 +++++++++++ downloader/setup.py | 1 + 2 files changed, 12 insertions(+) diff --git a/downloader/downloader/main.py b/downloader/downloader/main.py index c1524f4..54413e6 100644 --- a/downloader/downloader/main.py +++ b/downloader/downloader/main.py @@ -29,6 +29,12 @@ segments_downloaded = prom.Counter( ["partial", "stream", "variant"], ) +latest_segment = prom.Gauge( + "latest_segment", + "Timestamp of the time of the newest segment fully downloaded", + ["stream", "variant"], +) + class TimedOutError(Exception): pass @@ -542,6 +548,11 @@ class SegmentGetter(object): self.logger.debug("Saving completed segment {} as {}".format(temp_path, full_path)) common.rename(temp_path, full_path) segments_downloaded.labels(partial="False", stream=self.channel, variant=self.stream).inc() + # Prom doesn't provide a way to compare value to gauge's existing value, + # we need to reach into internals + stat = latest_segment.labels(stream=self.channel, variant=self.stream) + timestamp = (self.date - datetime.datetime(1970, 1, 1)).total_seconds() + stat.set(max(stat._value.get(), timestamp)) # NOTE: not thread-safe but is gevent-safe @argh.arg('channels', nargs="+", help="Twitch channels to watch") diff --git a/downloader/setup.py b/downloader/setup.py index 534ff51..2928579 100644 --- a/downloader/setup.py +++ b/downloader/setup.py @@ -9,6 +9,7 @@ setup( "python-dateutil", "gevent", "monotonic", + "prometheus-client==0.7.1", # locked version as we rely on internals "requests", "wubloader-common", ],