more backfiller chat fixes

fixup: more backfiller fixes

Enable backfilling of chat logs

backfiller/backfiller/main.py

@@ -21,7 +21,7 @@ import common
 from common import dateutil
 from common import database
 from common.requests import InstrumentedSession
-from common.segments import list_segment_files
+from common.segments import list_segment_files, unpadded_b64_decode
 
 # Wraps all requests in some metric collection
 requests = InstrumentedSession()
@@ -133,8 +133,9 @@ def get_remote_segment(base_dir, node, channel, quality, hour, missing_segment,
 	dir_name = os.path.dirname(path)
 	if quality == "chat":
 		# chat segment
-		_, filename_hash = os.path.basename(path).split('-', 1)
-		temp_name = "{}.{}.temp".format(os.path.basename(path), uuid4())
+		_, filename_hash = os.path.splitext(os.path.basename(path))[0].split('-', 1)
+		filename_hash = unpadded_b64_decode(filename_hash)
+		temp_name = "{}.{}.temp".format(os.path.basename(path), uuid.uuid4())
 	else:
 		# video segment
 		date, duration, _ = os.path.basename(path).split('-', 2)
@@ -486,6 +487,9 @@ class BackfillerWorker(object):
 							pass
 						continue
 
+					# For chat archives, just download whatever is there.
+					# Otherwise, do some basic checks.
+					if quality != 'chat':
 						# test to see if file is a segment and get the segments start time
 						try:
 							segment = common.parse_segment_path(path)

docker-compose.jsonnet

@@ -178,6 +178,8 @@
     // Twitch user to log in as and path to oauth token
     user: "dbvideostriketeam",
     token_path: "./chat_token.txt",
+    // Whether to enable backfilling of chat archives to this node (if backfiller enabled)
+    backfill: true,
   },
 
   // Extra options to pass via environment variables,
@@ -253,7 +255,7 @@
       command: $.clean_channels +
       [
         "--base-dir", "/mnt",
-        "--qualities", std.join(",", $.qualities),
+        "--qualities", std.join(",", $.qualities + (if $.chat_archiver.backfill then ["chat"] else [])),
         "--static-nodes", std.join(",", $.peers),
         "--backdoor-port", std.toString($.backdoor_port),
         "--node-database", $.db_connect,