chat_archiver: Allow passing multiple channels on CLI

pull/353/head
Mike Lang 1 year ago committed by Mike Lang
parent 91910c0972
commit 260293d40d

@ -457,7 +457,7 @@ def merge_batch_files(path, batch_time):
os.remove(batch_file)
def main(channel, nick, oauth_token_path, base_dir='/mnt', name=None, merge_interval=60, metrics_port=8008):
def main(nick, oauth_token_path, *channels, base_dir='/mnt', name=None, merge_interval=60, metrics_port=8008):
with open(oauth_token_path) as f:
oauth_token = f.read()
# To ensure uniqueness even if multiple instances are running on the same host,
@ -470,23 +470,27 @@ def main(channel, nick, oauth_token_path, base_dir='/mnt', name=None, merge_inte
stopping = gevent.event.Event()
gevent.signal_handler(signal.SIGTERM, stopping.set)
merger = gevent.spawn(merge_all,
mergers = [
gevent.spawn(merge_all,
os.path.join(base_dir, channel, "chat"),
interval=merge_interval,
stopping=stopping
)
) for channel in channels
]
prom.start_http_server(metrics_port)
logging.info("Starting")
for index in count():
# To ensure uniqueness between clients, include a client number
archiver = Archiver("{}.{}".format(name, index), base_dir, [channel], nick, oauth_token)
worker = gevent.spawn(archiver.run)
archiver = Archiver("{}.{}".format(name, index), base_dir, channels, nick, oauth_token)
archive_worker = gevent.spawn(archiver.run)
workers = mergers + [archive_worker]
# wait for either graceful exit, error, or for a signal from the archiver that a reconnect was requested
gevent.wait([stopping, worker, merger, archiver.got_reconnect], count=1)
gevent.wait([stopping, archiver.got_reconnect] + workers, count=1)
if stopping.is_set():
archiver.stop()
for worker in workers:
worker.get()
break
# if got reconnect, discard the old archiver (we don't care even if it fails after this)
@ -497,10 +501,11 @@ def main(channel, nick, oauth_token_path, base_dir='/mnt', name=None, merge_inte
# the only remaining case is that something failed. stop everything and re-raise.
logging.warning("Stopping due to worker dying")
stopping.set()
archiver.stop()
for worker in workers:
worker.join()
merger.join()
# at least one of these two should raise
# at least one of these should raise
for worker in workers:
worker.get()
merger.get()
assert False, "Worker unexpectedly exited successfully"
logging.info("Gracefully stopped")

@ -528,7 +528,7 @@
[if $.enabled.chat_archiver then "chat_archiver"]: {
image: $.get_image("chat_archiver"),
restart: "always",
command: [$.chat_archiver.channel, $.chat_archiver.user, "/token", "--name", $.localhost],
command: [$.chat_archiver.user, "/token", $.chat_archiver.channel, "--name", $.localhost],
volumes: ["%s:/mnt" % $.segments_path, "%s:/token" % $.chat_archiver.token_path],
[if "chat_archiver" in $.ports then "ports"]: ["%s:8008" % $.ports.chat_archiver],
environment: $.env,

Loading…
Cancel
Save