From 3ad24f5aece52983f25d7e0dcf92cde902f79ed9 Mon Sep 17 00:00:00 2001 From: Mike Lang Date: Sat, 9 Nov 2024 02:41:15 +0800 Subject: [PATCH] get working --- merge_v1.py | 7 ++++++- merge_v2.py | 11 ++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/merge_v1.py b/merge_v1.py index 164ee65..af6acd7 100644 --- a/merge_v1.py +++ b/merge_v1.py @@ -81,8 +81,13 @@ def merge_messages(left, right): return result +def load(file): + with open(file) as f: + return [json.loads(line) for line in f.read().strip().split("\n")] + + def main(*files): - batches = [json.load(open(file)) for file in files] + batches = [load(file) for file in files] result = batches[0] start = time.monotonic() for batch in batches[1:]: diff --git a/merge_v2.py b/merge_v2.py index 7752643..a2d0e69 100644 --- a/merge_v2.py +++ b/merge_v2.py @@ -50,7 +50,7 @@ def merge_messages(left, right): message, candidates = right.pop(0), left # Scan candidates for matching message until found or we know there is no match - id = message.get("tags", {}).get("id") + id = (message.get("tags") or {}).get("id") end = message['time'] + message['time_range'] merged = None for index, candidate in enumerate(candidates): @@ -62,7 +62,7 @@ def merge_messages(left, right): if merged is not None: candidates.pop(index) break - elif candidate.get("tags", {}).get("id") == id: + elif (candidate.get("tags") or {}).get("id") == id: merged = merge_message(message, candidate) if merged is None: raise ValueError("TODO") @@ -82,8 +82,13 @@ def merge_messages(left, right): return result +def load(file): + with open(file) as f: + return [json.loads(line) for line in f.read().strip().split("\n")] + + def main(*files): - batches = [json.load(open(file)) for file in files] + batches = [load(file) for file in files] result = batches[0] start = time.monotonic() for batch in batches[1:]: