CachedIterator: Re-serve any errors encountered while iterating

instead of the second one to reach the error treating it as a successful end of iterator.
pull/389/head
Mike Lang 1 year ago committed by Mike Lang
parent 909cece196
commit 8ede4622ca

@ -13,6 +13,7 @@ class CachedIterator():
""" """
def __init__(self, iterator): def __init__(self, iterator):
self.iterator = iterator # Replaced with None once it's exhausted self.iterator = iterator # Replaced with None once it's exhausted
self.error = None # Set if iterator errors, the error is repeated for all future consumption
self.items = [] self.items = []
self.lock = gevent.lock.RLock() self.lock = gevent.lock.RLock()
@ -24,8 +25,10 @@ class CachedIterator():
# If we're more than 1 beyond the end, something has gone horribly wrong. # If we're more than 1 beyond the end, something has gone horribly wrong.
# We should've already lengthened it last iteration # We should've already lengthened it last iteration
assert len(self.items) == i, "CachedIterator logic error: {} != {}".format(len(self.items), i) assert len(self.items) == i, "CachedIterator logic error: {} != {}".format(len(self.items), i)
# Check if the iterator is still active. If not, we've reached the end. # Check if the iterator is still active. If not, we've reached the end or an error.
if self.iterator is None: if self.iterator is None:
if self.error is not None:
raise self.error
return return
# Note we don't need the lock up until now because we're only trying to be gevent-safe, # Note we don't need the lock up until now because we're only trying to be gevent-safe,
# not thread-safe. Simple operations like checking lengths can't be interrupted. # not thread-safe. Simple operations like checking lengths can't be interrupted.
@ -40,5 +43,10 @@ class CachedIterator():
self.iterator = None self.iterator = None
# And we're done. # And we're done.
return return
except Exception as ex:
# Discard the iterator and save the error to be re-served
self.iterator = None
self.error = ex
raise
self.items.append(item) self.items.append(item)
yield self.items[i] yield self.items[i]

Loading…
Cancel
Save