added state to upload_errors

pull/101/head
Christopher Usher 5 years ago
parent 228468749c
commit a63e18c0a9

@ -30,7 +30,7 @@ videos_uploaded = prom.Counter(
upload_errors = prom.Counter(
'upload_errors',
'Number of errors uploading a video',
['video_channel', 'video_quality', 'upload_location']
['video_channel', 'video_quality', 'upload_location', 'final_state']
)
# A list of all the DB column names in CutJob
@ -298,6 +298,10 @@ class Cutter(object):
# Assumed error is not retryable, set state back to UNEDITED and set error.
if not set_row(state='UNEDITED', error="Error while cutting: {}".format(ex), uploader=None):
self.logger.warning("Tried to roll back row {} to unedited but it was already cancelled.".format(job.id))
upload_errors.labels(video_channel=job.video_channel,
video_quality=job.video_quality,
upload_location=job.upload_location,
final_state='UNEDITED').inc()
# Abort the cut without further error handling
raise ErrorHandled
@ -327,14 +331,11 @@ class Cutter(object):
)
except JobConsistencyError:
raise # this ensures it's not caught in the next except block
upload_errors.labels(video_channel=job.video_channel, video_quality=job.video_quality, upload_location=job.upload_location).inc()
except ErrorHandled:
# we're aborting the cut, error handling has already happened
upload_errors.labels(video_channel=job.video_channel, video_quality=job.video_quality, upload_location=job.upload_location).inc()
return
except Exception as ex:
self.refresh_conn()
upload_errors.labels(video_channel=job.video_channel, video_quality=job.video_quality, upload_location=job.upload_location).inc()
# for HTTPErrors, getting http response body is also useful
if isinstance(ex, requests.HTTPError):
@ -353,6 +354,10 @@ class Cutter(object):
"to EDITED and clear uploader. "
"Error: {}"
).format(ex)
upload_errors.labels(video_channel=job.video_channel,
video_quality=job.video_quality,
upload_location=job.upload_location,
final_state='FINALIZING').inc()
if not set_row(error=error):
# Not only do we not know if it was uploaded, we also failed to set that in the database!
raise JobConsistencyError(
@ -364,6 +369,10 @@ class Cutter(object):
# error before finalizing, assume it's a network issue / retryable.
# set back to EDITED but still set error
self.logger.exception("Retryable error when uploading job {}".format(format_job(job)))
upload_errors.labels(video_channel=job.video_channel,
video_quality=job.video_quality,
upload_location=job.upload_location,
final_state='EDITED').inc()
if not set_row(state='UNEDITED', error="Retryable error while uploading: {}".format(ex), uploader=None):
raise JobConsistencyError(
"No job with id {} and uploader {} when setting error while rolling back for retryable error"
@ -384,7 +393,9 @@ class Cutter(object):
)
self.logger.info("Successfully cut and uploaded job {} as {}".format(format_job(job), link))
videos_uploaded.labels(video_channel=job.video_channel, video_quality=job.video_quality, upload_location=job.upload_location).inc()
videos_uploaded.labels(video_channel=job.video_channel,
video_quality=job.video_quality,
upload_location=job.upload_location).inc()
def rollback_all_owned(self):
"""Roll back any in-progress jobs that claim to be owned by us,

Loading…
Cancel
Save