diff --git a/cutter/cutter/main.py b/cutter/cutter/main.py index e3dd51f..c93eb77 100644 --- a/cutter/cutter/main.py +++ b/cutter/cutter/main.py @@ -30,7 +30,7 @@ videos_uploaded = prom.Counter( upload_errors = prom.Counter( 'upload_errors', 'Number of errors uploading a video', - ['video_channel', 'video_quality', 'upload_location'] + ['video_channel', 'video_quality', 'upload_location', 'final_state'] ) # A list of all the DB column names in CutJob @@ -298,6 +298,10 @@ class Cutter(object): # Assumed error is not retryable, set state back to UNEDITED and set error. if not set_row(state='UNEDITED', error="Error while cutting: {}".format(ex), uploader=None): self.logger.warning("Tried to roll back row {} to unedited but it was already cancelled.".format(job.id)) + upload_errors.labels(video_channel=job.video_channel, + video_quality=job.video_quality, + upload_location=job.upload_location, + final_state='UNEDITED').inc() # Abort the cut without further error handling raise ErrorHandled @@ -327,14 +331,11 @@ class Cutter(object): ) except JobConsistencyError: raise # this ensures it's not caught in the next except block - upload_errors.labels(video_channel=job.video_channel, video_quality=job.video_quality, upload_location=job.upload_location).inc() except ErrorHandled: # we're aborting the cut, error handling has already happened - upload_errors.labels(video_channel=job.video_channel, video_quality=job.video_quality, upload_location=job.upload_location).inc() return except Exception as ex: self.refresh_conn() - upload_errors.labels(video_channel=job.video_channel, video_quality=job.video_quality, upload_location=job.upload_location).inc() # for HTTPErrors, getting http response body is also useful if isinstance(ex, requests.HTTPError): @@ -353,6 +354,10 @@ class Cutter(object): "to EDITED and clear uploader. " "Error: {}" ).format(ex) + upload_errors.labels(video_channel=job.video_channel, + video_quality=job.video_quality, + upload_location=job.upload_location, + final_state='FINALIZING').inc() if not set_row(error=error): # Not only do we not know if it was uploaded, we also failed to set that in the database! raise JobConsistencyError( @@ -364,6 +369,10 @@ class Cutter(object): # error before finalizing, assume it's a network issue / retryable. # set back to EDITED but still set error self.logger.exception("Retryable error when uploading job {}".format(format_job(job))) + upload_errors.labels(video_channel=job.video_channel, + video_quality=job.video_quality, + upload_location=job.upload_location, + final_state='EDITED').inc() if not set_row(state='UNEDITED', error="Retryable error while uploading: {}".format(ex), uploader=None): raise JobConsistencyError( "No job with id {} and uploader {} when setting error while rolling back for retryable error" @@ -384,7 +393,9 @@ class Cutter(object): ) self.logger.info("Successfully cut and uploaded job {} as {}".format(format_job(job), link)) - videos_uploaded.labels(video_channel=job.video_channel, video_quality=job.video_quality, upload_location=job.upload_location).inc() + videos_uploaded.labels(video_channel=job.video_channel, + video_quality=job.video_quality, + upload_location=job.upload_location).inc() def rollback_all_owned(self): """Roll back any in-progress jobs that claim to be owned by us,