added state to upload_errors

cutter/cutter/main.py

@@ -30,7 +30,7 @@ videos_uploaded  = prom.Counter(
 upload_errors  = prom.Counter(
 	'upload_errors',
 	'Number of errors uploading a video',
-	['video_channel', 'video_quality', 'upload_location']
+	['video_channel', 'video_quality', 'upload_location', 'final_state']
 )
 
 # A list of all the DB column names in CutJob
@@ -298,6 +298,10 @@ class Cutter(object):
 				# Assumed error is not retryable, set state back to UNEDITED and set error.
 				if not set_row(state='UNEDITED', error="Error while cutting: {}".format(ex), uploader=None):
 					self.logger.warning("Tried to roll back row {} to unedited but it was already cancelled.".format(job.id))
+				upload_errors.labels(video_channel=job.video_channel,
+						video_quality=job.video_quality,
+						upload_location=job.upload_location,
+						final_state='UNEDITED').inc()
 				# Abort the cut without further error handling
 				raise ErrorHandled
 
@@ -327,14 +331,11 @@ class Cutter(object):
 			)
 		except JobConsistencyError:
 			raise # this ensures it's not caught in the next except block
-			upload_errors.labels(video_channel=job.video_channel, video_quality=job.video_quality, upload_location=job.upload_location).inc()
 		except ErrorHandled:
 			# we're aborting the cut, error handling has already happened
-			upload_errors.labels(video_channel=job.video_channel, video_quality=job.video_quality, upload_location=job.upload_location).inc()
 			return
 		except Exception as ex:
 			self.refresh_conn()
-			upload_errors.labels(video_channel=job.video_channel, video_quality=job.video_quality, upload_location=job.upload_location).inc()
 
 			# for HTTPErrors, getting http response body is also useful
 			if isinstance(ex, requests.HTTPError):
@@ -353,6 +354,10 @@ class Cutter(object):
 					"to EDITED and clear uploader. "
 					"Error: {}"
 				).format(ex)
+				upload_errors.labels(video_channel=job.video_channel,
+						video_quality=job.video_quality,
+						upload_location=job.upload_location,
+						final_state='FINALIZING').inc()
 				if not set_row(error=error):
 					# Not only do we not know if it was uploaded, we also failed to set that in the database!
 					raise JobConsistencyError(
@@ -364,6 +369,10 @@ class Cutter(object):
 			# error before finalizing, assume it's a network issue / retryable.
 			# set back to EDITED but still set error
 			self.logger.exception("Retryable error when uploading job {}".format(format_job(job)))
+			upload_errors.labels(video_channel=job.video_channel,
+					video_quality=job.video_quality,
+					upload_location=job.upload_location,
+					final_state='EDITED').inc()			
 			if not set_row(state='UNEDITED', error="Retryable error while uploading: {}".format(ex), uploader=None):
 				raise JobConsistencyError(
 					"No job with id {} and uploader {} when setting error while rolling back for retryable error"
@@ -384,7 +393,9 @@ class Cutter(object):
 			)
 
 		self.logger.info("Successfully cut and uploaded job {} as {}".format(format_job(job), link))
-		videos_uploaded.labels(video_channel=job.video_channel, video_quality=job.video_quality, upload_location=job.upload_location).inc()
+		videos_uploaded.labels(video_channel=job.video_channel,
+				video_quality=job.video_quality,
+				upload_location=job.upload_location).inc()
 
 	def rollback_all_owned(self):
 		"""Roll back any in-progress jobs that claim to be owned by us,