wip:

encode-worker.sh

@@ -0,0 +1,120 @@
+#!/bin/bash
+
+set -eu
+
+if [ "$#" -lt 2 ]; then
+	echo "USAGE: $0 NAME CONNINFO"
+	echo "NAME should be a unique name for your node"
+	echo "CONNINFO should be a postgres connection url like postgresql://USER:PASS@HOSTNAME/DATABASE"
+	exit 1
+fi
+
+NAME="$1"
+CONNINFO="$2"
+WORKDIR=${WORKDIR:-.}
+
+db() {
+	psql "$CONNINFO" "$@"
+}
+
+# Returns USER PASS HOST PATH, assumes all but path contain no whitespace
+url_to_parts() {
+	# TODO
+}
+
+url_to_filename() {
+	url_to_parts "$1" | read -r user pass host path
+	basename "$path"
+}
+
+download_file() {
+	# TODO
+}
+
+upload_file() {
+	# TODO
+}
+
+encode() {
+	# TODO
+}
+
+existing=$(
+	db -At -v name="$NAME" <<-SQL
+		SELECT claimed_at, dest_url FROM encodes
+		WHERE claimed_by = :'name'
+	SQL
+)
+if [ -n "$existing" ]; then
+	echo "WARNING: The following files are already claimed by this node:"
+	echo "$existing"
+	echo
+	echo -n "This is likely due to a crash. Unclaim these files? [Y/n] > "
+	read -r resp
+	if [ "$resp" != "n" ]; then
+		db -v name="$NAME" <<-SQL
+			UPDATE encodes SET
+				claimed_by = NULL,
+				claimed_at = NULL
+			WHERE name = :'name'
+		SQL
+	fi
+fi
+
+while true; do
+	echo "Checking for jobs"
+	claimed=$(
+		db -At -v name="$NAME" <<-SQL
+			UPDATE encodes SET
+				claimed_by = :'name',
+				claimed_at = now()
+			WHERE dest_url = (
+				SELECT dest_url FROM encodes
+				WHERE claimed_by = NULL
+				LIMIT 1
+			)
+			RETURNING src_url, src_hash, dest_url, dest_hash
+		SQL
+	)
+	if [ -z "$claimed" ]; then
+		echo "No available jobs, will check again in 1min"
+		sleep 60
+		continue
+	fi
+
+	read -r src_url src_hash dest_url dest_hash <<<"$claimed"
+	src_file=$(url_to_filename "$src_url")
+	dest_file=$(url_to_filename "$dest_url")
+	echo "Got task to encode $dest_file"
+	# Read encode args seperately as we need to split out the array.
+	# The following query outputs one row per arg, seperated by nul chars.
+	# readarray -d '' will read into the given array after splitting on nul chars.
+	# TODO check that NUL bytes work as RS
+	{
+		db -t -R '\0' -v dest_url="$dest_url" <<-SQL
+			SELECT unnest(encode_args) FROM encodes
+			WHERE dest_url = :'dest_url'
+		SQL
+	} | readarray -td '' encode_args
+
+	echo "Downloading source file"
+	download_file "$src_url"
+	echo "Checking source file checksum"
+	sha256sum --status -c - <<<"$src_hash  $src_file"
+	echo "Starting encode"
+	encode "$src_file" "$dest_file" "${encode_args[@]}"
+	echo "Encode complete, uploading output file"
+	upload_file "$dest_url"
+	echo "Calculating output hash and marking complete"
+	dest_hash=$(sha256sum "$dest_file" | cut -d' ' -f1)
+	# Re-setting claimed_by *should* be a no-op here but if something has
+	# gone wrong at least we'll know which node is writing.
+	db -v dest_url="$dest_url" -v dest_hash="$dest_hash" -v name="$name" <<-SQL
+		UPDATE encodes SET
+			dest_hash = :'dest_hash',
+			claimed_by = :'name',
+			finished_at = now()
+		WHERE dest_url = :'dest_url'
+	SQL
+
+done

postgres/schema.sql

@@ -195,3 +195,22 @@ CREATE TABLE templates (
 	crop box_coords NOT NULL,
 	location box_coords NOT NULL
 );
+
+-- Used to farm out encoding jobs to encoder workers.
+-- Hash fields are hex strings containing sha256 hashes.
+-- encode_args should be passed verbatim to ffmpeg with the following substitutions:
+--   {SRC_FILE}: The path to the source file
+--   {DEST_FILE}: The path to the output file
+-- The job is considered complete once the dest hash is written.
+-- Jobs may be claimed by writing a worker name to claimed_by.
+-- Timestamp fields are indicative only.
+CREATE TABLE encodes (
+	src_url TEXT NOT NULL,
+	src_hash TEXT NOT NULL,
+	encode_args TEXT[] NOT NULL,
+	dest_url TEXT PRIMARY KEY,
+	dest_hash TEXT,
+	claimed_by TEXT,
+	claimed_at TIMESTAMP,
+	finished_at TIMESTAMP
+);