From f75f3e61e83510e061d9383d0abf3cb3d0c18b48 Mon Sep 17 00:00:00 2001 From: Christopher Usher Date: Tue, 24 Sep 2019 03:36:07 +0100 Subject: [PATCH] Removed schema from common/database.py --- common/common/database.py | 67 --------------------------------------- postgres/setup.sh | 19 +++++------ 2 files changed, 10 insertions(+), 76 deletions(-) diff --git a/common/common/database.py b/common/common/database.py index bd1f18d..fbf5bfb 100644 --- a/common/common/database.py +++ b/common/common/database.py @@ -13,71 +13,6 @@ import psycopg2.extras from psycogreen.gevent import patch_psycopg -# Schema is applied on startup and should be idemponent, -# and include any migrations potentially needed. -SCHEMA = """ - --- Create type if it doesn't already exist -DO $$ BEGIN - CREATE TYPE event_state as ENUM ( - 'UNEDITED', - 'EDITED', - 'CLAIMED', - 'FINALIZING', - 'TRANSCODING', - 'DONE' - ); -EXCEPTION WHEN duplicate_object THEN - NULL; -END $$; - -CREATE TABLE IF NOT EXISTS events ( - id UUID PRIMARY KEY, - event_start TIMESTAMP, - event_end TIMESTAMP, - category TEXT NOT NULL DEFAULT '', - description TEXT NOT NULL DEFAULT '', - submitter_winner TEXT NOT NULL DEFAULT '', - poster_moment BOOLEAN NOT NULL DEFAULT FALSE, - image_links TEXT[] NOT NULL DEFAULT '{}', -- default empty array - notes TEXT NOT NULL DEFAULT '', - allow_holes BOOLEAN NOT NULL DEFAULT FALSE, - uploader_whitelist TEXT[], - upload_location TEXT CHECK (state = 'UNEDITED' OR upload_location IS NOT NULL), - video_start TIMESTAMP CHECK (state IN ('UNEDITED', 'DONE') OR video_start IS NOT NULL), - video_end TIMESTAMP CHECK (state IN ('UNEDITED', 'DONE') OR video_end IS NOT NULL), - video_title TEXT CHECK (state IN ('UNEDITED', 'DONE') OR video_title IS NOT NULL), - video_description TEXT CHECK (state IN ('UNEDITED', 'DONE') OR video_description IS NOT NULL), - video_channel TEXT CHECK (state IN ('UNEDITED', 'DONE') OR video_channel IS NOT NULL), - video_quality TEXT NOT NULL DEFAULT 'source', - state event_state NOT NULL DEFAULT 'UNEDITED', - uploader TEXT CHECK (state IN ('UNEDITED', 'EDITED', 'DONE') OR uploader IS NOT NULL), - error TEXT, - video_id TEXT, - video_link TEXT CHECK (state != 'DONE' OR video_link IS NOT NULL), - editor TEXT, - edit_time TIMESTAMP CHECK (state = 'UNEDITED' OR editor IS NOT NULL), - upload_time TIMESTAMP CHECK (state != 'DONE' OR upload_time IS NOT NULL) - -); - --- Index on state, since that's almost always what we're querying on besides id -CREATE INDEX IF NOT EXISTS event_state ON events (state); - -CREATE TABLE IF NOT EXISTS nodes ( - name TEXT PRIMARY KEY, - url TEXT NOT NULL, - backfill_from BOOLEAN NOT NULL DEFAULT TRUE -); - -CREATE TABLE IF NOT EXISTS editors ( - email TEXT PRIMARY KEY, - name TEXT NOT NULL -); - -""" - - class DBManager(object): """Patches psycopg2 before any connections are created, and applies the schema. Stores connect info for easy creation of new connections, and sets some defaults before @@ -95,8 +30,6 @@ class DBManager(object): self.conns = [] self.connect_kwargs = connect_kwargs conn = self.get_conn() - with transaction(conn): - query(conn, SCHEMA) self.put_conn(conn) def put_conn(self, conn): diff --git a/postgres/setup.sh b/postgres/setup.sh index 26a908a..ab88c66 100644 --- a/postgres/setup.sh +++ b/postgres/setup.sh @@ -7,7 +7,7 @@ sed -i "/host all all all/d" "$PGDATA/pg_hba.conf" echo "host all $WUBLOADER_USER all md5" >> "$PGDATA/pg_hba.conf" echo "Creating $WUBLOADER_USER" -psql -v ON_ERROR_STOP=1 -U postgres <<-EOSQL +psql -v ON_ERROR_STOP=1 -U $POSTGRES_USER <<-EOSQL CREATE USER $WUBLOADER_USER LOGIN PASSWORD '$WUBLOADER_PASSWORD'; @@ -18,7 +18,7 @@ if [ -n "$REPLICATION_USER" ]; then echo "Creating $REPLICATION_USER" # allow the $REPLICATION user to replicate remotely echo "host replication $REPLICATION_USER all md5" >> "$PGDATA/pg_hba.conf" - psql -v ON_ERROR_STOP=1 -U postgres <<-EOSQL + psql -v ON_ERROR_STOP=1 -U $POSTGRES_USER <<-EOSQL CREATE USER $REPLICATION_USER LOGIN REPLICATION PASSWORD '$REPLICATION_PASSWORD'; @@ -34,7 +34,8 @@ if [ -n "$REPLICATION_USER" ]; then fi -psql -v ON_ERROR_STOP=1 -U $WUBLOADER_USER -d wubloader <<-EOSQL +echo "Applying schema for $POSTGRES_DB" +psql -v ON_ERROR_STOP=1 -U $WUBLOADER_USER -d $POSTGRES_DB <<-EOSQL -- Create type if it doesn't already exist DO \$\$ BEGIN CREATE TYPE event_state as ENUM ( @@ -49,7 +50,7 @@ EXCEPTION WHEN duplicate_object THEN NULL; END \$\$; -CREATE TABLE IF NOT EXISTS events ( +CREATE TABLE events ( id UUID PRIMARY KEY, event_start TIMESTAMP, event_end TIMESTAMP, @@ -80,15 +81,15 @@ CREATE TABLE IF NOT EXISTS events ( ); -- Index on state, since that's almost always what we're querying on besides id -CREATE INDEX IF NOT EXISTS event_state ON events (state); +CREATE INDEX event_state ON events (state); -CREATE TABLE IF NOT EXISTS nodes ( +CREATE TABLE nodes ( name TEXT PRIMARY KEY, url TEXT NOT NULL, backfill_from BOOLEAN NOT NULL DEFAULT TRUE ); -CREATE TABLE IF NOT EXISTS editors ( +CREATE TABLE editors ( email TEXT PRIMARY KEY, name TEXT NOT NULL ); @@ -96,14 +97,14 @@ EOSQL if [ -a /mnt/wubloader/nodes.csv ]; then echo "Loading nodes from nodes.csv" - psql -v ON_ERROR_STOP=1 -U postgres -d ${POSTGRES_DB} <<-EOF + psql -v ON_ERROR_STOP=1 -U $POSTGRES_USER -d $POSTGRES_DB <<-EOF COPY nodes FROM '/mnt/wubloader/nodes.csv' DELIMITER ',' CSV HEADER; EOF fi if [ -a /mnt/wubloader/editors.csv ]; then echo "Loading editors from editors.csv" - psql -v ON_ERROR_STOP=1 -U postgres -d ${POSTGRES_DB} <<-EOF + psql -v ON_ERROR_STOP=1 -U $POSTGRES_USER -d $POSTGRES_DB <<-EOF COPY editors FROM '/mnt/wubloader/editors.csv' DELIMITER ',' CSV HEADER; EOF fi