wubloader/docker-compose.jsonnet

// This is a jsonnet file, it generates a docker-compose.yml file.
// To generate, run "make docker-compose.yml".

{

  // These are the important top-level settings.
  // Change these to configure the services.

  // Image tag (application version) to use.
  // By default, will use the current commit, ie. the same thing that ./build would tag
  // things it builds with.
  // Note: "latest" is not recommended in production, as you can't be sure what version
  // you're actually running, and must manually re-pull to get an updated copy.
  image_tag:: std.extVar("tag"),
  database_tag:: "bb05e37", // tag for DB, which changes less and restarts are disruptive
  image_base:: "ghcr.io/dbvideostriketeam", // Change this to use images from a different source than the main one

  // For each service, whether to deploy that service.
  enabled:: {
    downloader: true,
    restreamer: true,
    backfiller: true,
    cutter: true,
    sheetsync: false,
    thrimshim: true,
    segment_coverage: true,
    playlist_manager: false,
    nginx: true,
    postgres: false,
    chat_archiver: false,
    schedulebot: false,
    tootbot: false,
    twitchbot: false,
    bus_analyzer: false,
    graphs: false,
  },

  // Twitch channels to capture. The first one will be used as the default channel in the editor.
  // Channels suffixed with a '!' are considered "important" and will be retried more aggressively
  // and warned about if they're not currently streaming.
  channels:: ["desertbus!", "db_chief", "db_high", "db_audio", "db_bus"],

  backfill_only_channels:: [],

  // Stream qualities to capture
  qualities:: ["source", "480p"],

  // Local path to save segments to. Full path must already exist. Cannot contain ':'.
  // On OSX you need to change this to /private/var/lib/wubloader
  segments_path:: "/var/lib/wubloader/",

  // Local path to save database to. Full path must already exist. Cannot
  // contain ':'. If this directory is non-empty, the database will start with
  // the database in this directory and not run the setup scripts to create a 
  // new database.
  // On OSX you need to change this to /private/var/lib/wubloader_postgres/
  database_path:: "/var/lib/wubloader_postgres/",

  // Local path to Thrimbletrimmer files. Useful if you're doing development on
  // Thrimbletrimmer (and probably not useful otherwise) to enable live updates
  // to Thrimbletrimmer without restarting/rebuilding Wubloader.
  // If you wish to use this, set this to the path containing the Thrimbletrimmer
  // web (HTML, CSS, JavaScript) files to serve (e.g. "/path/to/wubloader/thrimbletrimmer/").
  thrimbletrimmer_web_dev_path:: null,

  // The host's port to expose each service on.
  // Only nginx (and postgres if that is being deployed) needs to be externally accessible - the other non-database ports are routed through nginx.
  ports:: {
    restreamer: 8000,
    downloader: 8001,
    backfiller: 8002,
    cutter: 8003,
    thrimshim: 8004,
    sheetsync: 8005,
    segment_coverage: 8006,
    playlist_manager: 8007,
    chat_archiver: 8008,
    nginx: 80,
    nginx_ssl: 443,
    postgres: 5432,
  },

  // The local port within each container to bind the backdoor server on.
  // You can exec into the container and telnet to this port to get a python shell.
  backdoor_port:: 1234,

  // Other nodes to always backfill from. You should not include the local node.
  // If you are using the database to find peers, you should leave this empty.
  peers:: [
  ],

  localhost:: "node_name", // the name in the nodes table of the database
  
  authentication:: true, // set to false to disable auth in thrimshim

  thrimbletrimmer:: true, // set to false to not have nginx serve thrimbletrimmer pages.

  nginx_serve_segments:: true, // set to false to not have nginx serve segments directly, letting restreamer do it instead.

  ssl_certificate_path:: null, // set to path to SSL certs (cert chain + priv key in one file) to enable SSL

  // Connection args for the database.
  // If database is defined in this config, host and port should be postgres:5432.
  db_args:: {
    user: "vst",
    password: "dbfh2019", // don't use default in production. Must not contain ' or \ as these are not escaped.
    host: "postgres",
    port: 5432,
    dbname: "wubloader",
  },

  // Other database arguments
  db_super_user:: "postgres", // only accessible from localhost
  db_super_password:: "postgres", // Must not contain ' or \ as these are not escaped.
  db_replication_user:: "replicate", // if empty, don't allow replication
  db_replication_password:: "standby", // don't use default in production. Must not contain ' or \ as these are not escaped.
  db_readonly_user:: "readonly", // if empty, don't have a readonly account
  db_readonly_password:: "volunteer", // don't use default in production. Must not contain ' or \ as these are not escaped.  
  db_standby:: false, // set to true to have this database replicate another server

  // Path to a file containing a twitch OAuth token to use when downloading streams.
  // This is optional (null to omit) but may be helpful to bypass ads.
  downloader_creds_file:: null,

  // Path to a JSON file containing google credentials for cutter as keys
  // 'client_id', 'client_secret' and 'refresh_token'.
  cutter_creds_file:: "./google_creds.json",

  // Config for cutter upload locations. See cutter docs for full detail.
  cutter_config:: {
    // Default
    desertbus: {type: "youtube", cut_type: "smart"},
    // Backup options for advanced use, if the smart cut breaks things.
    desertbus_slow: {type: "youtube", cut_type: "full"},
    desertbus_emergency: {type: "youtube", cut_type: "fast"},
    // Non-uploading backend that lets us modify manually-updated youtube videos
    "youtube-manual": {type: "youtube", no_uploader: true}.
  },
  default_location:: "desertbus",
  // archive location is the default location for archive events,
  // only revelant if $.archive_worksheet is set.
  archive_location:: "archive",

  // Fixed tags to add to all videos
  video_tags:: ["DB13", "DB2019", "2019", "Desert Bus", "Desert Bus for Hope", "Child's Play Charity", "Child's Play", "Charity Fundraiser"],

  // The header to put at the front of video titles, eg. a video with a title
  // of "hello world" with title header "foo" becomes: "foo - hello world".
  title_header:: "DB2019",

  // The footer to put at the bottom of descriptions, in its own paragraph.
  description_footer:: |||
    https://www.desertbus.org
    Uploaded by the Desert Bus Video Strike Team
  |||,

  // Path to a JSON file containing google credentials for sheetsync as keys
  // 'client_id', 'client_secret' and 'refresh_token'.
  // May be the same as cutter_creds_file.
  sheet_creds_file:: "./google_creds.json",

  // Path to a text file containing the auth token for the streamlog server
  streamlog_creds_file:: "./streamlog_token.txt",

  // The URL to write to the sheet for edit links, with {} being replaced by the id
  edit_url:: "https://wubloader.example.com/thrimbletrimmer/edit.html?id={}",

  // The timestamp corresponding to 00:00 in bustime
  bustime_start:: "1970-01-01T00:00:00Z",

  // The timestamps to start/end segment coverage maps at.
  // Generally 1 day before and 7 days after bus start.
  coverage_start:: "1969-12-31T00:00:00Z",
  coverage_end:: "1970-01-07T00:00:00Z",

  // Max hours ago to backfill, ie. do not backfill for times before this many hours ago.
  // Set to null to disable.
  backfill_max_hours_ago:: 24 * 30 * 6, // approx 6 months

  // Extra directories (besides segments) to backfill
  backfill_dirs:: ["emotes"],

  // The spreadsheet id and worksheet names for sheet sync to act on
  // Set to null to disable syncing from sheets.
  sheet_id:: "your_id_here",
  worksheets:: ["Tech Test & Preshow"] + ["Day %d" % n for n in std.range(1, 8)],
  playlist_worksheet:: "Tags",

  // The archive worksheet, if given, points to a worksheet containing events with a different
  // schema and alternate behaviour suitable for long-term archival videos instead of uploads.
  archive_worksheet:: "Video Trim Times",

  // Set to true to enable reverse-sync mode into Sheets, instead of syncing from it.
  sheet_reverse_sync:: false,

  // The StreamLog server and event id to use, or null to disable sync from StreamLog.
  streamlog_url:: "https://streamlog.example.com",
  streamlog_event:: "id_goes_here",

  // A map from youtube playlist IDs to a list of tags.
  // Playlist manager will populate each playlist with all videos which have all those tags.
  // For example, tags ["Day 1", "Technical"] will populate the playlist with all Technical
  // youtube videos from Day 1.
  // Note that you can make an "all videos" playlist by specifying no tags (ie. []).
  playlists:: {
    // Replaced entirely by tags sheet
  },

  // Which upload locations should be added to playlists
  youtube_upload_locations:: [
    "desertbus",
    "desertbus_slow",
    "desertbus_emergency",
    "youtube-manual",
  ],

  chat_archiver:: {
    // Twitch user to log in as and path to oauth token
    user: "dbvideostriketeam",
    token_path: "./chat_token.txt",
    // Whether to enable backfilling of chat archives to this node (if backfiller enabled)
    backfill: true,
    // Channels to watch. Defaults to "all twitch channels in $.channels" but you can add extras.
    channels: [
      std.split(c, '!')[0]
      for c in $.channels
      if std.length(std.split(c, ":")) == 1
    ],
  },

  // The channel to use for bus_analyzer
  bus_channel:: "buscam",

  zulip_url:: "https://chat.videostrike.team",

  schedulebot:: {
    // Creds for zulip api calls. Can't be a bot user due to annoying arbitrary restrictions.
    api_user: {
      email: "vst-zulip-bot@ekime.kim",
      api_key: "",
    },
    // Creds for the bot user to send messages as
    send_user: {
      email: "schedule-bot@chat.videostrike.team",
      api_key: "",
    },
    // Path to the schedule CSV
    schedule_path:: "",
    // Map from group names to zulip internal ids
    groups: {
      Sheeter: 16,
      Editor: 17,
      ChatOps: 18,
    },
    // Map from group id to 4 hard-coded lists of user ids, one per shift.
    groups_by_shift: {
    },
    // Map from schedule names to zulip user ids
    members: {
      ekimekim: 8,
    },
    // Extra args, see schedulebot.py.
    // --no-initial prevents re-posting current hour on restart.
    // --omega and --last enable special behaviour for omega shift and end of run, once known.
    args:: ["--no-initial"],
  },

  tootbot:: {
    zulip: {
      email: "tootbot-bot@chat.videostrike.team",
      api_key: "",
    },
    mastodon: {
      url: "https://kind.social",
      // Obtain an access token by running: python -m zulip_bots.tootbot get-access-token
      access_token: "",
    },
    args:: [],
  },

  twitchbot:: {
    twitch_username: $.chat_archiver.user,
    twitch_oauth_token: "",
    zulip_email: "twitch-chat-bot@chat.videostrike.team",
    zulip_api_key: "",
    args:: [],
  },

  // template for donation data urls
  donation_url_template:: "https://example.com/DB{}/DB{}.json",

  // Extra options to pass via environment variables,
  // eg. log level, disabling stack sampling.
  env:: {
    // Uncomment this to set log level to debug
    // WUBLOADER_LOG_LEVEL: "DEBUG",
    // Uncomment this to enable stacksampling performance monitoring
    // WUBLOADER_ENABLE_STACKSAMPLER: "true",
  },

  // Now for the actual docker-compose config

  // The connection string for the database. Constructed from db_args.
  db_connect:: std.join(" ", [
    "%s='%s'" % [key, $.db_args[key]]
    for key in std.objectFields($.db_args)
  ]),

  // Cleaned up version of $.channels without importance/type markers.
  // General form is CHANNEL[!][:TYPE:URL].
  clean_channels:: [std.split(std.split(c, ":")[0], '!')[0] for c in $.channels] + $.backfill_only_channels,

  // Image format helper
  get_image(name, tag=$.image_tag):: "%s/wubloader-%s:%s" % [
    $.image_base,
    name,
    tag,
  ],

  // docker-compose version
  version: "3",

  services: {

    [if $.enabled.downloader then "downloader"]: {
      image: $.get_image("downloader"),
      // Args for the downloader: set channel and qualities
      command: $.channels +
      [  
        "--base-dir", "/mnt",
        "--qualities", std.join(",", $.qualities),
        "--backdoor-port", std.toString($.backdoor_port),
      ] + if $.downloader_creds_file != null then ["--twitch-auth-file", "/token"] else [],
      // Mount the segments directory at /mnt
      volumes: ["%s:/mnt" % $.segments_path]
        + if $.downloader_creds_file != null then ["%s:/token" % $.downloader_creds_file] else [],
      // If the application crashes, restart it.
      restart: "on-failure",
      // Expose on the configured host port by mapping that port to the default
      // port for downloader, which is 8001.
      [if "downloader" in $.ports then "ports"]: ["%s:8001" % $.ports.downloader],
      environment: $.env,
    },

    [if $.enabled.restreamer then "restreamer"]: {
      image: $.get_image("restreamer"),
      // Mount the segments directory at /mnt
      volumes: ["%s:/mnt" % $.segments_path],
      // If the application crashes, restart it.
      restart: "on-failure",
      // Expose on the configured host port by mapping that port to the default
      // port for restreamer, which is 8000.
      [if "restreamer" in $.ports then "ports"]: ["%s:8000" % $.ports.restreamer],
      command: [
        "--base-dir", "/mnt",
        "--backdoor-port", std.toString($.backdoor_port),
      ],
      environment: $.env,
    },

    [if $.enabled.backfiller then "backfiller"]: {
      image: $.get_image("backfiller"),
      // Args for the backfiller: set channel and qualities
      command: $.clean_channels +
      [
        "--base-dir", "/mnt",
        "--qualities", std.join(",", $.qualities + (if $.chat_archiver.backfill then ["chat"] else [])),
        "--extras", std.join(",", $.backfill_dirs),
        "--static-nodes", std.join(",", $.peers),
        "--backdoor-port", std.toString($.backdoor_port),
        "--node-database", $.db_connect,
        "--localhost", $.localhost,
      ] + (if $.backfill_max_hours_ago == null then [] else [
        "--start", std.toString($.backfill_max_hours_ago),
      ]),
      // Mount the segments directory at /mnt
      volumes: ["%s:/mnt" % $.segments_path],
      // If the application crashes, restart it.
      restart: "on-failure",
      // Expose on the configured host port by mapping that port to the default
      // port for backfiller, which is 8002.
      [if "backfiller" in $.ports then "ports"]: ["%s:8002" % $.ports.backfiller],
      environment: $.env,
    },

    [if $.enabled.cutter then "cutter"]: {
      image: $.get_image("cutter"),
      // Args for the cutter: DB and creds
      command: [
        "--base-dir", "/mnt",
        "--backdoor-port", std.toString($.backdoor_port),
        "--tags", std.join(",", $.video_tags),
        "--name", $.localhost,
        $.db_connect,
        std.manifestJson($.cutter_config),
        "/etc/wubloader-creds.json",
      ],
      volumes: [
        // Mount the segments directory at /mnt
        "%s:/mnt" % $.segments_path,
      ] + [
        // Mount the creds file into /etc
        "%s:/etc/wubloader-creds.json" % $.cutter_creds_file,
      ],
      // If the application crashes, restart it.
      restart: "on-failure",
      // Expose on the configured host port by mapping that port to the default
      // port for cutter, which is 8003.
      [if "cutter" in $.ports then "ports"]: ["%s:8003" % $.ports.cutter],
      environment: $.env,
    },

    [if $.enabled.thrimshim then "thrimshim"]: {
      image: $.get_image("thrimshim"),
      // Args for the thrimshim: database connection string 
      command: [
        "--backdoor-port", std.toString($.backdoor_port),
        "--title-header", $.title_header,
        "--description-footer", $.description_footer,
        "--upload-locations", std.join(",", [$.default_location] + [
          location for location in std.objectFields($.cutter_config)
          if location != $.default_location
        ]),
        $.db_connect,
        $.clean_channels[0], // use first element as default channel
        $.bustime_start,
      ]
      + (if $.authentication then [] else ["--no-authentication"])
      + (if $.archive_worksheet != null then [
        "--archive-sheet", $.archive_worksheet,
        "--archive-location", $.archive_location,
      ] else []),
      // Mount the segments directory at /mnt
      volumes: ["%s:/mnt" % $.segments_path],
      // If the application crashes, restart it.
      restart: "on-failure",
      // Expose on the configured host port by mapping that port to the default
      // port for thrimshim, which is 8004.
      [if "thrimshim" in $.ports then "ports"]: ["%s:8004" % $.ports.thrimshim],
      environment: $.env,
    },

    [if $.enabled.sheetsync then "sheetsync"]: {
      local sync_sheet = {
          type: "sheets",
          creds: "/etc/sheet-creds.json",
          sheet_id: $.sheet_id,
          worksheets: $.worksheets,
          allocate_ids: true,
          edit_url: $.edit_url,
          bustime_start: $.bustime_start,
          playlist_worksheet: $.playlist_worksheet,
          reverse_sync: $.sheet_reverse_sync,
      },
      local sync_streamlog = {
          type: "streamlog",
          creds: "/etc/streamlog-token.txt",
          url: $.streamlog_url,
          event_id: $.streamlog_event,
      },
      local config = std.prune([
          if $.sheet_id != null then sync_sheet,
          if $.streamlog_url != null then sync_streamlog,
      ]),
      image: $.get_image("sheetsync"),
      // Args for the sheetsync
      command: [
        "--backdoor-port", std.toString($.backdoor_port),
        $.db_connect,
      ] + std.map(std.manifestJson, config),
      // Mount the creds file(s) into /etc
      volumes: std.prune([
        if $.sheet_id != null then "%s:/etc/sheet-creds.json" % $.sheet_creds_file,
        if $.streamlog_url != null then "%s:/etc/streamlog-token.txt" % $.streamlog_creds_file,
      ]),
      // If the application crashes, restart it.
      restart: "on-failure",
      // Expose on the configured host port by mapping that port to the default
      // port for sheetsync, which is 8005.
      [if "sheetsync" in $.ports then "ports"]: ["%s:8005" % $.ports.sheetsync],
      environment: $.env,
    },

    [if $.enabled.segment_coverage then "segment_coverage"]: {
      image: $.get_image("segment_coverage"),
      // Args for the segment_coverage
      command: $.clean_channels +
      [
        "--base-dir", "/mnt",
        "--qualities", std.join(",", $.qualities),
        "--first-hour", $.coverage_start,
        "--last-hour", $.coverage_end,
        // Render a html page showing all the images from all nodes
        "--make-page",
        "--connection-string", $.db_connect,
      ],
      // Mount the segments directory at /mnt
      volumes: ["%s:/mnt" % $.segments_path],
      // If the application crashes, restart it.
      restart: "on-failure",
      // Expose on the configured host port by mapping that port to the default
      // port for thrimshim, which is 8004.
      [if "segment_coverage" in $.ports then "ports"]: ["%s:8006" % $.ports.segment_coverage],
      environment: $.env,
    },

    [if $.enabled.playlist_manager then "playlist_manager"]: {
      image: $.get_image("playlist_manager"),
      // Args for the playlist_manager
      command: [
        "--backdoor-port", std.toString($.backdoor_port),
        "--upload-location-allowlist", std.join(",", $.youtube_upload_locations),
        $.db_connect,
        "/etc/wubloader-creds.json",
      ] + [
        "%s=%s" % [playlist, std.join(",", $.playlists[playlist])]
        for playlist in std.objectFields($.playlists)
      ],
      volumes: [
        // Mount the creds file into /etc
        "%s:/etc/wubloader-creds.json" % $.cutter_creds_file,
      ],
      // If the application crashes, restart it.
      restart: "on-failure",
      // Expose on the configured host port by mapping that port to the default
      // port for playlist_manager, which is 8007.
      [if "playlist_manager" in $.ports then "ports"]: ["%s:8007" % $.ports.playlist_manager],
      environment: $.env,
    },

    [if $.enabled.nginx then "nginx"]: {
      # mapping of services to internal ports for nginx to forward
      local forward_ports = {
        restreamer: 8000,
        downloader: 8001,
        backfiller: 8002,
        cutter: 8003,
        thrimshim: 8004,
        sheetsync: 8005,
        segment_coverage: 8006,
        playlist_manager: 8007,
        chat_archiver: 8008,
      },
      image: $.get_image("nginx"),
      restart: "on-failure",
      ports: std.prune([
        if "nginx" in $.ports then "%s:80" % $.ports.nginx,
        if "nginx_ssl" in $.ports then "%s:443" % $.ports.nginx_ssl,
      ]),
      environment: $.env + {
        SERVICES: std.join("\n", [
          "%s %s" % [service, forward_ports[service]]
          for service in std.objectFields(forward_ports)
          if service in $.enabled && $.enabled[service]
        ]),
        THRIMBLETRIMMER: if $.thrimbletrimmer then "true" else "",
        SEGMENTS: if $.nginx_serve_segments then "/mnt" else "",
        SSL: if $.ssl_certificate_path != null then "/certs.pem" else "",
      },
      volumes: std.prune([
        if $.nginx_serve_segments then "%s:/mnt" % $.segments_path,
        if $.ssl_certificate_path != null then "%s:/certs.pem" % $.ssl_certificate_path,
        if $.thrimbletrimmer_web_dev_path != null then "%s:/etc/nginx/html/thrimbletrimmer" % $.thrimbletrimmer_web_dev_path,
      ]),
    },

    [if $.enabled.postgres then "postgres"]: {
      image: $.get_image("postgres", $.database_tag),
      restart: "on-failure",
      [if "postgres" in $.ports then "ports"]: ["%s:5432" % $.ports.postgres],
      environment: $.env + {
        POSTGRES_USER: $.db_super_user,
        POSTGRES_PASSWORD: $.db_super_password,
        POSTGRES_DB: $.db_args.dbname,
        PGDATA: "/mnt/database",
        WUBLOADER_USER: $.db_args.user,
        WUBLOADER_PASSWORD: $.db_args.password,
        REPLICATION_USER: $.db_replication_user,
        REPLICATION_PASSWORD: $.db_replication_password,
        READONLY_USER: $.db_readonly_user,
        READONLY_PASSWORD: $.db_readonly_password,
        MASTER_NODE: $.db_args.host,
      },
      volumes: ["%s:/mnt/database" % $.database_path, "%s:/mnt/wubloader" % $.segments_path],
      [if $.db_standby then "command"]: ["/standby_setup.sh"],
    },

    [if $.enabled.chat_archiver then "chat_archiver"]: {
      image: $.get_image("chat_archiver"),
      restart: "always",
      command: [$.chat_archiver.user, "/token"] + $.chat_archiver.channels + ["--name", $.localhost],
      volumes: ["%s:/mnt" % $.segments_path, "%s:/token" % $.chat_archiver.token_path],
      [if "chat_archiver" in $.ports then "ports"]: ["%s:8008" % $.ports.chat_archiver],
      environment: $.env,
    },

    [if $.enabled.bus_analyzer then "bus_analyzer"]: {
      image: $.get_image("bus_analyzer"),
      restart: "always",
      command: [$.db_connect, $.bus_channel, "--base-dir", "/mnt"],
      volumes: ["%s:/mnt" % $.segments_path],
      environment: $.env,
    },

    [if $.enabled.graphs then "graphs"]: {
      image: $.get_image("graphs"),
      restart: "always",
      command: [$.donation_url_template, "--base-dir", "/mnt/graphs"],
      volumes: ["%s:/mnt" % $.segments_path],
      environment: $.env,
    },

    local bot_service(name, config, args, subcommand=null) = {
      image: $.get_image("zulip_bots"),
      restart: "always",
      entrypoint: ["python3", "-m", "zulip_bots.%s" % name]
        + (if subcommand == null then [] else [subcommand])
        + [std.manifestJson(config)]
        + args,
      environment: $.env,
    },

    [if $.enabled.schedulebot then "schedulebot"]:
      bot_service("schedulebot", $.schedulebot + {
        url: $.zulip_url,
        start_time: $.bustime_start,
        schedule: "/schedule",
      }, $.schedulebot.args) + {
        volumes: ["%s:/schedule" % $.schedulebot.schedule_path],
      },

    [if $.enabled.tootbot then "tootbot"]:
      bot_service("tootbot", $.tootbot + {
        zulip+: { url: $.zulip_url },
      }, $.tootbot.args, subcommand="main"),

    [if $.enabled.twitchbot then "twitchbot"]:
      bot_service("twitchbot", $.twitchbot + {
        zulip_url: $.zulip_url,
      }, $.tootbot.args),

  },

}