Confused on running Peertube behind remote nginx reverse proxy

Hi,

I recently started having trouble with my peertube instance running on FreeBSD. It looks like this issue is just me, so I’m trying to migrate my Peertube instance from FreeBSD to Ubuntu Server 20.04 to see if that fixes it.

That entails moving Peertube off of the FreeBSD jail that runs nginx proxying and SSL front-end to my other services. Until now I’d been working around this proxy issue by just having the nginx config for Peertube run along side the other reverse proxies on that FreeBSD server.

At first I got hung up trying to reach ip:9000 on my local network, but it seems that only works for a fresh Peertube instance? And once the domain name (host name?) is set, the IP address will not work?

In any case, I believe that Peertube is running correctly on the Ubuntu 20.04 VM now.

$ systemctl status peertube
● peertube.service - PeerTube daemon
     Loaded: loaded (/etc/systemd/system/peertube.service; enabled; vendor preset: enabled)
     Active: active (running) since Mon 2021-09-13 02:04:57 UTC; 5s ago
   Main PID: 41038 (peertube)
      Tasks: 11 (limit: 2277)
     Memory: 90.9M
     CGroup: /system.slice/peertube.service
             └─41038 peertube

Sep 13 02:04:57 peertube systemd[1]: Started PeerTube daemon.

From this discussion, I first thought that the stock Peertube nginx config was going to do the remote proxying for me. So I backed up the existing FreeBSD nginx config, and copied over the example Peertube one and set the IP to the new Ubuntu machine.

FreeBSD public nginx server:

# cat /usr/local/etc/nginx/sites-available/peertube.conf 
# Minimum Nginx version required:  1.13.0 (released Apr 25, 2017)
# Please check your Nginx installation features the following modules via 'nginx -V':
# STANDARD HTTP MODULES: Core, Proxy, Rewrite, Access, Gzip, Headers, HTTP/2, Log, Real IP, SSL, Thread Pool, Upstream, AIO Multithreading.
# THIRD PARTY MODULES:   None.

server {
  listen 80;
  listen [::]:80;
  server_name video.berocs.com;

  location /.well-known/acme-challenge/ {
    default_type "text/plain";
    root /var/www/certbot;
  }
  location / { return 301 https://$host$request_uri; }
}

upstream backend {
  server 192.168.13.88:9000;
}

server {
  listen 443 ssl http2;
  listen [::]:443 ssl http2;
  server_name video.berocs.com;

  access_log /var/log/nginx/peertube.access.log; # reduce I/0 with buffer=10m flush=5m
  error_log  /var/log/nginx/peertube.error.log;

  ##
  # Certificates
  # you need a certificate to run in production. see https://letsencrypt.org/
  ##
  #ssl_certificate     /etc/letsencrypt/live/video.berocs.com/fullchain.pem;
  #ssl_certificate_key /etc/letsencrypt/live/video.berocs.com/privkey.pem;
  ssl_certificate     /usr/local/etc/letsencrypt/live/video.berocs.com/fullchain.pem;
  ssl_certificate_key /usr/local/etc/letsencrypt/live/video.berocs.com/privkey.pem;

  location ^~ '/.well-known/acme-challenge' {
    default_type "text/plain";
    root /var/www/certbot;
  }

  ##
  # Security hardening (as of Nov 15, 2020)
  # based on Mozilla Guideline v5.6
  ##

  ssl_protocols             TLSv1.2 TLSv1.3;
  ssl_prefer_server_ciphers on;
  ssl_ciphers               ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256; # add ECDHE-RSA-AES256-SHA if you want compatibility with Android 4
  ssl_session_timeout       1d; # defaults to 5m
  #ssl_session_cache         shared:SSL:10m; # estimated to 40k sessions
  ssl_session_tickets       off;
  ssl_stapling              on;
  ssl_stapling_verify       on;
  # HSTS (https://hstspreload.org), requires to be copied in 'location' sections that have add_header directives
  #add_header Strict-Transport-Security "max-age=63072000; includeSubDomains";

  ##
  # Application
  ##

  location @api {
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header Host            $host;
    proxy_set_header X-Real-IP       $remote_addr;

    client_max_body_size  100k; # default is 1M

    proxy_connect_timeout 10m;
    proxy_send_timeout    10m;
    proxy_read_timeout    10m;
    send_timeout          10m;

    proxy_pass http://backend;
  }

  location / {
    try_files /dev/null @api;
  }

  location = /api/v1/videos/upload-resumable {
    client_max_body_size    0;
    proxy_request_buffering off;

    try_files /dev/null @api;
  }

  location = /api/v1/videos/upload {
    limit_except POST HEAD { deny all; }

    # This is the maximum upload size, which roughly matches the maximum size of a video file.
    # Note that temporary space is needed equal to the total size of all concurrent uploads.
    # This data gets stored in /var/lib/nginx by default, so you may want to put this directory
    # on a dedicated filesystem.
    client_max_body_size                      12G; # default is 1M
    add_header            X-File-Maximum-Size 8G always; # inform backend of the set value in bytes before mime-encoding (x * 1.4 >= client_max_body_size)

    try_files /dev/null @api;
  }

  location ~ ^/api/v1/(videos|video-playlists|video-channels|users/me) {
    client_max_body_size                      6M; # default is 1M
    add_header            X-File-Maximum-Size 4M always; # inform backend of the set value in bytes before mime-encoding (x * 1.4 >= client_max_body_size)

    try_files /dev/null @api;
  }

  ##
  # Websocket
  ##

  location @api_websocket {
    proxy_http_version 1.1;
    proxy_set_header   X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header   Host            $host;
    proxy_set_header   X-Real-IP       $remote_addr;
    proxy_set_header   Upgrade         $http_upgrade;
    proxy_set_header   Connection      "upgrade";

    proxy_pass http://backend;
  }

  location /socket.io {
    try_files /dev/null @api_websocket;
  }

  location /tracker/socket {
    # Peers send a message to the tracker every 15 minutes
    # Don't close the websocket before then
    proxy_read_timeout 15m; # default is 60s

    try_files /dev/null @api_websocket;
  }

  ##
  # Performance optimizations
  # For extra performance please refer to https://github.com/denji/nginx-tuning
  ##

  root /var/www/peertube/storage;

  # Enable compression for JS/CSS/HTML, for improved client load times.
  # It might be nice to compress JSON/XML as returned by the API, but
  # leaving that out to protect against potential BREACH attack.
  gzip              on;
  gzip_vary         on;
  gzip_types        # text/html is always compressed by HttpGzipModule
                    text/css
                    application/javascript
                    font/truetype
                    font/opentype
                    application/vnd.ms-fontobject
                    image/svg+xml;
  gzip_min_length   1000; # default is 20 bytes
  gzip_buffers      16 8k;
  gzip_comp_level   2; # default is 1

  client_body_timeout       30s; # default is 60
  client_header_timeout     10s; # default is 60
  send_timeout              10s; # default is 60
  keepalive_timeout         10s; # default is 75
  resolver_timeout          10s; # default is 30
  reset_timedout_connection on;
  proxy_ignore_client_abort on;

  tcp_nopush                on; # send headers in one piece
  tcp_nodelay               on; # don't buffer data sent, good for small data bursts in real time

  # If you have a small /var/lib partition, it could be interesting to store temp nginx uploads in a different place
  # See https://nginx.org/en/docs/http/ngx_http_core_module.html#client_body_temp_path
  #client_body_temp_path /var/www/peertube/storage/nginx/;

  # Bypass PeerTube for performance reasons. Optional.
  # Should be consistent with client-overrides assets list in /server/controllers/client.ts
  location ~ ^/client/(assets/images/(icons/icon-36x36\.png|icons/icon-48x48\.png|icons/icon-72x72\.png|icons/icon-96x96\.png|icons/icon-144x144\.png|icons/icon-192x192\.png|icons/icon-512x512\.png|logo\.svg|favicon\.png))$ {
    add_header Cache-Control "public, max-age=31536000, immutable"; # Cache 1 year

    root /var/www/peertube;

    try_files /storage/client-overrides/$1 /peertube-latest/client/dist/$1 @api;
  }

  # Bypass PeerTube for performance reasons. Optional.
  location ~ ^/client/(.*\.(js|css|png|svg|woff2|otf|ttf|woff|eot))$ {
    add_header Cache-Control "public, max-age=31536000, immutable"; # Cache 1 year

    alias /var/www/peertube/peertube-latest/client/dist/$1;
  }

  # Bypass PeerTube for performance reasons. Optional.
  location ~ ^/static/(thumbnails|avatars)/ {
    if ($request_method = 'OPTIONS') {
      add_header Access-Control-Allow-Origin  '*';
      add_header Access-Control-Allow-Methods 'GET, OPTIONS';
      add_header Access-Control-Allow-Headers 'Range,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
      add_header Access-Control-Max-Age       1728000; # Preflight request can be cached 20 days
      add_header Content-Type                 'text/plain charset=UTF-8';
      add_header Content-Length               0;
      return 204;
    }

    add_header Access-Control-Allow-Origin    '*';
    add_header Access-Control-Allow-Methods   'GET, OPTIONS';
    add_header Access-Control-Allow-Headers   'Range,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
    add_header Cache-Control                  "public, max-age=7200"; # Cache response 2 hours

    rewrite ^/static/(.*)$ /$1 break;

    try_files $uri @api;
  }

  # Bypass PeerTube for performance reasons. Optional.
  location ~ ^/static/(webseed|redundancy|streaming-playlists)/ {
    limit_rate_after            5M;

    # Clients usually have 4 simultaneous webseed connections, so the real limit is 3MB/s per client
    set $peertube_limit_rate    800k;

    # Increase rate limit in HLS mode, because we don't have multiple simultaneous connections
    if ($request_uri ~ -fragmented.mp4$) {
      set $peertube_limit_rate  5M;
    }

    # Use this line with nginx >= 1.17.0
    #limit_rate $peertube_limit_rate;
    # Or this line if your nginx < 1.17.0
    set $limit_rate $peertube_limit_rate;

    if ($request_method = 'OPTIONS') {
      add_header Access-Control-Allow-Origin  '*';
      add_header Access-Control-Allow-Methods 'GET, OPTIONS';
      add_header Access-Control-Allow-Headers 'Range,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
      add_header Access-Control-Max-Age       1728000; # Preflight request can be cached 20 days
      add_header Content-Type                 'text/plain charset=UTF-8';
      add_header Content-Length               0;
      return 204;
    }

    if ($request_method = 'GET') {
      add_header Access-Control-Allow-Origin  '*';
      add_header Access-Control-Allow-Methods 'GET, OPTIONS';
      add_header Access-Control-Allow-Headers 'Range,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';

      # Don't spam access log file with byte range requests
      access_log off;
    }

    # Enabling the sendfile directive eliminates the step of copying the data into the buffer
    # and enables direct copying data from one file descriptor to another.
    sendfile on;
    sendfile_max_chunk 1M; # prevent one fast connection from entirely occupying the worker process. should be > 800k.
    aio threads;

    rewrite ^/static/webseed/(.*)$ /videos/$1 break;
    rewrite ^/static/(.*)$         /$1        break;

    try_files $uri @api;
  }
}

That doesn’t work, and navigating to the page returns 502 Bad Gateway.

Next I tried following along with this thread but - as was discussed there - it results in a 301 loop since apparently proxy_pass sends only to port 80(?).

I’d appreciate any pointers on how to do this properly!

Found the issue: error existed between chair and keyboard.

The instance was hanging up on a database initialization issue, and wasn’t starting.

Using the peertube nginx config as a remote reverse proxy is now working!

Can you show the configurations of NGINX, main proxy server and Peertube server?

1 « J'aime »

Sure thing!

My original post was fueled by a lack of understanding in how Peertube works, so I should clarify what I had wrong:

  • I had run into trouble setting up Peertube the first time (several versions back) and assumed that the Peertube process itself wanted the SSL cert files to be on the same system as it. Easy to look back now and see how silly that is, because the permissions on LetsEncrypt certs is carefully controlled, and Peertube doesn’t have any access to them anyway.
  • Because of the above point I expected that Peertube and the nginx service that runs the included config had to be on the same machine. That is not the case (at least, not anymore).
  • Since I was moving the Peertube instance to a new machine, I thought that I would not be able to access/see anything by browsing the Peertube machine’s local IP address on port 9000. That is not the case, Peertube loads (if it is truly running) when I visit http://peertube-ip-address:9000.

Some things to note:

  • The " old " server is a FreeBSD jail (similar to a virtual machine (VM)) with IP 192.168.13.203
  • The old server has a multi-config nginx process that proxies requests for different domains to different backend servers. It also handles SSL certs for everything. It used to also have Peertube running on it.
  • The " new " server is an Ubuntu VM with IP 192.168.13.88. It only has Peertube running on it, no nginx process!

What I was initially trying to do:

                     ┌─────────────────┐      ┌────────────────────────────────────┐
                     │  FreeBSD Jail   │      │                Ubuntu VM           │
┌───────────────┐    │ ┌─────────────┐ │      │ ┌───────────────┐                  │
│               │    │ │    Nginx    │ │      │ │     Nginx     │    ┌──────────┐  │
│ The Internet  ├───►│ │Reverse Proxy│ ├─────►│ │Peertube Config├───►│ Peertube │  │
│               │    │ └─────────────┘ │      │ └───────────────┘    └──────────┘  │
└───────────────┘    │                 │      │                                    │
                     └─────────────────┘      └────────────────────────────────────┘

What I wound up doing, that actually works:

                     ┌───────────────────┐      ┌──────────────┐
                     │  FreeBSD Jail     │      │  Ubuntu VM   │
┌───────────────┐    │ ┌───────────────┐ │      │              │
│               │    │ │     Nginx     │ │      │ ┌──────────┐ │
│ The Internet  ├───►│ │Peertube Config│ ├─────►│ │ Peertube │ │
│               │    │ └───────────────┘ │      │ └──────────┘ │
└───────────────┘    │                   │      │              │
                     └───────────────────┘      └──────────────┘

This means that when Peertube updates, I have to take the new Peertube example nginx config and copy it to the FreeBSD jail, and then compare it to the production config running there. It also means that the SSL certs are all handled on the FreeBSD box, and are not shared at all with the Ubuntu VM.

I assume this setup also invalidates the nginx caching that recent Peertube configs do. If it does, it fails gracefully and the site still loads for me.

Without any further ado, here’s the Peertube nginx config that’s running on the FreeBSD machine. Note that I had to set the backend IP address.

# cat /usr/local/etc/nginx/sites-available/peertube.conf 
# Minimum Nginx version required:  1.13.0 (released Apr 25, 2017)
# Please check your Nginx installation features the following modules via 'nginx -V':
# STANDARD HTTP MODULES: Core, Proxy, Rewrite, Access, Gzip, Headers, HTTP/2, Log, Real IP, SSL, Thread Pool, Upstream, AIO Multithreading.
# THIRD PARTY MODULES:   None.

server {
  listen 80;
  listen [::]:80;
  server_name video.berocs.com;

  location /.well-known/acme-challenge/ {
    default_type "text/plain";
    root /var/www/certbot;
  }
  location / { return 301 https://$host$request_uri; }
}

upstream backend {
  server 192.168.13.88:9000; ### <-- THIS IS THE UBUNTU VM IP ADDRESS
}

server {
  listen 443 ssl http2;
  listen [::]:443 ssl http2;
  server_name video.berocs.com;

  access_log /var/log/nginx/peertube.access.log; # reduce I/0 with buffer=10m flush=5m
  error_log  /var/log/nginx/peertube.error.log;

  ##
  # Certificates
  # you need a certificate to run in production. see https://letsencrypt.org/
  ##
  #ssl_certificate     /etc/letsencrypt/live/video.berocs.com/fullchain.pem;
  #ssl_certificate_key /etc/letsencrypt/live/video.berocs.com/privkey.pem;
  ssl_certificate     /usr/local/etc/letsencrypt/live/video.berocs.com/fullchain.pem;
  ssl_certificate_key /usr/local/etc/letsencrypt/live/video.berocs.com/privkey.pem;

  location ^~ '/.well-known/acme-challenge' {
    default_type "text/plain";
    root /var/www/certbot;
  }

  ##
  # Security hardening (as of Nov 15, 2020)
  # based on Mozilla Guideline v5.6
  ##

  ssl_protocols             TLSv1.2 TLSv1.3;
  ssl_prefer_server_ciphers on;
  ssl_ciphers               ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256; # add ECDHE-RSA-AES256-SHA if you want compatibility with Android 4
  ssl_session_timeout       1d; # defaults to 5m
  #ssl_session_cache         shared:SSL:10m; # estimated to 40k sessions
  ssl_session_tickets       off;
  ssl_stapling              on;
  ssl_stapling_verify       on;
  # HSTS (https://hstspreload.org), requires to be copied in 'location' sections that have add_header directives
  #add_header Strict-Transport-Security "max-age=63072000; includeSubDomains";

  ##
  # Application
  ##

  location @api {
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header Host            $host;
    proxy_set_header X-Real-IP       $remote_addr;

    client_max_body_size  100k; # default is 1M

    proxy_connect_timeout 10m;
    proxy_send_timeout    10m;
    proxy_read_timeout    10m;
    send_timeout          10m;

    proxy_pass http://backend;
  }

  location / {
    try_files /dev/null @api;
  }

  location = /api/v1/videos/upload-resumable {
    client_max_body_size    0;
    proxy_request_buffering off;

    try_files /dev/null @api;
  }

  location = /api/v1/videos/upload {
    limit_except POST HEAD { deny all; }

    # This is the maximum upload size, which roughly matches the maximum size of a video file.
    # Note that temporary space is needed equal to the total size of all concurrent uploads.
    # This data gets stored in /var/lib/nginx by default, so you may want to put this directory
    # on a dedicated filesystem.
    client_max_body_size                      12G; # default is 1M
    add_header            X-File-Maximum-Size 8G always; # inform backend of the set value in bytes before mime-encoding (x * 1.4 >= client_max_body_size)

    try_files /dev/null @api;
  }

  location ~ ^/api/v1/(videos|video-playlists|video-channels|users/me) {
    client_max_body_size                      6M; # default is 1M
    add_header            X-File-Maximum-Size 4M always; # inform backend of the set value in bytes before mime-encoding (x * 1.4 >= client_max_body_size)

    try_files /dev/null @api;
  }

  ##
  # Websocket
  ##

  location @api_websocket {
    proxy_http_version 1.1;
    proxy_set_header   X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header   Host            $host;
    proxy_set_header   X-Real-IP       $remote_addr;
    proxy_set_header   Upgrade         $http_upgrade;
    proxy_set_header   Connection      "upgrade";

    proxy_pass http://backend;
  }

  location /socket.io {
    try_files /dev/null @api_websocket;
  }

  location /tracker/socket {
    # Peers send a message to the tracker every 15 minutes
    # Don't close the websocket before then
    proxy_read_timeout 15m; # default is 60s

    try_files /dev/null @api_websocket;
  }

  ##
  # Performance optimizations
  # For extra performance please refer to https://github.com/denji/nginx-tuning
  ##

  root /var/www/peertube/storage;

  # Enable compression for JS/CSS/HTML, for improved client load times.
  # It might be nice to compress JSON/XML as returned by the API, but
  # leaving that out to protect against potential BREACH attack.
  gzip              on;
  gzip_vary         on;
  gzip_types        # text/html is always compressed by HttpGzipModule
                    text/css
                    application/javascript
                    font/truetype
                    font/opentype
                    application/vnd.ms-fontobject
                    image/svg+xml;
  gzip_min_length   1000; # default is 20 bytes
  gzip_buffers      16 8k;
  gzip_comp_level   2; # default is 1

  client_body_timeout       30s; # default is 60
  client_header_timeout     10s; # default is 60
  send_timeout              10s; # default is 60
  keepalive_timeout         10s; # default is 75
  resolver_timeout          10s; # default is 30
  reset_timedout_connection on;
  proxy_ignore_client_abort on;

  tcp_nopush                on; # send headers in one piece
  tcp_nodelay               on; # don't buffer data sent, good for small data bursts in real time

  # If you have a small /var/lib partition, it could be interesting to store temp nginx uploads in a different place
  # See https://nginx.org/en/docs/http/ngx_http_core_module.html#client_body_temp_path
  #client_body_temp_path /var/www/peertube/storage/nginx/;

  # Bypass PeerTube for performance reasons. Optional.
  # Should be consistent with client-overrides assets list in /server/controllers/client.ts
  location ~ ^/client/(assets/images/(icons/icon-36x36\.png|icons/icon-48x48\.png|icons/icon-72x72\.png|icons/icon-96x96\.png|icons/icon-144x144\.png|icons/icon-192x192\.png|icons/icon-512x512\.png|logo\.svg|favicon\.png))$ {
    add_header Cache-Control "public, max-age=31536000, immutable"; # Cache 1 year

    root /var/www/peertube;

    try_files /storage/client-overrides/$1 /peertube-latest/client/dist/$1 @api;
  }

  # Bypass PeerTube for performance reasons. Optional.
  location ~ ^/client/(.*\.(js|css|png|svg|woff2|otf|ttf|woff|eot))$ {
    add_header Cache-Control "public, max-age=31536000, immutable"; # Cache 1 year

    alias /var/www/peertube/peertube-latest/client/dist/$1;
  }

  # Bypass PeerTube for performance reasons. Optional.
  location ~ ^/static/(thumbnails|avatars)/ {
    if ($request_method = 'OPTIONS') {
      add_header Access-Control-Allow-Origin  '*';
      add_header Access-Control-Allow-Methods 'GET, OPTIONS';
      add_header Access-Control-Allow-Headers 'Range,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
      add_header Access-Control-Max-Age       1728000; # Preflight request can be cached 20 days
      add_header Content-Type                 'text/plain charset=UTF-8';
      add_header Content-Length               0;
      return 204;
    }

    add_header Access-Control-Allow-Origin    '*';
    add_header Access-Control-Allow-Methods   'GET, OPTIONS';
    add_header Access-Control-Allow-Headers   'Range,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
    add_header Cache-Control                  "public, max-age=7200"; # Cache response 2 hours

    rewrite ^/static/(.*)$ /$1 break;

    try_files $uri @api;
  }

  # Bypass PeerTube for performance reasons. Optional.
  location ~ ^/static/(webseed|redundancy|streaming-playlists)/ {
    limit_rate_after            5M;

    # Clients usually have 4 simultaneous webseed connections, so the real limit is 3MB/s per client
    set $peertube_limit_rate    800k;

    # Increase rate limit in HLS mode, because we don't have multiple simultaneous connections
    if ($request_uri ~ -fragmented.mp4$) {
      set $peertube_limit_rate  5M;
    }

    # Use this line with nginx >= 1.17.0
    #limit_rate $peertube_limit_rate;
    # Or this line if your nginx < 1.17.0
    set $limit_rate $peertube_limit_rate;

    if ($request_method = 'OPTIONS') {
      add_header Access-Control-Allow-Origin  '*';
      add_header Access-Control-Allow-Methods 'GET, OPTIONS';
      add_header Access-Control-Allow-Headers 'Range,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
      add_header Access-Control-Max-Age       1728000; # Preflight request can be cached 20 days
      add_header Content-Type                 'text/plain charset=UTF-8';
      add_header Content-Length               0;
      return 204;
    }

    if ($request_method = 'GET') {
      add_header Access-Control-Allow-Origin  '*';
      add_header Access-Control-Allow-Methods 'GET, OPTIONS';
      add_header Access-Control-Allow-Headers 'Range,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';

      # Don't spam access log file with byte range requests
      access_log off;
    }

    # Enabling the sendfile directive eliminates the step of copying the data into the buffer
    # and enables direct copying data from one file descriptor to another.
    sendfile on;
    sendfile_max_chunk 1M; # prevent one fast connection from entirely occupying the worker process. should be > 800k.
    aio threads;

    rewrite ^/static/webseed/(.*)$ /videos/$1 break;
    rewrite ^/static/(.*)$         /$1        break;

    try_files $uri @api;
  }
}

And on the Ubuntu VM, the top of Peertube’s production.yaml file looks like. Note that I edited the hostname on line 2, and the trust_proxy IP address.

# cat config/production.yaml 
listen:
  hostname: '0.0.0.0'
  port: 9000

# Correspond to your reverse proxy server_name/listen configuration (i.e., your public PeerTube instance URL)
webserver:
  https: true
  hostname: 'video.berocs.com'
  port: 443

rates_limit:
  api:
    # 50 attempts in 10 seconds
    window: 10 seconds
    max: 50
  login:
    # 15 attempts in 5 min
    window: 5 minutes
    max: 15
  signup:
    # 2 attempts in 5 min (only succeeded attempts are taken into account)
    window: 5 minutes
    max: 2
  ask_send_email:
    # 3 attempts in 5 min
    window: 5 minutes
    max: 3

# Proxies to trust to get real client IP
# If you run PeerTube just behind a local proxy (nginx), keep 'loopback'
# If you run PeerTube behind a remote proxy, add the proxy IP address (or subnet)
trust_proxy:
  - '192.168.13.203' # <-- THIS IS THE FREEBSD NGINX IP ADDRESS

Thanks for the detailed answer. I’ve got one more question…
The NGINX configuration specifies the Peertube directory and the Storage directory, namely:

root /var/www/peertube;

and

root /var/www/peertube/storage;

What if the Peertube directory is on your Ubuntu server (192.168.13.88)? Probably the Storage is also located on the same server.

This is the most important question for me!

I am now trying to set up such a configuration, but I have a blank white page - (If I specify the wrong port (for example 9002), then an error appears 502 - it means the proxy is working. But why am I seeing a blank page?

I am now trying to set up such a configuration, but I have a blank white page - (If I specify the wrong port (for example 9002), then an error appears 502 - it means the proxy is working. But why am I seeing a blank page?

That’s a great question!

I was going to respond with some more info about how it’s working for me, but then I took a closer look. Turns out I had left the old peertube folder on my FreeBSD server, and nginx was still serving the static content from there. That would have caused me quite a bit of trouble whenever the next video got around to being uploaded.

I removed the peertube directory, and now I am also seeing the blank white page!

But this should work. It looks like peertube has the config to support this kind of reverse proxy. So let’s see if anything can be done.


1st attempt at a workaround, this did not work.

I looked at the peertube error log and config, and noticed that it was using alias in a place where we want files to be proxied, so I replaced the line with try_files:

 # Bypass PeerTube for performance reasons. Optional.
  location ~ ^/client/(.*\.(js|css|png|svg|woff2|otf|ttf|woff|eot))$ {
    add_header Cache-Control "public, max-age=31536000, immutable"; # Cache 1 year

    #alias /var/www/peertube/peertube-latest/client/dist/$1;    # <-- Won't proxy to peertube
    try_files /var/www/peertube/peertube-latest/client/dist/$1 @api;    # <-- Will proxy
  }

That worked! Now the home page loads… except for thumbnails, which do not. Confusingly, in the network inspector the thumbnail URLs are returning ‹ 200 OK › status. So why aren’t they loading?

If I copy the URL and open it in a browser, it returns a full webpage that says « Error 404 » but in itself is not returning HTTP status code 404. So essentially we are getting a 404, just not a proper one. Maybe that’s a bug with Peertube.

Back in the peertube config, it looks like a rewrite directive is mangling the URL so that it can be fetched by nginx, but then fails to be found by the @api proxy… Maybe another bug?

  # Bypass PeerTube for performance reasons. Optional.
  location ~ ^/static/(thumbnails|avatars)/ {
    if ($request_method = 'OPTIONS') {
      add_header Access-Control-Allow-Origin  '*';
      add_header Access-Control-Allow-Methods 'GET, OPTIONS';
      add_header Access-Control-Allow-Headers 'Range,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
      add_header Access-Control-Max-Age       1728000; # Preflight request can be cached 20 days
      add_header Content-Type                 'text/plain charset=UTF-8';
      add_header Content-Length               0;
      return 204;
    }

    add_header Access-Control-Allow-Origin    '*';
    add_header Access-Control-Allow-Methods   'GET, OPTIONS';
    add_header Access-Control-Allow-Headers   'Range,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
    add_header Cache-Control                  "public, max-age=7200"; # Cache response 2 hours

    #rewrite ^/static/(.*)$ /$1 break;

    try_files $uri @api;
  }

I commented out the rewrite line and now thumbnails load!

So close! But now when I click on a video and go to the watch page, the video fails to play. The network inspector shows that the video file and fragments list both return 404…

I spent the most time trying to figure out what was going on here. Initially I thought the rewrite statements in the /static/ section of the nginx config were messing up the request, just like they had for the thumbnails. But removing these did not help. Confusingly, nginx appeared to refuse to use the @api proxy, insisting on trying to open the file, and then returning 404 when the file wasn’t found.

Eventually I figured out that the Peertube process doesn’t (appear to) support sending the video file itself. Instead it sends a new request that’s supposed to have nginx grab the file from disk…

Here’s a screenshot of loading the site by the local Ubuntu IP address:

So Peertube is requesting to go back out and use nginx to grab the file there.

That’s as far as I’m able to troubleshoot. Maybe someone who actually knows what they’re doing can figure out a solution.


2nd attempt at a workaround, this does work for me.

So, I wish the above changes had worked. Since they do not, the only solution I can think of is to mirror the Peertube folder between the two servers. To do this I set /var/www/peertube up as an NFS mount that is shared from the FreeBSD/nginx to the Ubuntu/Peertube one. In my reasoning, that way the nginx caching can access the files fast, and hopefully webpage loading speed won’t be seriously affected.

Once the NFS share was configured, I undid the nginx config changes made in attempt #1 above.

# cat /etc/fstab 
# /etc/fstab: static file system information.
# <file system> <mount point>   <type>  <options>       <dump>  <pass>

192.168.x.x:/mnt/main-pool/jail-share/peertube-storage /var/www/peertube nfs defaults 0 0

In my very basic testing this is noticeably slower, but at least it works at all.

Without NFS share:

With NFS share:

Just thinking out loud here. My conclusion to attempt #1 in the previous post isn’t right.

If I copy the path to the 404’ed master.m3u8 and append it to the Ubuntu IP address, the Peertube service does appear to produce the file:

So this must still be an nginx issue here, where the reverse proxy isn’t working, and nginx is resorting to trying to grab the file from disk… Or something?

OK, I’m not smart enough to know what’s going on in this nginx config. But I have it working.

If I remove the NFS mounts, and follow the config change steps from Attempt #1 above, and then also do this next step, HLS videos will load and play for me.

I just commented out the /static/ block at the end of the config. Somehow that gets nginx to respect the @api proxy and get the video file from the peertube process. I wish I knew why.

The whole nginx config now looks like this:

# cat /usr/local/etc/nginx/sites-available/peertube.conf
# Minimum Nginx version required:  1.13.0 (released Apr 25, 2017)
# Please check your Nginx installation features the following modules via 'nginx -V':
# STANDARD HTTP MODULES: Core, Proxy, Rewrite, Access, Gzip, Headers, HTTP/2, Log, Real IP, SSL, Thread Pool, Upstream, AIO Multithreading.
# THIRD PARTY MODULES:   None.

server {
  listen 80;
  listen [::]:80;
  server_name video.berocs.com;

  location /.well-known/acme-challenge/ {
    default_type "text/plain";
    root /var/www/certbot;
  }
  location / { return 301 https://$host$request_uri; }
}

upstream backend {
  server 192.168.13.88:9000;
}

server {
  listen 443 ssl http2;
  listen [::]:443 ssl http2;
  server_name video.berocs.com;

  access_log /var/log/nginx/peertube.access.log; # reduce I/0 with buffer=10m flush=5m
  error_log  /var/log/nginx/peertube.error.log info;

  ##
  # Certificates
  # you need a certificate to run in production. see https://letsencrypt.org/
  ##
  #ssl_certificate     /etc/letsencrypt/live/video.berocs.com/fullchain.pem;
  #ssl_certificate_key /etc/letsencrypt/live/video.berocs.com/privkey.pem;
  ssl_certificate     /usr/local/etc/letsencrypt/live/video.berocs.com/fullchain.pem;
  ssl_certificate_key /usr/local/etc/letsencrypt/live/video.berocs.com/privkey.pem;

  location ^~ '/.well-known/acme-challenge' {
    default_type "text/plain";
    root /var/www/certbot;
  }

  ##
  # Security hardening (as of Nov 15, 2020)
  # based on Mozilla Guideline v5.6
  ##

  ssl_protocols             TLSv1.2 TLSv1.3;
  ssl_prefer_server_ciphers on;
  ssl_ciphers               ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256; # add ECDHE-RSA-AES256-SHA if you want compatibility with Android 4
  ssl_session_timeout       1d; # defaults to 5m
  #ssl_session_cache         shared:SSL:10m; # estimated to 40k sessions
  ssl_session_tickets       off;
  ssl_stapling              on;
  ssl_stapling_verify       on;
  # HSTS (https://hstspreload.org), requires to be copied in 'location' sections that have add_header directives
  #add_header Strict-Transport-Security "max-age=63072000; includeSubDomains";

  ##
  # Application
  ##

  location @api {
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header Host            $host;
    proxy_set_header X-Real-IP       $remote_addr;

    client_max_body_size  100k; # default is 1M

    proxy_connect_timeout 10m;
    proxy_send_timeout    10m;
    proxy_read_timeout    10m;
    send_timeout          10m;

    proxy_pass http://backend;
  }

  location / {
    try_files /dev/null @api;
  }

  location = /api/v1/videos/upload-resumable {
    client_max_body_size    0;
    proxy_request_buffering off;

    try_files /dev/null @api;
  }

  location = /api/v1/videos/upload {
    limit_except POST HEAD { deny all; }

    # This is the maximum upload size, which roughly matches the maximum size of a video file.
    # Note that temporary space is needed equal to the total size of all concurrent uploads.
    # This data gets stored in /var/lib/nginx by default, so you may want to put this directory
    # on a dedicated filesystem.
    client_max_body_size                      12G; # default is 1M
    add_header            X-File-Maximum-Size 8G always; # inform backend of the set value in bytes before mime-encoding (x * 1.4 >= client_max_body_size)

    try_files /dev/null @api;
  }

  location ~ ^/api/v1/(videos|video-playlists|video-channels|users/me) {
    client_max_body_size                      6M; # default is 1M
    add_header            X-File-Maximum-Size 4M always; # inform backend of the set value in bytes before mime-encoding (x * 1.4 >= client_max_body_size)

    try_files /dev/null @api;
  }

  ##
  # Websocket
  ##

  location @api_websocket {
    proxy_http_version 1.1;
    proxy_set_header   X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header   Host            $host;
    proxy_set_header   X-Real-IP       $remote_addr;
    proxy_set_header   Upgrade         $http_upgrade;
    proxy_set_header   Connection      "upgrade";

    proxy_pass http://backend;
  }

  location /socket.io {
    try_files /dev/null @api_websocket;
  }

  location /tracker/socket {
    # Peers send a message to the tracker every 15 minutes
    # Don't close the websocket before then
    proxy_read_timeout 15m; # default is 60s

    try_files /dev/null @api_websocket;
  }

  ##
  # Performance optimizations
  # For extra performance please refer to https://github.com/denji/nginx-tuning
  ##

  root /var/www/peertube/storage;

  # Enable compression for JS/CSS/HTML, for improved client load times.
  # It might be nice to compress JSON/XML as returned by the API, but
  # leaving that out to protect against potential BREACH attack.
  gzip              on;
  gzip_vary         on;
  gzip_types        # text/html is always compressed by HttpGzipModule
                    text/css
                    application/javascript
                    font/truetype
                    font/opentype
                    application/vnd.ms-fontobject
                    image/svg+xml;
  gzip_min_length   1000; # default is 20 bytes
  gzip_buffers      16 8k;
  gzip_comp_level   2; # default is 1

  client_body_timeout       30s; # default is 60
  client_header_timeout     10s; # default is 60
  send_timeout              10s; # default is 60
  keepalive_timeout         10s; # default is 75
  resolver_timeout          10s; # default is 30
  reset_timedout_connection on;
  proxy_ignore_client_abort on;

  tcp_nopush                on; # send headers in one piece
  tcp_nodelay               on; # don't buffer data sent, good for small data bursts in real time

  # If you have a small /var/lib partition, it could be interesting to store temp nginx uploads in a different place
  # See https://nginx.org/en/docs/http/ngx_http_core_module.html#client_body_temp_path
  #client_body_temp_path /var/www/peertube/storage/nginx/;

  # Bypass PeerTube for performance reasons. Optional.
  # Should be consistent with client-overrides assets list in /server/controllers/client.ts
  location ~ ^/client/(assets/images/(icons/icon-36x36\.png|icons/icon-48x48\.png|icons/icon-72x72\.png|icons/icon-96x96\.png|icons/icon-144x144\.png|icons/icon-192x192\.png|icons/icon-512x512\.png|logo\.svg|favicon\.png))$ {
    add_header Cache-Control "public, max-age=31536000, immutable"; # Cache 1 year

    root /var/www/peertube;

    try_files /storage/client-overrides/$1 /peertube-latest/client/dist/$1 @api;
  }

  # Bypass PeerTube for performance reasons. Optional.
  location ~ ^/client/(.*\.(js|css|png|svg|woff2|otf|ttf|woff|eot))$ {
    add_header Cache-Control "public, max-age=31536000, immutable"; # Cache 1 year

    #alias /var/www/peertube/peertube-latest/client/dist/$1;
    try_files /var/www/peertube/peertube-latest/client/dist/$1 @api;
  }

  # Bypass PeerTube for performance reasons. Optional.
  location ~ ^/static/(thumbnails|avatars)/ {
    if ($request_method = 'OPTIONS') {
      add_header Access-Control-Allow-Origin  '*';
      add_header Access-Control-Allow-Methods 'GET, OPTIONS';
      add_header Access-Control-Allow-Headers 'Range,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
      add_header Access-Control-Max-Age       1728000; # Preflight request can be cached 20 days
      add_header Content-Type                 'text/plain charset=UTF-8';
      add_header Content-Length               0;
      return 204;
    }

    add_header Access-Control-Allow-Origin    '*';
    add_header Access-Control-Allow-Methods   'GET, OPTIONS';
    add_header Access-Control-Allow-Headers   'Range,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
    add_header Cache-Control                  "public, max-age=7200"; # Cache response 2 hours

    #rewrite ^/static/(.*)$ /$1 break;

    try_files $uri @api;
  }

  # Bypass PeerTube for performance reasons. Optional.
  #location ~ ^/static/(webseed|redundancy|streaming-playlists)/ {
  #  limit_rate_after            5M;

  #  # Clients usually have 4 simultaneous webseed connections, so the real limit is 3MB/s per client
  #  set $peertube_limit_rate    800k;

  #  # Increase rate limit in HLS mode, because we don't have multiple simultaneous connections
  #  if ($request_uri ~ -fragmented.mp4$) {
  #    set $peertube_limit_rate  5M;
  #  }

  #  # Use this line with nginx >= 1.17.0
  #  limit_rate $peertube_limit_rate;
  #  # Or this line if your nginx < 1.17.0
  #  #set $limit_rate $peertube_limit_rate;

  #  if ($request_method = 'OPTIONS') {
  #    add_header Access-Control-Allow-Origin  '*';
  #    add_header Access-Control-Allow-Methods 'GET, OPTIONS';
  #    add_header Access-Control-Allow-Headers 'Range,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
  #    add_header Access-Control-Max-Age       1728000; # Preflight request can be cached 20 days
  #    add_header Content-Type                 'text/plain charset=UTF-8';
  #    add_header Content-Length               0;
  #    return 204;
  #  }

  #  if ($request_method = 'GET') {
  #    add_header Access-Control-Allow-Origin  '*';
  #    add_header Access-Control-Allow-Methods 'GET, OPTIONS';
  #    add_header Access-Control-Allow-Headers 'Range,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';

  #    # Don't spam access log file with byte range requests
  #    access_log off;
  #  }

  #  # Enabling the sendfile directive eliminates the step of copying the data into the buffer
  #  # and enables direct copying data from one file descriptor to another.
  #  sendfile on;
  #  sendfile_max_chunk 1M; # prevent one fast connection from entirely occupying the worker process. should be > 800k.
  #  aio threads;

  #  #rewrite ^/static/webseed/(.*)$ /videos/$1 break;
  #  #rewrite ^/static/(.*)$         /$1        break;

  #  #try_files $uri @api;
  #  try_files /dev/null @api;
  #}
}