Making Slurm, Jupyterhub, letsencrypt and Nginx work together

I spent too much time on this and couldn’t find a good resource for a simple jupyterhub setup that would spawn notebooks in a slurm queue and still work with nginx as a reverse proxy with letsencrypt handling ssl.

Required packages:

pip, nginx, letsencrypt, python3-venv

pip packages:

jupyterhub, batchspawner

Jupyterhub Config

jupyterhub is installed in a virtual environment via pip in /opt/jupyterhub with the following config:

import batchspawner
import os

c.Spawner.env_keep = [ 'PATH',
'PYTHONPATH',
'CONDA_ROOT',
'CONDA_DEFAULT_ENV',
'VIRTUAL_ENV',
'LANG',
'LC_ALL',
'R_LIBS_SITE',
'LD_LIBRARY_PATH',
'LIBRARY_PATH',
'HTTP_PROXY',
'HTTPS_PROXY',
'NO_PROXY',
'SSL_CERT_DIR',
'CA_BUNDLE',
'SSL_CERT_FILE',
'JUPYTERLAB_DIR',
'JUPYTER_CONFIG_PATH',
'JUPYTER_CONFIG_DIR',
'JUPYTERHUB_API_URL',
'JUPYTERHUB_BASE_URL',
'JUPYTERHUB_CLIENT_ID',
'JUPYTERHUB_OAUTH_CALLBACK_URL',
'JUPYTERHUB_SERVER_NAME',
'JUPYTERHUB_SERVICE_PREFIX',
'JUPYTERHUB_USER'
]

c.JupyterHub.proxy_class = 'jupyterhub.proxy.ConfigurableHTTPProxy'
c.Authenticator.allow_all = True

c.JupyterHub.bind_url = 'http://:8000/jupyter'
c.Spawner.default_url = '/lab'
c.Spawner.args = ['--NotebookApp.allow_origin=*']

c.JupyterHub.hub_ip = 'hostname.com'
c.JupyterHub.spawner_class = 'wrapspawner.ProfilesSpawner'
c.Spawner.http_timeout = 300
c.ProfilesSpawner.profiles = [
#   ("Local server", 'local', 'jupyterhub.spawner.LocalProcessSpawner', {'ip':'0.0.0.0'} ), #uncomment if you want the option to spawn outside the queue
   ('Slurm Cluster Small - 2 CPU cores, 4 GB mem', 'lab', 'batchspawner.SlurmSpawner',
      dict(req_nprocs='2', req_memory='4G', req_partition='basic',)),
   ('Slurm Cluster Medium - 8 CPU cores, 16 GB mem', 'lab', 'batchspawner.SlurmSpawner',
      dict(req_nprocs='8', req_memory='16G', req_partition='basic',)),
   ('Slurm Cluster Large - 16 CPU cores, 32 GB mem', 'lab', 'batchspawner.SlurmSpawner',
      dict(req_nprocs='16', req_memory='32G', req_partition='basic',)),
   ]

c.SlurmSpawner.batch_script = '''#!/bin/bash
#SBATCH --output=/home/%u/jupyterhub_slurmspawner_%j.log
#SBATCH --job-name=spawner-jupyterhub
#SBATCH --chdir=/home/%u
#SBATCH --get-user-env=L
#SBATCH --partition={partition}
#SBATCH --cpus-per-task={nprocs}
#SBATCH --mem={memory}
#SBATCH --export=ALL

srun /opt/jupyterhub/bin/batchspawner-singleuser /opt/jupyterhub/bin/jupyterhub-singleuser --ServerApp.root_dir="$HOME"
echo "jupyterhub-singleuser ended gracefully"
'''

Nginx Config:

map $http_upgrade $connection_upgrade {
        default upgrade;
        '' close;
    }
# Default server configuration
#
server {

	root /var/www/html;

	index index.html index.htm index.nginx-debian.html;

	server_name <hostname>;

	location / {
		# First attempt to serve request as file, then
		# as directory, then fall back to displaying a 404.
		try_files $uri $uri/ =404;
	}

  	location /jupyter/ {
    	# NOTE important to also set base url of jupyterhub to /jupyter in its config
    		proxy_pass http://127.0.0.1:8000;

    		proxy_redirect   off;
    		proxy_set_header X-Real-IP $remote_addr;
    		proxy_set_header Host $host;
    		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    		proxy_set_header X-Forwarded-Proto $scheme;

    		# websocket headers
    		proxy_set_header Upgrade $http_upgrade;
    		proxy_set_header Connection $connection_upgrade;
		client_max_body_size 10M;
  	}	

    location /rstudio/ {
	rewrite ^/rstudio/(.*)$ /$1 break;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;

        proxy_pass http://127.0.0.1:8787/;
        proxy_redirect http://127.0.0.1:8787/ $scheme://$host/rstudio/;
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection $connection_upgrade;
        proxy_read_timeout 20d;
    }


    listen [::]:443 ssl ipv6only=on; # managed by Certbot
    listen 443 ssl; # managed by Certbot
    ssl_certificate /etc/letsencrypt/live/<hostname>/fullchain.pem; # managed by Certbot
    ssl_certificate_key /etc/letsencrypt/live/<hostname>/privkey.pem; # managed by Certbot
    include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
    ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot

}



server {
    if ($host = <hostname>) {
        return 301 https://$host$request_uri;
    } # managed by Certbot


	listen 80 ;
	listen [::]:80 ;

	server_name <hostname>;
    return 404; # managed by Certbot


}