Got it working on my instance by using the following config for the nginx outside of the docker container :

upstream lemmy {
    # depending on your setup, you may want to update this
    server 127.0.0.1:1380;
}

server {
    listen 80;
    listen [::]:80;
    server_name lemmy.pierre-couy.fr;
    location / { return 301 https://$host$request_uri; }
}
server {
    listen      443 ssl;
    listen [::]:443 ssl;
    server_name lemmy.pierre-couy.fr;

    # TLS
    ssl_protocols TLSv1.2;
    ssl_ciphers HIGH:!MEDIUM:!LOW:!aNULL:!NULL:!SHA;
    ssl_prefer_server_ciphers on;
    ssl_session_cache shared:SSL:10m;
    ssl_certificate     /etc/ssl/certs/pcouy.pem;
    ssl_certificate_key /etc/ssl/private/pcouy.key;

    # HSTS
    add_header Strict-Transport-Security "max-age=31536000";

    # Security related headers

    # If you are using S3 to host your files, remember to add your S3 URL to the
    # media-src and img-src headers (e.g. img-src 'self' https://<your-S3-URL> data:)

    # add_header Content-Security-Policy "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self' data:; object-src 'none'; media-src 'self' data:";

    location / {
        proxy_pass http://lemmy;
        proxy_set_header Host $host;
        # include proxy_params;
    }

    location ~ ^/(api|pictrs|feeds|nodeinfo|.well-known) {
        proxy_pass "http://lemmy";
        # proxy common stuff
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection "upgrade";

        # Send actual client IP upstream
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header Host $host;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    }
}