How do I get peer's IP when using Nginx reverse proxy?

Hey, currently all peers using my reverse proxy appear as I’m setting up fail2ban jails for certain behaviours, but this totally prevents me from accurately finding their IP. I’m using Nginx as a reverse proxy, and here’s my config:

map $http_upgrade $connection_upgrade {
	default upgrade;
	'' close;

upstream websocket {

map $remote_addr $proxy_forwarded_elem {
	# IPv4 addresses can be sent as-is
	~^[0-9.]+$          "for=$remote_addr";

	# IPv6 addresses need to be bracketed and quoted
	~^[0-9A-Fa-f:.]+$   "for=\"[$remote_addr]\"";

	# Unix domain socket names cannot be represented in RFC 7239 syntax
	default             "for=unknown";

map $http_forwarded $proxy_add_forwarded {
	# If the incoming Forwarded header is syntactically valid, append to it
	"~^(,[ \\t]*)*([!#$%&'*+.^_`|~0-9A-Za-z-]+=([!#$%&'*+.^_`|~0-9A-Za-z-]+|\"([\\t \\x21\\x23-\\x5B\\x5D-\\x7E\\x80-\\xFF]|\\\\[\\t \\x21-\\x7E\\x80-\\xFF])*\"))?(;([!#$%&'*+.^_`|~0-9A-Za-z-]+=([!#$%&'*+.^_`|~0-9A-Za-z-]+|\"([\\t \\x21\\x23-\\x5B\\x5D-\\x7E\\x80-\\xFF]|\\\\[\\t \\x21-\\x7E\\x80-\\xFF])*\"))?)*([ \\t]*,([ \\t]*([!#$%&'*+.^_`|~0-9A-Za-z-]+=([!#$%&'*+.^_`|~0-9A-Za-z-]+|\"([\\t \\x21\\x23-\\x5B\\x5D-\\x7E\\x80-\\xFF]|\\\\[\\t \\x21-\\x7E\\x80-\\xFF])*\"))?(;([!#$%&'*+.^_`|~0-9A-Za-z-]+=([!#$%&'*+.^_`|~0-9A-Za-z-]+|\"([\\t \\x21\\x23-\\x5B\\x5D-\\x7E\\x80-\\xFF]|\\\\[\\t \\x21-\\x7E\\x80-\\xFF])*\"))?)*)?)*$" "$http_forwarded, $proxy_forwarded_elem";

	# Otherwise, replace it
	default "$proxy_forwarded_elem";

server {
	listen 4430 ssl;
	ssl_certificate /etc/letsencrypt/live/; # managed by Certbot
	ssl_certificate_key /etc/letsencrypt/live/; # managed by Certbot
	include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
	ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
	location / {
		proxy_set_header Forwarded $proxy_add_forwarded;

		proxy_set_header X-Real-IP $remote_addr;
		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

		proxy_pass http://websocket;
		proxy_http_version 1.1;
		proxy_set_header Upgrade $http_upgrade;
		proxy_set_header Connection $connection_upgrade;
		proxy_set_header Host $host;

Does go-ipfs look at these headers at all? Is there a better way?