slow file transfer speeds on nextcloud 19 behind haproxy jail

[blacksteel75]

Hi all,

I’m running nextcloud in a freenas jail with an haproxy reverse proxy in front of it (running in a separate jail).

My problem is that the file transfer speeds are extremely slow when going through the haproxy connection.

• When I connect to nextcloud via the haproxy jail, using the full domain name, I’m seeing ~200KB/s upload speeds max
• When I connect directly to the nextcloud jail via IP address, bypassing the reverse proxy, I’m seeing 5-10MB/s upload speeds. Same server, same everything else.

I am terminating the SSL connection at haproxy, and connecting via IP address to the server also uses SSL.

Based on those side-by-side tests using the same file set, I’m assuming that the problem is either in the freenas jail configuration of the haproxy jail or in the haproxy config itself. Maybe some performance tuning on haproxy?

Here’s my haproxy.conf:

Code:

global
    daemon
    maxconn 500
    tune.h2.initial-window-size 1048576
    tune.ssl.default-dh-param 2048
    ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
    ssl-default-bind-options no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets
    ssl-default-server-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
    ssl-default-server-options no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets
    log 192.168.1.104:9901 len 8096 local0 err
    log-send-hostname
    stats socket /var/run/haproxy.sock mode 600 level admin

defaults
    log global
    option dontlognull
    option forwardfor
    option http-buffer-request
    option log-health-checks
    timeout connect 30s
    timeout client 2h
    timeout server 2h
    timeout check 5s
    timeout http-request 15s
    timeout http-keep-alive 15s

frontend default
    mode tcp
    bind 0.0.0.0:443

    http-request capture req.hdr(Host) len 30
    http-request capture req.hdr(User-Agent) len 200
    http-request capture req.hdr(Referer) len 800

    tcp-request inspect-delay 5s
    tcp-request content accept if { req.ssl_hello_type 1 }

    use_backend http-traffic-redirect if { req.ssl_hello_type 1 }

    default_backend no-match-tcp

frontend main-http
    mode http

    log 192.168.1.104:9901 len 8096 local0 info
    log-format '{"clientIP": "%ci","clientPort": "%cp","dateTime": "%t","frontendNameTransport": "%ft","backend": "%b","serverName": "%s","Tw": %Tw,"Tc": %Tc,"Tt": %Tt,"bytesRead": %B,"terminationState": "%ts","actconn": %ac,"FrontendCurrentConn": %fc,"backendCurrentConn": %bc,"serverConcurrentConn": %sc,"retries": %rc,"srvQueue": %sq,"backendQueue": %bq,"backendSourceIP": "%bi","backendSourcePort": "%bp","statusCode": "%ST","serverIP": "%si","serverPort": "%sp","frontendIP": "%fi","frontendPort": "%fp","httpRequest": "%r","capturedRequestHeaders": "%hr"}'

    capture request header Host len 30
    capture request header User-Agent len 200
    capture request header Referer len 800
    capture request header X-Forwarded-For len 20

    bind 127.0.0.1:443 ssl crt /usr/local/etc/ssl/certs/mydomain.com.combined.pem accept-proxy alpn h2,http/1.1

    http-request deny if HTTP_1.0

    acl plex hdr(host) -i plex.mydomain.com
    acl nextcloud hdr(host) -i nextcloud.mydomain.com

    use_backend plex.mydomain.com if plex
    use_backend nextcloud.mydomain.com if nextcloud

    default_backend no-match-http

backend http-traffic-redirect
    mode tcp
    server haproxy 127.0.0.1:443 maxconn 200 send-proxy

backend plex.mydomain.com
    mode http
    option http-server-close
    http-response set-header Strict-Transport-Security "max-age=63072000; incluudeSubDomains; preload;"
    http-response set-header X-Frame-Options DENY
    http-response set-header X-XSS-Protection 1;mode=block
    http-response set-header X-Content-Type-Options nosniff
    http-response set-header Referrer-Policy no-referrer-when-downgrade
    server plex 192.168.1.101:32400 maxconn 20 check ssl verify none

backend nextcloud.mydomain.com
    mode http
    option http-server-close
    fullconn 100
    timeout connect 30s
    timeout client 2h
    timeout server 2h
    http-request set-header X-Forwarded-Port %[dst_port]
    http-request add-header X-Forwarded-Proto https if { ssl_fc }
    option httpchk HEAD / HTTP/1.1\r\nHost:localhost
    http-response set-header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload;"
    http-response set-header X-Frame-Options SAMEORIGIN
    http-response set-header X-XSS-Protection 1;mode=block
    http-response set-header X-Content-Type-Options nosniff
    http-response set-header Referrer-Policy no-referrer-when-downgrade
    server nextcloud 192.168.1.102:443 maxconn 100 check ssl verify none

backend no-match-http
    mode http
    http-request deny deny_status 503

backend no-match-tcp
    mode tcp
    tcp-request content reject

listen stats
    mode http
    bind :9000
    maxconn 20
    timeout client 30s
    timeout server 30s
    timeout connect 30s
    timeout queue 100s
    stats enable
    stats uri /stats
    stats hide-version
    stats refresh 30s
    stats show-node

The jail configs are using the default settings, with allow_raw_sockets = 1.

I do have a few other backends on this server, running on other jails, but I removed them from the above config as I don’t think they are relevant.

There’s no errors in the logs, so it appears to be working correctly, just very slowly.

Nextcloud 19 is running on nginx 1.18.0_20,2 and php 7.4

haproxy is running v2.2.2

I tested iperf between the haproxy jail IP and the nextcloud jail IP and speeds were over 1Gb/s, which makes me suspect the issue is in haproxy or related.

Thank you for your help!

____________

Version: FreeNAS-11.3-U3.2
Platform: HP Gen 10 Microserver
RAM: 16Gb

 

[blacksteel75]

Solving my own problem:

• I moved the haproxy server to the firewall, so it's no longer in the freenas jail.
  • The performance of the file upload didn't change just by doing this.
• After adding "nbthread 4" to the "global" section of my haproxy.conf file, the throughput on upload improved dramatically, and I am now seeing upload speeds via haproxy at the same speed as directly to the server's IP.