NGINX Can't download large file (4.5GB)

Hello all,

I'm trying to download a large file from my webserver, it's a backup of all the site files. The file size is around 4.5GB but when I try to download it either via the browser or using wget it returns as an error 404 file not found despite the file clearly being there.

All permissions and ownerships are correct, the file is stored as a tar.gz file and when I save smaller files of the same extension there is never any problems with download.

Any help would be greatly appreciated. Thanks, Paul

nginx -V:

nginx version: nginx/1.4.1
TLS SNI support enabled
configure arguments: --prefix=/etc/nginx --sbin-path=/usr/sbin/nginx --conf-path=/etc/nginx/nginx.conf --error-log-path=/var/log/nginx/error.log --http-log-path=/var/log/nginx/access.log --pid-path=/var/run/nginx.pid --lock-path=/var/run/nginx.lock --http-client-body-temp-path=/var/cache/nginx/client_temp --http-proxy-temp-path=/var/cache/nginx/proxy_temp --http-fastcgi-temp-path=/var/cache/nginx/fastcgi_temp --http-uwsgi-temp-path=/var/cache/nginx/uwsgi_temp --http-scgi-temp-path=/var/cache/nginx/scgi_temp --user=nginx --group=nginx --with-http_ssl_module --with-http_realip_module --with-http_addition_module --with-http_sub_module --with-http_dav_module --with-http_flv_module --with-http_mp4_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_random_index_module --with-http_secure_link_module --with-http_stub_status_module --with-mail --with-mail_ssl_module --with-file-aio --with-ipv6

My NGINX conf:

user  nginx;
worker_processes  8;

error_log  /var/log/nginx/error.log warn;
pid        /var/run/nginx.pid;

events {
    worker_connections  1024;

    multi_accept on;
    use epoll;
}

http {

    # Let NGINX get the real client IP for its access logs - added from http://calendar.perfplanet.com/2012/using-nginx-php-fpmapc-and-varnish-to-make-wordpress-websites-fly/ - 28th Dec 2012
    set_real_ip_from 127.0.0.1;
set_real_ip_from 127.0.0.1/32;

# list of incapsula's IP addresses
set_real_ip_from 199.83.128.0/21;
set_real_ip_from 198.143.32.0/19;
set_real_ip_from 149.126.72.0/21;
set_real_ip_from 103.28.248.0/22;
set_real_ip_from 46.51.174.78;
set_real_ip_from 122.248.247.129;
set_real_ip_from 50.16.241.95;
set_real_ip_from 50.16.241.176;
set_real_ip_from 184.169.135.107;
set_real_ip_from 184.169.135.111;
set_real_ip_from 46.137.108.0;
set_real_ip_from 46.51.168.190;
set_real_ip_from 79.125.110.134;
set_real_ip_from 79.125.118.62;
set_real_ip_from 185.11.124.0/22;
    real_ip_header X-Forwarded-For;

# Set Cache and Timeout for SSL Certificates
ssl_session_cache    shared:SSL:60m;
ssl_session_timeout  60m;

    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;

    sendfile        on;
    #tcp_nopush     on;

    fastcgi_buffers 8 16k;
    fastcgi_buffer_size 32k;

    # added from http://calendar.perfplanet.com/2012/using-nginx-php-fpmapc-and-varnish-to-make-wordpress-websites-fly/ - 28th Dec 2012
    tcp_nopush     on;
    client_max_body_size 5G;
    client_body_timeout 120;
    client_header_timeout 120;
    client_body_buffer_size  128K;
    client_header_buffer_size 1k;
    large_client_header_buffers 4 8k;
    send_timeout 60;
    reset_timedout_connection on;
    types_hash_max_size 2048;
    server_tokens off;
    gzip on;
    gzip_static on;
    gzip_min_length 512;

    keepalive_timeout  60;

    gzip_http_version 1.1;
    gzip_vary on;
    gzip_comp_level 6;
    gzip_proxied any;
    gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript application/javascript text/x-js;
    gzip_buffers 16 8k;
    gzip_disable "MSIE [1-6]\.(?!.*SV1)";

    include /etc/nginx/conf.d/*.conf;

    port_in_redirect off;
}

virtual host file (edited out none-useful stuff):

    server_tokens off;
    autoindex off;
    client_max_body_size 15m;
    client_body_buffer_size 128k;
    expires 86400s;
    add_header Pragma public;
    add_header Cache-Control "max-age=86400, public, must-revalidate, proxy-revalidate";
    error_page 403 = 404;
    location ~ /(\.|wp-config.php|readme.html|licence.txt) {
        return 404;
    }

    rewrite /wp-admin$ $scheme://$host$uri/ permanent;

    # Rewrite for versioned CSS+JS via filemtime
    location ~* ^.+\.(css|js)$ {
        rewrite ^(.+)\.(\d+)\.(css|js)$ $1.$3 last;
        expires 31536000s;
        access_log off;
        log_not_found off;
        add_header Pragma public;
        add_header Cache-Control "max-age=31536000, public";
    }

    location ~* \.(asf|asx|wax|wmv|wmx|avi|bmp|class|divx|doc|docx|eot|exe|gif|gz|gzip|ico|jpg|jpeg|jpe|mdb|mid|midi|mov|qt|mp3|m4a|mp4|m4v|mpeg|mpg|mpe|mpp|odb|odc|odf|odg|odp|ods|odt|ogg|ogv|otf|pdf|png|pot|pps|ppt|pptx|ra|ram|svg|svgz|swf|tar|t?gz|tif|tiff|ttf|wav|webm|wma|woff|wri|xla|xls|xlsx|xlt|xlw|zip)$ {
        expires 360d;
        access_log off;
        log_not_found off;
        add_header Pragma public;
        add_header Cache-Control "max-age=31536000, public";
    }

    location ~* (^(?!(?:(?!(php|inc)).)*/uploads/).*?(php)) {
        try_files $uri = 404;
        fastcgi_split_path_info ^(.+.php)(.*)$;
        fastcgi_pass unix:/dev/shm/php-fpm-www.sock;
        fastcgi_index index.php;
        fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
        include fastcgi_params;
        fastcgi_intercept_errors on;
        fastcgi_ignore_client_abort off;
        fastcgi_connect_timeout 120;
        fastcgi_send_timeout 180;
        fastcgi_read_timeout 180;
        fastcgi_buffer_size 128k;
        fastcgi_buffers 4 256k;
        fastcgi_busy_buffers_size 256k;
        fastcgi_temp_file_write_size 256k;
    }

    proxy_read_timeout 120;

    include conf.d/drop;

        location / {
                # This is cool because no php is touched for static content
            try_files $uri $uri/ /index.php?q=$uri&$args;   
    }

# BEGIN Better WP Security
    set $susquery 0;
    set $rule_2 0;
    set $rule_3 0;
    if ($request_method ~* "^(TRACE|DELETE|TRACK)"){ return 403; }
    location /wp-comments-post.php {
        valid_referers jetpack.wordpress.com/jetpack-comment/ *.livefreedietravelling.com;
        set $rule_0 0;
        if ($request_method ~ "POST"){ set $rule_0 1$rule_0; }
        if ($invalid_referer) { set $rule_0 2$rule_0; }
        if ($http_user_agent ~ "^$"){ set $rule_0 3$rule_0; }
        if ($rule_0 = "3210") { return 403; }
    }    if ($http_cookie !~* "wordpress_logged_in_" ) {
        set $susquery 2$susquery;
        set $rule_2 1;
        set $rule_3 1;
    }
    if ($args !~ "^loggedout=true") { set $susquery 3$susquery; }
# END Better WP Security

}

16 Replies

What do your nginx error logs say?

Absolutely nothing!! That's the weird thing…

Have you tried using FTP to transfer the file?

Could you test with some more verbose debug logging? http://wiki.nginx.org/CoreModule#error_log

Then see what the logs say.

Another possibility would be to strace the nginx pids and see which files are being accessed that it can't find / other errors.

Could you create a similar file (4.5 GB of /dev/urandom, perhaps) and post a URL that can reproduce the problem? (Bonus points if it can be reproduced with a smaller file.)

Hello chaps, thanks for the responses.

@hoopycat, I've created a number of different sized randomly generated files that you can try and download yourself:

@Main Street James, I can download the file via FTP but this is not a suitable solution for my situation. Ideally I would need WGET or browser download as eventually I'd need visitors to download large files from the front-end.

I changed the error logging level to notice and got the following back (I modified error_log /var/log/nginx/error.log warn; to display notices), I'm not sure if there is a deeper level of error reporting that is relevant to this issue? If so, how would I go about doing it?:

2013/05/19 08:40:11 [notice] 10452#0: *15 "/wp-admin$" does not match "/testfile5gb.tar.gz", client: 198.143.34.33, server: livefreedietravelling.com, request: "GET /testfile5gb.tar.gz HTTP/1.1", host: "www.livefreedietravelling.com"
2013/05/19 08:40:11 [notice] 10452#0: *15 "^(TRACE|DELETE|TRACK)" does not match "GET", client: 198.143.34.33, server: livefreedietravelling.com, request: "GET /testfile5gb.tar.gz HTTP/1.1", host: "www.livefreedietravelling.com"
2013/05/19 08:40:11 [notice] 10452#0: *15 "wordpress_logged_in_" does not match "", client: 198.143.34.33, server: livefreedietravelling.com, request: "GET /testfile5gb.tar.gz HTTP/1.1", host: "www.livefreedietravelling.com"
2013/05/19 08:40:11 [notice] 10452#0: *15 "^loggedout=true" does not match "", client: 198.143.34.33, server: livefreedietravelling.com, request: "GET /testfile5gb.tar.gz HTTP/1.1", host: "www.livefreedietravelling.com"

It seems like when I'm logged in I can download the file. Maybe it has something to do with the Better WP Security rules at the bottom of my nginx .conf file? I'm lacking some knowledge on NGINX rewrite rules and even Better WP Security itself so any help here would also be greatly appreciated… It just seems odd that only large files are affected, yet smaller files of the same extension can be downloaded without a problem.

# BEGIN Better WP Security
    set $susquery 0;
    set $rule_2 0;
    set $rule_3 0;
    if ($request_method ~* "^(TRACE|DELETE|TRACK)"){ return 403; }
    location /wp-comments-post.php {
        valid_referers jetpack.wordpress.com/jetpack-comment/ *.livefreedietravelling.com;
        set $rule_0 0;
        if ($request_method ~ "POST"){ set $rule_0 1$rule_0; }
        if ($invalid_referer) { set $rule_0 2$rule_0; }
        if ($http_user_agent ~ "^$"){ set $rule_0 3$rule_0; }
        if ($rule_0 = "3210") { return 403; }
    }    if ($http_cookie !~* "wordpress_logged_in_" ) {
        set $susquery 2$susquery;
        set $rule_2 1;
        set $rule_3 1;
    }
    if ($args !~ "^loggedout=true") { set $susquery 3$susquery; }
# END Better WP Security

Instead of notice, you could set it to debug. It'll generate A LOT of lines so only do it temporarily while testing the request.

You are using Varnish? Perhaps the problem lies there, as that is the server that responds when I request the file:

telnet www.livefreedietravelling.com 80
Trying 149.126.77.251...
Connected to pnnjr.x.incapdns.net.
Escape character is '^]'.
GET /testfile5gb.tar.gz HTTP/1.1
host: www.livefreedietravelling.com

HTTP/1.1 404 Not Found
Server: Varnish
Accept-Ranges: bytes
Date: Sun, 19 May 2013 08:50:34 GMT
Age: 1
Connection: close
X-Cache: MISS
X-Sending-out-good-intentions: This website will make $5000/month soon!
Set-Cookie: incap_ses_108_35107=KLood6QWkHJrXqNM67F/AVmSmFEAAAAAkjfoEmrxp4Qk3aB7XkD2jA==; path=/; Domain=.livefreedietravelling.com
Set-Cookie: visid_incap_35107=aoZDYw4YTxG2nbD1O6GCWlmSmFEAAAAAAAAAAAAAAACqgOugIdesHDKSDQPXovhU; expires=Mon, 18 May 2015 17:00:12 GMT; path=/; Domain=.livefreedietravelling.com
X-Iinfo: 15-206242534-206242879 NNNN CT(12 -1 0) RT(1368953430982 536) q(6 6 7 -1) r(18 18) U5
X-CDN: Incapsula

Same here…

[email protected]:/tmp$ curl -v -o /dev/null http://www.livefreedietravelling.com/testfile512mb.tar.gz
* About to connect() to www.livefreedietravelling.com port 80 (#0)
*   Trying 199.83.132.251... connected
* Connected to www.livefreedietravelling.com (199.83.132.251) port 80 (#0)
> GET /testfile512mb.tar.gz HTTP/1.1
> User-Agent: curl/7.19.7 (x86_64-pc-linux-gnu) libcurl/7.19.7 OpenSSL/0.9.8k zlib/1.2.3.3 libidn/1.15
> Host: www.livefreedietravelling.com
> Accept: */*
> 
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
  0     0    0     0    0     0      0      0 --:--:--  0:00:01 --:--:--     0< HTTP/1.1 404 Not Found
< Server: Varnish
< Accept-Ranges: bytes
< Date: Sun, 19 May 2013 12:01:49 GMT
< Age: 1
< Connection: close
< X-Cache: MISS
< X-Sending-out-good-intentions: This website will make $5000/month soon!
< Set-Cookie: incap_ses_144_35107=y0fnOwzr1ihO05vUsKn/AS2/mFEAAAAABHtTZ/j3AO3hMafXmgrLAQ==; path=/; Domain=.livefreedietravelling.com
< Set-Cookie: ___utmvmoFuFwZB=luaIwfKpiCV; path=/; Max-Age=900
< Set-Cookie: ___utmvaoFuFwZB=AbwXNlB; path=/; Max-Age=900
< Set-Cookie: ___utmvboFuFwZB=QZm
<     XJPORalm: ltU; path=/; Max-Age=900
< Set-Cookie: visid_incap_35107=JYuhmw1ATVa2Mc37WqAlOi2/mFEAAAAAAAAAAAAAAACL2QptoQ+a8xyOzyxTYC+D; expires=Tue, 19 May 2015 02:50:11 GMT; path=/; Domain=.livefreedietravelling.com
< X-Iinfo: 15-4713978-4713979 NNNY CT(76 -1 0) RT(1368964908595 0) q(0 0 1 -1) r(13 13) U5
< X-CDN: Incapsula
< 
{ [data not shown]
  0     0    0     0    0     0      0      0 --:--:--  0:00:01 --:--:--     0* Closing connection #0

Given that 199.83.132.251 is NOT a Linode IP address, I'd recommend reducing the number of variables before further testing.

Thanks alot for your input guys, I think it was a problem with my Varnish configuration. I've since added the following code snippets to my default.vcl file and I think it's solved the issue:

sub vcl_recv {

if (req.url ~ "\.(tar|gz|zip|7z|exe)$") {
return (pipe);
}

.....

}

sub vcl_pipe {
set bereq.http.connection = "close";
return (pipe);
}

I'm currently working from Nepal and as such my connection is beyond poor, I would be VERY grateful if someone might be kind enough to download the largest test file from my server on a high speed connection to confirm it downloads in full and complete? Also, as a side request if willing volunteers could also report on the download speed that would be great!

Again, many thanks to all!

Paul

I am downloading this file now. So far so good, will confirm when completed

EDIT: Downloaded 2.2GB, after that showed as completed. Something still wrong me thinks.

Thanks for that Nuvini… it was looking so promising! Damn!

Ok, well it's not a Varnish issue anymore, at least we're getting connection. I did just restart the varnish server with a couple more tweaks so maybe that broke the piped connection? Could I be a pain and ask for you to download it again? - This time I promise I won't touch a thing….!!

The second download went fine:

[email protected]:~$ wget --verbose http://www.livefreedietravelling.com/testfile5gb.tar.gz -O /dev/null
--2013-05-19 14:53:27--  http://www.livefreedietravelling.com/testfile5gb.tar.gz
Resolving www.livefreedietravelling.com (www.livefreedietravelling.com)... 149.126.74.251
Connecting to www.livefreedietravelling.com (www.livefreedietravelling.com)|149.126.74.251|:80... connected.
HTTP request sent, awaiting response... 200 OK
Length: 5368709120 (5.0G) [application/octet-stream]
Saving to: `/dev/null'

100%[================================================================================================================================================================================================>] 5,368,709,120 26.1M/s   in 3m 26s  

2013-05-19 14:56:54 (24.8 MB/s) - `/dev/null' saved [5368709120/5368709120]

Might've been a restart of varnish that caused it.

I'm trying it one more time from my local connection (which is a tad bit slower than Linode's) - but if you give me 8 minutes I'll be able to tell you if it succeeded as well :)

EDIT: Connection cut off at 2.00GB from home :(

Thanks Nuvini,

Wget went ok for me too but unable to test using local connection. I'll see what the debug logs show and post results, going to have a power cut soon for 5-6 hours so might not be able to get back on until tomorrow. Thanks for your help today,

Paul

@livefree:

Thanks Nuvini,

Wget went ok for me too but unable to test using local connection. I'll see what the debug logs show and post results, going to have a power cut soon for 5-6 hours so might not be able to get back on until tomorrow. Thanks for your help today,

Paul

I'll test it a few more times - Currently doing a wget at home, which at currently 4GB downloaded seems to be going okay as well. Will do it another time through the browser after that, but so far it has only failed when downloading it through a browser. Using Chrome here.

EDIT: So far, all my tests have worked through wget, but failed through Chrome. The location where it fails varies, normally around ~2-2.5GB, but one of them also failed at 700MB.

I tested via wget in Atlanta, it worked fine for me :)

Thanks all,

I resolved the issue by using vcl_pipe() in my Varnish default.vcl.

ref: ~~[https://www.varnish-cache.org/trac/wiki/VCLExampleDefault" target="_blank">](https://www.varnish-cache.org/trac/wiki … pleDefault">https://www.varnish-cache.org/trac/wiki/VCLExampleDefault](

Thanks for everyones help,

Paul :-)

Reply

Please enter an answer
Tips:

You can mention users to notify them: @username

You can use Markdown to format your question. For more examples see the Markdown Cheatsheet.

> I’m a blockquote.

I’m a blockquote.

[I'm a link] (https://www.google.com)

I'm a link

**I am bold** I am bold

*I am italicized* I am italicized

Community Code of Conduct