Nginx + php-fpm woes

Hi everyone!

I recently switched from Apache/mod_php to Nginx/php-fpm, and ever since then, I've been having a weird problem…

Customers have complained that they'll click on a link to a .php file on my site, the page will turn white (as it does normally when a page is loading), but then nothing happens. This happens very rarely, but enough where a few people have mentioned it.

I'm not sure if it's related, but I've also seen a lot of 499 errors in my access logs. I've copied and pasted a sample below, but there are a LOT more where that came from!

I'm using Ubuntu 13.10.

Here are all of my config files and logs.

If you have any idea what the problem might be, I would very much appreciate your help! I don't want to have to switch back to Apache/mod_php, but if I can't find a solution, I will have to :(

/etc/nginx/nginx.conf

user www-data;
worker_processes 8;
pid /run/nginx.pid;

events {
    worker_connections 768;
    # multi_accept on;
}

http {

    ##
    # Basic Settings
    ##

    sendfile on;
    tcp_nopush on;
    tcp_nodelay on;
    keepalive_timeout 65;
    types_hash_max_size 2048;
    # server_tokens off;

    # server_names_hash_bucket_size 64;
    # server_name_in_redirect off;

    include /etc/nginx/mime.types;
    default_type application/octet-stream;

    ##
    # Logging Settings
    ##

    access_log /var/log/nginx/access.log;
    error_log /var/log/nginx/error.log;

    ##
    # Gzip Settings
    ##

    gzip on;
    gzip_disable "msie6";

    # gzip_vary on;
    # gzip_proxied any;
    # gzip_comp_level 6;
    # gzip_buffers 16 8k;
    # gzip_http_version 1.1;
    # gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;

    ##
    # nginx-naxsi config
    ##
    # Uncomment it if you installed nginx-naxsi
    ##

    #include /etc/nginx/naxsi_core.rules;

    ##
    # nginx-passenger config
    ##
    # Uncomment it if you installed nginx-passenger
    ##

    #passenger_root /usr;
    #passenger_ruby /usr/bin/ruby;

    ##
    # Virtual Host Configs
    ##

    include /etc/nginx/conf.d/*.conf;
    include /etc/nginx/sites-enabled/*;
}

#mail {
#    # See sample authentication script at:
#    # http://wiki.nginx.org/ImapAuthenticateWithApachePhpScript
# 
#    # auth_http localhost/auth.php;
#    # pop3_capabilities "TOP" "USER";
#    # imap_capabilities "IMAP4rev1" "UIDPLUS";
# 
#    server {
#        listen     localhost:110;
#        protocol   pop3;
#        proxy      on;
#    }
# 
#    server {
#        listen     localhost:143;
#        protocol   imap;
#        proxy      on;
#    }
#}

worker_rlimit_nofile 1000000;

/etc/nginx/sites-available/nginx_config

#This prevents nginx from sending its version number and the host operating system in the "Server" HTTP header
server_tokens off;

server {
    listen 80;
    listen [::]:80;
    #The "ssl" is required for HTTPS
    listen 443 ssl;
    listen [::]:443 ssl;

    server_name .REDACTED.com;

    #SSL
    #http://nginx.org/en/docs/http/configuring_https_servers.html
    ssl_certificate REDACTED;
    ssl_certificate_key REDACTED;
    ssl_prefer_server_ciphers on;
    ssl_session_cache shared:SSL:10m;
    ssl_session_timeout 10m;

    root /var/www/;
    index index.html index.php index.htm =404;
    autoindex off;
    log_not_found off;

    #return 503;

    #Socket.IO
    location ~* /socket.io {
        proxy_pass http://127.0.0.1:8080;
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection "upgrade";
        proxy_set_header Host $host;
    }

    #PHP Files
    location ~* \.php {
        try_files $uri =404;
        fastcgi_pass unix:/var/run/php5-fpm.sock;
        #fastcgi_pass 127.0.0.1:9000;
        include fastcgi_params;
        fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
        fastcgi_read_timeout 600s;
    }

    #CSS/Javascript/Image Files
    location ~* \.(css|js|jpeg|jpg|png|gif|ico) {
        expires 1y;
    }

    keepalive_timeout 300;

    include /etc/nginx/nginx_rewrite_rules;
}

/etc/php5/fpm/php.ini

http://pastebin.com/Xat1JQTQ

/etc/php5/fpm/pool.d/www.conf

; Start a new pool named 'www'.
; the variable $pool can we used in any directive and will be replaced by the
; pool name ('www' here)
[www]

; Per pool prefix
; It only applies on the following directives:
; - 'slowlog'
; - 'listen' (unixsocket)
; - 'chroot'
; - 'chdir'
; - 'php_values'
; - 'php_admin_values'
; When not set, the global prefix (or /usr) applies instead.
; Note: This directive can also be relative to the global prefix.
; Default Value: none
;prefix = /path/to/pools/$pool

; Unix user/group of processes
; Note: The user is mandatory. If the group is not set, the default user's group
;       will be used.
user = www-data
group = www-data

; The address on which to accept FastCGI requests.
; Valid syntaxes are:
;   'ip.add.re.ss:port'    - to listen on a TCP socket to a specific address on
;                            a specific port;
;   'port'                 - to listen on a TCP socket to all addresses on a
;                            specific port;
;   '/path/to/unix/socket' - to listen on a unix socket.
; Note: This value is mandatory.
listen = /var/run/php5-fpm.sock
;listen = 9000

; Set listen(2) backlog.
; Default Value: 128 (-1 on FreeBSD and OpenBSD)
listen.backlog = 128

; Set permissions for unix socket, if one is used. In Linux, read/write
; permissions must be set in order to allow connections from a web server. Many
; BSD-derived systems allow connections regardless of permissions. 
; Default Values: user and group are set as the running user
;                 mode is set to 0666
;listen.owner = www-data
;listen.group = www-data
;listen.mode = 0666

; List of ipv4 addresses of FastCGI clients which are allowed to connect.
; Equivalent to the FCGI_WEB_SERVER_ADDRS environment variable in the original
; PHP FCGI (5.2.2+). Makes sense only with a tcp listening socket. Each address
; must be separated by a comma. If this value is left blank, connections will be
; accepted from any ip address.
; Default Value: any
;listen.allowed_clients = 127.0.0.1

; Specify the nice(2) priority to apply to the pool processes (only if set)
; The value can vary from -19 (highest priority) to 20 (lower priority)
; Note: - It will only work if the FPM master process is launched as root
;       - The pool processes will inherit the master process priority
;         unless it specified otherwise
; Default Value: no set
; priority = -19

; Choose how the process manager will control the number of child processes.
; Possible Values:
;   static  - a fixed number (pm.max_children) of child processes;
;   dynamic - the number of child processes are set dynamically based on the
;             following directives. With this process management, there will be
;             always at least 1 children.
;             pm.max_children      - the maximum number of children that can
;                                    be alive at the same time.
;             pm.start_servers     - the number of children created on startup.
;             pm.min_spare_servers - the minimum number of children in 'idle'
;                                    state (waiting to process). If the number
;                                    of 'idle' processes is less than this
;                                    number then some children will be created.
;             pm.max_spare_servers - the maximum number of children in 'idle'
;                                    state (waiting to process). If the number
;                                    of 'idle' processes is greater than this
;                                    number then some children will be killed.
;  ondemand - no children are created at startup. Children will be forked when
;             new requests will connect. The following parameter are used:
;             pm.max_children           - the maximum number of children that
;                                         can be alive at the same time.
;             pm.process_idle_timeout   - The number of seconds after which
;                                         an idle process will be killed.
; Note: This value is mandatory.
pm = dynamic

; The number of child processes to be created when pm is set to 'static' and the
; maximum number of child processes when pm is set to 'dynamic' or 'ondemand'.
; This value sets the limit on the number of simultaneous requests that will be
; served. Equivalent to the ApacheMaxClients directive with mpm_prefork.
; Equivalent to the PHP_FCGI_CHILDREN environment variable in the original PHP
; CGI. The below defaults are based on a server without much resources. Don't
; forget to tweak pm.* to fit your needs.
; Note: Used when pm is set to 'static', 'dynamic' or 'ondemand'
; Note: This value is mandatory.
pm.max_children = 6144

; The number of child processes created on startup.
; Note: Used only when pm is set to 'dynamic'
; Default Value: min_spare_servers + (max_spare_servers - min_spare_servers) / 2
pm.start_servers = 10

; The desired minimum number of idle server processes.
; Note: Used only when pm is set to 'dynamic'
; Note: Mandatory when pm is set to 'dynamic'
pm.min_spare_servers = 5

; The desired maximum number of idle server processes.
; Note: Used only when pm is set to 'dynamic'
; Note: Mandatory when pm is set to 'dynamic'
pm.max_spare_servers = 10

; The number of seconds after which an idle process will be killed.
; Note: Used only when pm is set to 'ondemand'
; Default Value: 10s
;pm.process_idle_timeout = 10s;

; The number of requests each child process should execute before respawning.
; This can be useful to work around memory leaks in 3rd party libraries. For
; endless request processing specify '0'. Equivalent to PHP_FCGI_MAX_REQUESTS.
; Default Value: 0
pm.max_requests = 100

; The URI to view the FPM status page. If this value is not set, no URI will be
; recognized as a status page. It shows the following informations:
;   pool                 - the name of the pool;
;   process manager      - static, dynamic or ondemand;
;   start time           - the date and time FPM has started;
;   start since          - number of seconds since FPM has started;
;   accepted conn        - the number of request accepted by the pool;
;   listen queue         - the number of request in the queue of pending
;                          connections (see backlog in listen(2));
;   max listen queue     - the maximum number of requests in the queue
;                          of pending connections since FPM has started;
;   listen queue len     - the size of the socket queue of pending connections;
;   idle processes       - the number of idle processes;
;   active processes     - the number of active processes;
;   total processes      - the number of idle + active processes;
;   max active processes - the maximum number of active processes since FPM
;                          has started;
;   max children reached - number of times, the process limit has been reached,
;                          when pm tries to start more children (works only for
;                          pm 'dynamic' and 'ondemand');
; Value are updated in real time.
; Example output:
;   pool:                 www
;   process manager:      static
;   start time:           01/Jul/2011:17:53:49 +0200
;   start since:          62636
;   accepted conn:        190460
;   listen queue:         0
;   max listen queue:     1
;   listen queue len:     42
;   idle processes:       4
;   active processes:     11
;   total processes:      15
;   max active processes: 12
;   max children reached: 0
;
; By default the status page output is formatted as text/plain. Passing either
; 'html', 'xml' or 'json' in the query string will return the corresponding
; output syntax. Example:
;   http://www.foo.bar/status
;   http://www.foo.bar/status?json
;   http://www.foo.bar/status?html
;   http://www.foo.bar/status?xml
;
; By default the status page only outputs short status. Passing 'full' in the
; query string will also return status for each pool process.
; Example: 
;   http://www.foo.bar/status?full
;   http://www.foo.bar/status?json&full
;   http://www.foo.bar/status?html&full
;   http://www.foo.bar/status?xml&full
; The Full status returns for each process:
;   pid                  - the PID of the process;
;   state                - the state of the process (Idle, Running, ...);
;   start time           - the date and time the process has started;
;   start since          - the number of seconds since the process has started;
;   requests             - the number of requests the process has served;
;   request duration     - the duration in µs of the requests;
;   request method       - the request method (GET, POST, ...);
;   request URI          - the request URI with the query string;
;   content length       - the content length of the request (only with POST);
;   user                 - the user (PHP_AUTH_USER) (or '-' if not set);
;   script               - the main script called (or '-' if not set);
;   last request cpu     - the %cpu the last request consumed
;                          it's always 0 if the process is not in Idle state
;                          because CPU calculation is done when the request
;                          processing has terminated;
;   last request memory  - the max amount of memory the last request consumed
;                          it's always 0 if the process is not in Idle state
;                          because memory calculation is done when the request
;                          processing has terminated;
; If the process is in Idle state, then informations are related to the
; last request the process has served. Otherwise informations are related to
; the current request being served.
; Example output:
;   ************************
;   pid:                  31330
;   state:                Running
;   start time:           01/Jul/2011:17:53:49 +0200
;   start since:          63087
;   requests:             12808
;   request duration:     1250261
;   request method:       GET
;   request URI:          /test_mem.php?N=10000
;   content length:       0
;   user:                 -
;   script:               /home/fat/web/docs/php/test_mem.php
;   last request cpu:     0.00
;   last request memory:  0
;
; Note: There is a real-time FPM status monitoring sample web page available
;       It's available in: ${prefix}/share/fpm/status.html
;
; Note: The value must start with a leading slash (/). The value can be
;       anything, but it may not be a good idea to use the .php extension or it
;       may conflict with a real PHP file.
; Default Value: not set 
;pm.status_path = /status

; The ping URI to call the monitoring page of FPM. If this value is not set, no
; URI will be recognized as a ping page. This could be used to test from outside
; that FPM is alive and responding, or to
; - create a graph of FPM availability (rrd or such);
; - remove a server from a group if it is not responding (load balancing);
; - trigger alerts for the operating team (24/7).
; Note: The value must start with a leading slash (/). The value can be
;       anything, but it may not be a good idea to use the .php extension or it
;       may conflict with a real PHP file.
; Default Value: not set
;ping.path = /ping

; This directive may be used to customize the response of a ping request. The
; response is formatted as text/plain with a 200 response code.
; Default Value: pong
;ping.response = pong

; The access log file
; Default: not set
;access.log = log/$pool.access.log

; The access log format.
; The following syntax is allowed
;  %%: the '%' character
;  %C: %CPU used by the request
;      it can accept the following format:
;      - %{user}C for user CPU only
;      - %{system}C for system CPU only
;      - %{total}C  for user + system CPU (default)
;  %d: time taken to serve the request
;      it can accept the following format:
;      - %{seconds}d (default)
;      - %{miliseconds}d
;      - %{mili}d
;      - %{microseconds}d
;      - %{micro}d
;  %e: an environment variable (same as $_ENV or $_SERVER)
;      it must be associated with embraces to specify the name of the env
;      variable. Some exemples:
;      - server specifics like: %{REQUEST_METHOD}e or %{SERVER_PROTOCOL}e
;      - HTTP headers like: %{HTTP_HOST}e or %{HTTP_USER_AGENT}e
;  %f: script filename
;  %l: content-length of the request (for POST request only)
;  %m: request method
;  %M: peak of memory allocated by PHP
;      it can accept the following format:
;      - %{bytes}M (default)
;      - %{kilobytes}M
;      - %{kilo}M
;      - %{megabytes}M
;      - %{mega}M
;  %n: pool name
;  %o: output header
;      it must be associated with embraces to specify the name of the header:
;      - %{Content-Type}o
;      - %{X-Powered-By}o
;      - %{Transfert-Encoding}o
;      - ....
;  %p: PID of the child that serviced the request
;  %P: PID of the parent of the child that serviced the request
;  %q: the query string 
;  %Q: the '?' character if query string exists
;  %r: the request URI (without the query string, see %q and %Q)
;  %R: remote IP address
;  %s: status (response code)
;  %t: server time the request was received
;      it can accept a strftime(3) format:
;      %d/%b/%Y:%H:%M:%S %z (default)
;  %T: time the log has been written (the request has finished)
;      it can accept a strftime(3) format:
;      %d/%b/%Y:%H:%M:%S %z (default)
;  %u: remote user
;
; Default: "%R - %u %t \"%m %r\" %s"
;access.format = "%R - %u %t \"%m %r%Q%q\" %s %f %{mili}d %{kilo}M %C%%"

; The log file for slow requests
; Default Value: not set
; Note: slowlog is mandatory if request_slowlog_timeout is set
;slowlog = log/$pool.log.slow

; The timeout for serving a single request after which a PHP backtrace will be
; dumped to the 'slowlog' file. A value of '0s' means 'off'.
; Available units: s(econds)(default), m(inutes), h(ours), or d(ays)
; Default Value: 0
;request_slowlog_timeout = 0

; The timeout for serving a single request after which the worker process will
; be killed. This option should be used when the 'max_execution_time' ini option
; does not stop script execution for some reason. A value of '0' means 'off'.
; Available units: s(econds)(default), m(inutes), h(ours), or d(ays)
; Default Value: 0
;request_terminate_timeout = 0

; Set open file descriptor rlimit.
; Default Value: system defined value
;rlimit_files = 1024

; Set max core size rlimit.
; Possible Values: 'unlimited' or an integer greater or equal to 0
; Default Value: system defined value
;rlimit_core = 0

; Chroot to this directory at the start. This value must be defined as an
; absolute path. When this value is not set, chroot is not used.
; Note: you can prefix with '$prefix' to chroot to the pool prefix or one
; of its subdirectories. If the pool prefix is not set, the global prefix
; will be used instead.
; Note: chrooting is a great security feature and should be used whenever 
;       possible. However, all PHP paths will be relative to the chroot
;       (error_log, sessions.save_path, ...).
; Default Value: not set
;chroot = 

; Chdir to this directory at the start.
; Note: relative path can be used.
; Default Value: current directory or / when chroot
chdir = /

; Redirect worker stdout and stderr into main error log. If not set, stdout and
; stderr will be redirected to /dev/null according to FastCGI specs.
; Note: on highloaded environement, this can cause some delay in the page
; process time (several ms).
; Default Value: no
;catch_workers_output = yes

; Limits the extensions of the main script FPM will allow to parse. This can
; prevent configuration mistakes on the web server side. You should only limit
; FPM to .php extensions to prevent malicious users to use other extensions to
; exectute php code.
; Note: set an empty value to allow all extensions.
; Default Value: .php
;security.limit_extensions = .php .php3 .php4 .php5

; Pass environment variables like LD_LIBRARY_PATH. All $VARIABLEs are taken from
; the current environment.
; Default Value: clean env
;env[HOSTNAME] = $HOSTNAME
;env[PATH] = /usr/local/bin:/usr/bin:/bin
;env[TMP] = /tmp
;env[TMPDIR] = /tmp
;env[TEMP] = /tmp

; Additional php.ini defines, specific to this pool of workers. These settings
; overwrite the values previously defined in the php.ini. The directives are the
; same as the PHP SAPI:
;   php_value/php_flag             - you can set classic ini defines which can
;                                    be overwritten from PHP call 'ini_set'. 
;   php_admin_value/php_admin_flag - these directives won't be overwritten by
;                                     PHP call 'ini_set'
; For php_*flag, valid values are on, off, 1, 0, true, false, yes or no.

; Defining 'extension' will load the corresponding shared extension from
; extension_dir. Defining 'disable_functions' or 'disable_classes' will not
; overwrite previously defined php.ini values, but will append the new value
; instead.

; Note: path INI options can be relative and will be expanded with the prefix
; (pool, global or /usr)

; Default Value: nothing is defined by default except the values in php.ini and
;                specified at startup with the -d argument
;php_admin_value[sendmail_path] = /usr/sbin/sendmail -t -i -f [email protected]
;php_flag[display_errors] = off
;php_admin_value[error_log] = /var/log/fpm-php.www.log
;php_admin_flag[log_errors] = on
;php_admin_value[memory_limit] = 32M

499 Errors

REDACTED - - [14/Nov/2013:08:43:40 -0800] "POST /wits/API.php HTTP/1.1" 200 33 "https://www.rundowncreator.com/wits/Script.php?RundownID=13&RowID=270" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36"
REDACTED - - [14/Nov/2013:08:43:41 -0800] "POST /wits/API.php HTTP/1.1" 200 32 "https://www.rundowncreator.com/wits/Script.php?RundownID=13&RowID=270" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36"
REDACTED - - [14/Nov/2013:08:43:42 -0800] "POST /wits/API.php HTTP/1.1" 200 35 "https://www.rundowncreator.com/wits/Script.php?RundownID=13&RowID=270" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36"
REDACTED - - [14/Nov/2013:08:43:43 -0800] "POST /wits/API.php HTTP/1.1" 200 32 "https://www.rundowncreator.com/wits/Script.php?RundownID=13&RowID=270" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36"
REDACTED - - [14/Nov/2013:08:43:53 -0800] "POST /wits/API.php HTTP/1.1" 200 32 "https://www.rundowncreator.com/wits/Script.php?RundownID=13&RowID=270" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36"
REDACTED - - [14/Nov/2013:08:43:53 -0800] "POST /wits/API.php HTTP/1.1" 200 35 "https://www.rundowncreator.com/wits/Script.php?RundownID=13&RowID=270" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36"
REDACTED - - [14/Nov/2013:08:43:54 -0800] "POST /wits/API.php HTTP/1.1" 200 47 "https://www.rundowncreator.com/wits/Script.php?RundownID=13&RowID=270" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36"
REDACTED - - [14/Nov/2013:08:43:54 -0800] "POST /wits/API.php HTTP/1.1" 499 0 "https://www.rundowncreator.com/wits/Script.php?RundownID=13&RowID=270" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36"
REDACTED - - [14/Nov/2013:08:43:54 -0800] "POST /wits/API.php HTTP/1.1" 499 0 "https://www.rundowncreator.com/wits/Script.php?RundownID=13&RowID=270" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36"
REDACTED - - [14/Nov/2013:08:43:54 -0800] "POST /wits/API.php HTTP/1.1" 499 0 "https://www.rundowncreator.com/wits/Script.php?RundownID=13&RowID=270" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36"
REDACTED - - [14/Nov/2013:08:43:54 -0800] "POST /wits/API.php HTTP/1.1" 200 194 "https://www.rundowncreator.com/wits/Rundown.php?RundownID=13" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36"
REDACTED - - [14/Nov/2013:08:44:02 -0800] "GET /wits/Script.php?RundownID=13&RowID=273 HTTP/1.1" 200 3804 "https://www.rundowncreator.com/wits/Rundown.php?RundownID=13" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36"
REDACTED - - [14/Nov/2013:08:44:02 -0800] "GET /socket.io/socket.io.js HTTP/1.1" 200 74747 "https://www.rundowncreator.com/wits/Script.php?RundownID=13&RowID=273" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36"

/var/log/php5-fpm.log has nothing

/var/log/phperror.log has nothing

14 Replies

499 is nginx's way of saying the browser closed the connection before nginx could finish the request, an example would be the user closing their browser.

A white page in PHP normally means a fatal php error. You note your php log is empty, have you checked php can write to it's error log and error logging is enabled?

Yes, the PHP error log is writable. It's not actually empty. There is stuff in there. However, they're all non-relevant app errors (user tried to do x, etc).

I should also note the problem seems to occur during periods of heavy load.

Anything in /var/log/syslog or /var/log/messages relating to 'out of memory' it could be you're running out of memory and the kernel is killing PHP which would also cause a white page.

Wow, this is really weird but /var/log/syslog is empty. I mean it literally this time too! There's nothing in it. I checked syslog.1, and the newest message was from October 26, the day the server was setup.

[email protected]:/var/log$ ls -lah

total 588K
drwxr-xr-x 15 root       root     4.0K Nov 15 19:24 .
drwxr-xr-x 15 root       root     4.0K Oct 25 21:40 ..
-rw-r--r--  1 root       root        0 Nov 14 09:21 alternatives.log
-rw-r--r--  1 root       root      171 Nov  9 19:58 alternatives.log.1
-rw-r--r--  1 root       root     1.9K Oct 25 21:46 alternatives.log.2.gz
drwxr-xr-x  2 root       root     4.0K Nov 14 09:21 apt
-rw-r--r--  1 root       root        0 Apr 26  2013 aptitude
-rw-r-----  1 messagebus adm         0 Apr 26  2013 auth.log
-rw-r-----  1 root       adm         0 Apr 26  2013 boot
-rw-r--r--  1 root       root        0 Apr 26  2013 boot.log
-rw-r--r--  1 root       root        0 Apr 26  2013 bootstrap.log
-rw-rw----  1 root       utmp        0 Nov 15 19:24 btmp
-rw-rw----  1 root       utmp        0 Nov 14 09:21 btmp.1
drwxr-xr-x  2 root       root     4.0K Apr 25  2013 ConsoleKit
drwxr-xr-x  3 root       root     4.0K Oct 25 21:32 dist-upgrade
-rw-r-----  1 root       adm       12K Nov  9 19:59 dmesg
-rw-r-----  1 root       adm       12K Oct 25 22:23 dmesg.0
-rw-r-----  1 root       adm      4.6K Oct 25 22:14 dmesg.1.gz
-rw-r-----  1 root       adm      4.6K Oct 25 22:07 dmesg.2.gz
-rw-r-----  1 root       adm      4.6K Oct 25 21:58 dmesg.3.gz
-rw-r-----  1 root       adm      4.7K Oct 25 21:44 dmesg.4.gz
-rw-r--r--  1 root       root        0 Nov 14 09:21 dpkg.log
-rw-r--r--  1 root       root      11K Nov  9 19:58 dpkg.log.1
-rw-r--r--  1 root       root      27K Oct 25 21:46 dpkg.log.2.gz
-rw-r--r--  1 root       root      32K Oct 25 21:39 faillog
drwxr-xr-x  2 root       root     4.0K Apr 25  2013 fsck
drwxr-xr-x  3 root       root     4.0K Apr 25  2013 installer
-rw-r-----  1 messagebus adm         0 Apr 26  2013 kern.log
drwxr-xr-x  2 landscape  root     4.0K Apr 25  2013 landscape
-rw-rw-r--  1 root       utmp     286K Nov 16 06:58 lastlog
-rw-r-----  1 messagebus adm         0 Apr 26  2013 mail.err
-rw-r-----  1 messagebus adm         0 Apr 26  2013 mail.log
drwxr-s---  2 mysql      adm      4.0K Nov 15 19:24 mysql
-rw-r-----  1 mysql      adm         0 Oct 25 21:39 mysql.err
-rw-r-----  1 mysql      adm         0 Nov 15 19:24 mysql.log
-rw-r-----  1 mysql      adm         0 Nov 15 10:25 mysql.log.1
drwxr-xr-x  2 root       root     4.0K Apr 25  2013 news
drwxrwxr-x  2 www-data   www-data 4.0K Oct 25 21:44 nginx
drwxr-xr-x  2 ntp        ntp      4.0K Apr  4  2013 ntpstats
-rw-------  1 root       root        0 Nov 15 19:24 php5-fpm.log
-rw-------  1 root       root      291 Nov 15 11:20 php5-fpm.log.1
-rw-------  1 root       root      111 Nov 10 16:45 php5-fpm.log.2.gz
-rw-------  1 root       root      146 Nov  9 19:59 php5-fpm.log.3.gz
-rw-------  1 root       root      196 Oct 30 13:09 php5-fpm.log.4.gz
-rw-------  1 root       root      160 Oct 25 22:23 php5-fpm.log.5.gz
-rwxrwxr-x  1 www-data   www-data  41K Nov 15 13:57 phperror.log
-rw-r-----  1 messagebus adm         0 Oct 27 10:23 syslog
-rw-r-----  1 messagebus adm       33K Oct 25 21:40 syslog.1
drwxr-xr-x  2 root       root     4.0K May 24  2012 sysstat
-rw-r--r--  1 root       root     279K Nov  9 19:59 udev
-rw-r-----  1 messagebus adm         0 Apr 26  2013 ufw.log
drwxr-xr-x  2 root       root     4.0K Apr 25  2013 unattended-upgrades
drwxr-xr-x  2 root       root     4.0K Nov 15 19:25 upstart
-rw-rw-r--  1 root       utmp     2.3K Nov 16 06:58 wtmp
-rw-rw-r--  1 root       utmp      11K Nov 15 19:24 wtmp.1

As you can see, /var/log/messages doesn't exist.

I am wondering if I should just rebuild this server? Linode did not have the 13.10 server image up when I needed it, so (on their advice), I installed whatever the latest Ubuntu release was that they had and ran do-release-upgrade. I wonder if that has anything to do with this.

Your thoughts?

BTW, I forgot to mention, this is a 2GB Linode.

Well this is off the charts IMO:

pm.max_children = 6144

Any idea of how many php-fpm processes are actually running when you're under heavy load?

Just by casually glancing at the output of top, I've seen maybe a dozen or so.

Yes, pm.max_children is set rather high. It was originally 25 or lower. I was attempting to figure out if that was the bottleneck by setting it to something outrageously high.

Your syslog is owned by messagebus that's just weird. It should be owned by syslog. All files/folders owned by messagebus in /var/log/ you should chown to syslog.

So you were having the php problem when max_children was set to 25?

OK, I just chown'ed everything owned by messagebus. I'm tailing the syslog right now, and can see stuff being written to it again. It's mostly cron jobs, but I'll keep an eye on it to see if it reports anything interesting…

And yeah, I had the PHP problem even with max_children set to 25.

I'm not seeing anything out of the ordinary in the syslog, so I think I might rebuild tonight using a proper Ubuntu 13.10 image and see what happens…

I rebuilt my servers just now, and will wait to see if the problem persists over the next couple of days, then report back.

Thanks for your help today obs!

Unfortunately, rebuilding did not solve the problem.

My Logwatch reports look like this:

http://pastebin.com/t9FwwJvs

I'm experimenting with some things. I'll report back next week…

Update update!

It turns out it wasn't nginx or PHP. It was the REST API I was using to communicate between PHP and Node.js. I replaced it with Redis Pub/Sub and the problem went away.

Thanks for all of your help!

Reply

Please enter an answer
Tips:

You can mention users to notify them: @username

You can use Markdown to format your question. For more examples see the Markdown Cheatsheet.

> I’m a blockquote.

I’m a blockquote.

[I'm a link] (https://www.google.com)

I'm a link

**I am bold** I am bold

*I am italicized* I am italicized

Community Code of Conduct