![Apache 上的内存泄漏](https://linux22.com/image/662051/Apache%20%E4%B8%8A%E7%9A%84%E5%86%85%E5%AD%98%E6%B3%84%E6%BC%8F.png)
我出现了轻微的内存泄漏,显然是由于 Apache(配置为反向代理)未释放信号量造成的。当平台加载更多流量时,这种泄漏会随着时间的推移变得更加严重。当我们发出以下命令以及使用硬重启重新启动 Apache 时,可以释放内存。优雅重启不会释放内存。
/usr/bin/ipcrm sem $(/usr/bin/ipcs -s | grep www-data | awk '{print$2}')
如果我们不强制释放信号量,我们就会遇到类似下面帖子中描述的问题。 什么是信号量以及它是如何产生的?
然后 Apache 拒绝重新启动,我们在日志中没有收到任何错误消息。我们已经进行了几次测试,但都未能成功修复该问题。我们用 kernel.sem 值更改了 sysctl.conf 文件
当前值为:
$ cat /proc/sys/kernel/sem
500 64000 64 256
我逐个停用了以下模块(并相应地更改了 Apache 配置文件):
LoadModule security2_module /usr/lib/apache2/modules/mod_security2.so
LoadModule geoip_module /usr/lib/apache2/modules/mod_geoip.so
LoadModule pagespeed_module /usr/lib/apache2/modules/mod_pagespeed_ap24.so
LoadModule evasive20_module /usr/lib/apache2/modules/mod_evasive20.so
LoadModule proxy_protocol_module /usr/lib/apache2/modules/mod_proxy_protocol.so
然后我尝试从 mpm_event 转移到 mpm_worker,但也没有成功。
Apache 的当前版本和模块是:
$ apache2 -V
Server version: Apache/2.4.7 (Ubuntu)
Server built: Mar 10 2015 13:05:59
Server's Module Magic Number: 20120211:27
Server loaded: APR 1.5.1-dev, APR-UTIL 1.5.3
Compiled using: APR 1.5.1-dev, APR-UTIL 1.5.3
Architecture: 64-bit
Server MPM: worker
threaded: yes (fixed thread count)
forked: yes (variable process count)
服务器编译:
-D APR_HAS_SENDFILE
-D APR_HAS_MMAP
-D APR_HAVE_IPV6 (IPv4-mapped addresses enabled)
-D APR_USE_SYSVSEM_SERIALIZE
-D APR_USE_PTHREAD_SERIALIZE
-D SINGLE_LISTEN_UNSERIALIZED_ACCEPT
-D APR_HAS_OTHER_CHILD
-D AP_HAVE_RELIABLE_PIPED_LOGS
-D DYNAMIC_MODULE_LIMIT=256
-D HTTPD_ROOT="/etc/apache2"
-D SUEXEC_BIN="/usr/lib/apache2/suexec"
-D DEFAULT_PIDLOG="/var/run/apache2.pid"
-D DEFAULT_SCOREBOARD="logs/apache_runtime_status"
-D DEFAULT_ERRORLOG="logs/error_log"
-D AP_TYPES_CONFIG_FILE="mime.types"
-D SERVER_CONFIG_FILE="apache2.conf"
Ubuntu 软件包:
apache2 : 2.4.7-1ubuntu4.4
apache2-bin : 2.4.7-1ubuntu4.4
apache2-data : 2.4.7-1ubuntu4.4
apache2-dev : 2.4.7-1ubuntu4.4
libapache2-mod-evasive: 1.10.1-2
libapache2-mod-security2: 2.7.7-2
libapache2-modsecurity: 2.7.7-2
libgeoip-dev : 1.6.0-1
其他编译包
geoip-api-mod_geoip2 : 1.2.9
mod_pagespeed : latest stable version
proxy-protocol (https://github.com/roadrunner2/mod-proxy-protocol): latest version
这是我的 apache2.conf
# GENERIC DECLARATIONS
User www-data
Group www-data
Listen 80
ServerAdmin admin@xxx
ServerName server:80
ServerRoot "/etc/apache2"
# MODULES LOADING
# Generic modulesE
#LoadModule mpm_event_module /usr/lib/apache2/modules/mod_mpm_event.so
LoadModule mpm_worker_module /usr/lib/apache2/modules/mod_mpm_worker.so
LoadModule ratelimit_module /usr/lib/apache2/modules/mod_ratelimit.so
LoadModule reqtimeout_module /usr/lib/apache2/modules/mod_reqtimeout.so
LoadModule mime_module /usr/lib/apache2/modules/mod_mime.so
LoadModule headers_module /usr/lib/apache2/modules/mod_headers.so
LoadModule setenvif_module /usr/lib/apache2/modules/mod_setenvif.so
LoadModule proxy_module /usr/lib/apache2/modules/mod_proxy.so
LoadModule proxy_connect_module /usr/lib/apache2/modules/mod_proxy_connect.so
LoadModule proxy_http_module /usr/lib/apache2/modules/mod_proxy_http.so
LoadModule ssl_module /usr/lib/apache2/modules/mod_ssl.so
LoadModule status_module /usr/lib/apache2/modules/mod_status.so
LoadModule autoindex_module /usr/lib/apache2/modules/mod_autoindex.so
LoadModule dir_module /usr/lib/apache2/modules/mod_dir.so
LoadModule alias_module /usr/lib/apache2/modules/mod_alias.so
LoadModule rewrite_module /usr/lib/apache2/modules/mod_rewrite.so
LoadModule authz_core_module /usr/lib/apache2/modules/mod_authz_core.so
LoadModule authz_host_module /usr/lib/apache2/modules/mod_authz_host.so
LoadModule unique_id_module /usr/lib/apache2/modules/mod_unique_id.so
LoadModule socache_shmcb_module /usr/lib/apache2/modules/mod_socache_shmcb.so
LoadModule substitute_module /usr/lib/apache2/modules/mod_substitute.so
LoadModule filter_module /usr/lib/apache2/modules/mod_filter.so
LoadModule deflate_module /usr/lib/apache2/modules/mod_deflate.so
# Specific modules
LoadModule security2_module /usr/lib/apache2/modules/mod_security2.so
LoadModule geoip_module /usr/lib/apache2/modules/mod_geoip.so
LoadModule pagespeed_module /usr/lib/apache2/modules/mod_pagespeed_ap24.so
#LoadModule evasive20_module /usr/lib/apache2/modules/mod_evasive20.so
LoadModule proxy_protocol_module /usr/lib/apache2/modules/mod_proxy_protocol.so
#### PROXY SETTINGS ####
# LOG FORMATS
LogFormat "%t \"%r\" %>s %I %O [%{GEOIP_COUNTRY_CODE}e] %{UNIQUE_ID}e" o
CustomLog "/var/log/access_log" o
# TIMEOUT SETTINGS
Timeout 60
ProxyTimeout 60
# ERROR LOGS
ErrorLog "/var/log/error_log"
LogLevel error
#### WEB PAGES CONFIGURATION ####
DocumentRoot "/var/www"
<Directory "/var/www">
Options Indexes FollowSymLinks
AllowOverride None
Require all granted
</Directory>
DirectoryIndex index.html
<Files ".ht*">
Require all denied
</Files>
TypesConfig /etc/apache2/mime.types
AddType application/x-compress .Z
AddType application/x-gzip .gz .tgz
#### MODULES CONFIGURATION ####
# Server-pool management (MPM specific)
Include /etc/apache2/httpd-mpm.conf
# Secure (SSL/TLS) connections
Include /etc/apache2/httpd-ssl.conf
#### GEOIP CONF ####
GeoIPEnable On
GeoIPScanProxyHeaders On
GeoIPDBFile /usr/share/GeoIP/GeoIP.dat IndexCache
#### MODSECURITY CONF ####
SecPcreMatchLimit 15000
SecPcreMatchLimitRecursion 15000
SecTmpDir /tmp/
SecDataDir /tmp/
#### MODPAGESPEED CONF ####
ModPagespeed on
ModPagespeedInheritVHostConfig on
AddOutputFilterByType MOD_PAGESPEED_OUTPUT_FILTER text/html
AddOutputFilterByType MOD_PAGESPEED_OUTPUT_FILTER text/html
ModPagespeedXHeaderValue "Powered"
ModPagespeedRewriteLevel OptimizeForBandwidth
ModPagespeedFileCachePath "/var/cache/mod_pagespeed/"
ModPagespeedFileCacheInodeLimit 500000
ModPagespeedStatisticsLogging off
ModPagespeedMessageBufferSize 100000
Include /etc/apache2/pagespeed_libraries.conf
#### DDOS PROTECTION ####
#DOSHashTableSize 3097
#DOSPageCount 2
#DOSPageInterval 1
#DOSSiteCount 150
#DOSSiteInterval 1
#DOSBlockingPeriod 10
#DOSLogDir "/var/lock/mod_evasive"
#### VHOSTS #####
Include /etc/apache2/sites/*.conf
httpd-mpm.conf: Pid文件“/var/log/apache2/httpd.pid”
<IfModule mod_mpm_event.c>
StartServers 3
MinSpareThreads 75
MaxSpareThreads 250
ThreadsPerChild 25
MaxRequestWorkers 400
MaxConnectionsPerChild 10000
</IfModule>
<IfModule mod_mpm_worker.c>
# Default values
ServerLimit 16
MaxMemFree 2048
StartServers 2
MinSpareThreads 75
MaxSpareThreads 250
# Custom values
MaxConnectionsPerChild 10000
ThreadsPerChild 1000
MaxRequestWorkers 16000
</IfModule>
Vhosts 配置示例:
<VirtualHost *:80>
#### SSL FALLBACK ####
ServerName server
ProxyProtocol on
RewriteEngine On
RewriteRule ^(.*)$ https://%{HTTP_HOST}$1 [R=301,L]
######################
</VirtualHost>
<VirtualHost *:443>
#### GENERIC CONFIGURATION ####
ServerName server
ProxyProtocol on
ProxyPreserveHost On
RewriteEngine On
RewriteRule "^/errors/(.*)" "https://server/v1/errors/$1" [R]
ProxyPass / https://server/ keepalive=On
ProxyPassReverse / https://server/
###############################
#### COMPRESSION ####
ModPagespeedFetchHttps enable
DeflateFilterNote Input instream
DeflateFilterNote Output outstream
DeflateFilterNote Ratio ratio
#####################
#### LOG FORMATS ####
LogFormat "[3] [%{Host}i] %t \"%r\" %>s [response_time %D] [request_size %I] [response_size_before_compression %O] [response_size_after_compression %{outstream}n] [country \"%{GEOIP_COUNTRY_CODE}e\"] [uuid \"%{uuid}C\"] [user-agent \"%{User-agent}i\"] [unique_id \"%{UNIQUE_ID}e\"]" O
LogFormat "[3] [%{Host}i] %t [pid %{pid}P:tid %{tid}P] [tag \"GEOBLOCKING\"] [tag \"%{GEOIP_COUNTRY_CODE}e\"] [client %a] [uri \"%U\"] [unique_id \"%{UNIQUE_ID}e\"]" GeoIPblocked
ErrorLog "/var/log/error_log"
CustomLog "/var/log/access_log" O
##########################
#### SSL CONFIGURATION ####
SSLEngine on
SSLCertificateFile "/etc/apache2/sites/3/3.crt"
SSLCertificateKeyFile "/etc/apache2/sites/3/3.key"
###########################
#### DDOS PROTECTION ####
#DOSHashTableSize 3097
#DOSPageCount 10
#DOSSiteCount 150
#DOSPageInterval 1
#DOSSiteInterval 1
#DOSBlockingPeriod 10
#############################
#### MODSECURITY ####
SecRuleEngine on
SecDefaultAction "setenv:unique_id=%{UNIQUE_ID},phase:2,redirect:/errors/block?unique_id=%{unique_id}&site_id=3"
Include /etc/apache2/modsecurity.conf
Include /etc/apache2/sites/3/3-exceptions.conf
Include /etc/apache2/sites/3/3-cms-exceptions.conf
Include /etc/apache2/sites/3/3-vp.conf
##########################
#### SCRIPT INJECTION ####
SetInputFilter DEFLATE
AddOutputFilterByType SUBSTITUTE text/html
Substitute "s\</body>\<script src=\"script_source"></script><script type=\"text/javascript\"> 1kB script </script></body>\in"
#########################################
Modsecurity.conf SecRequestBodyAccess 开启
SecRule REQUEST_HEADERS:Content-Type "text/xml" "id:'200000',phase:1,nolog,allow"
SecRule REQUEST_HEADERS:Content-Type "application/json" "id:'200001',phase:1,nolog,allow"
SecRequestBodyLimit 13107200
SecRequestBodyNoFilesLimit 13107200
SecRequestBodyInMemoryLimit 13107200
SecRequestBodyLimitAction Reject
SecRule REQBODY_ERROR "!@eq 0" \
"id:'200002', phase:2,t:none,log,deny,status:400,msg:'Failed to parse request body.',logdata:'%{reqbody_error_msg}',severity:2"
SecRule MULTIPART_STRICT_ERROR "!@eq 0" \
"id:'200003',phase:2,t:none,log,deny,status:400, \
msg:'Multipart request body failed strict validation: \
PE %{REQBODY_PROCESSOR_ERROR}, \
BQ %{MULTIPART_BOUNDARY_QUOTED}, \
BW %{MULTIPART_BOUNDARY_WHITESPACE}, \
DB %{MULTIPART_DATA_BEFORE}, \
DA %{MULTIPART_DATA_AFTER}, \
HF %{MULTIPART_HEADER_FOLDING}, \
LF %{MULTIPART_LF_LINE}, \
SM %{MULTIPART_MISSING_SEMICOLON}, \
IQ %{MULTIPART_INVALID_QUOTING}, \
IP %{MULTIPART_INVALID_PART}, \
IH %{MULTIPART_INVALID_HEADER_FOLDING}, \
FL %{MULTIPART_FILE_LIMIT_EXCEEDED}'"
SecRule TX:/^MSC_/ "!@streq 0" \
"id:'200005',phase:2,t:none,deny,msg:'ModSecurity internal error flagged: %{MATCHED_VAR_NAME}'"
SecResponseBodyAccess Off
SecArgumentSeparator &
SecCookieFormat 0
SecUnicodeMapFile unicode.mapping 20127
Include /etc/apache2/rules/*.conf
Apache 规则是标准 crs 规则,通过 SecRuleRemoveById 指令设置了大约 50 个例外。
答案1
我们在反向代理配置中大量使用 Apache,并且经常会看到内存泄漏,因为服务器只有 512MB RAM,所以内存泄漏非常明显。
我不知道我们是否有与信号量相关的泄漏或其他问题,但我们通过减少每个子节点的最大连接数指令设置为默认值的四分之一,即 2500。在我们的流量水平下,这意味着子进程每小时回收两次,这解决了问题。没有明显性能影响。