数据测试 记一次 MySQL5 初始化被 kill 的问题排查 | 京东云技术团队

京东云开发者 · 2023年10月17日 · 2917 次阅读

写在前面

由于测试环境 JED 申请比较繁琐,所以 Eone 提供了单机版 Mysql 供用户使用,近期 Eone 搭建 Mysql5 的时候发现莫名被 kill 了,容器规格是 4C8G,磁盘 30G

这不科学,之前都是可以的,镜像没变,配置没变,咋就不行了呢,一定不是我的问题,是机器的问题

问题排查

重现

通过多次搭建 mysql5 进行采样,发现并不是稳定复现,有一些容器是可以正常启动提供服务的,找到被 mysql 服务被 kill 的容器日志,发现是 MySQL 初始化被 kill 了,

/usr/local/mysql/bin/mysqld --defaults-file=/etc/my.cnf --basedir=/usr/local/mysql --datadir=/export/data/mysql/data --user=admin --initialize-insecure

mysql 配置文件:

[client]
socket=/tmp/mysql.sock

[mysqld]
basedir=/usr/local/mysql
datadir=/export/data/mysql/data
port=3306
server_id=1
socket=/tmp/mysql.sock
log-error=/tmp/mysqld.err
pid-file=/tmp/mysqld.pid

skip-host-cache
skip-name-resolve
skip-grant-tables


问题排查

初始化为什么被 kill?

手动执行了下初始化的命令,发现直接被 kill 了

通过dmesg命令发现,貌似 oom 了

纳尼?怎么用了这么多内存?容器都没这么大内存诶

通过排查,发现 MySQL 有一个 8G 的匿名内存,这是哪来的???

将容器规格调整为 32G 内存,再次尝试,发现成功启动了,但是内存占用貌似不太正常

这不科学,正常的容器内存使用情况是这样的👇

MySQL 配置不合理?

难道是因为 mysql 设置的大小不合理?默认不设置innodb_buffer_pool_size按理说应该是128M才对,抱着侥幸心理设置下试下

然而问题依旧,改了配置也没有效果,看来不是配置的问题

真是机器的问题?

对比了一下正常启动的机器,拿着 ip 找到运维,发现异常的机器是近期添加的云舰系统的机器,之前centos系统宿主机是正常的,新加的云舰系统宿主机都是异常的,看来真是机器的问题,这两台机器有啥差别呢,接下来运维同学开启了排查之路:

1. 首先,看看两台宿主机是不是物理 CPU 总数不一样

2. 再确认下 CentOS 是否开启了富容器功能

对比了下两台宿主机的资源配置,发现好像是资源配置的问题

正常的机器配置是这样的

或许真的有可能,修改下open files限制再试下

ulimit -n 1048576

竟然可以了!!!MySQL 正常提供服务了,真的是机器的问题

问题分析

strace log 分析

28139 execve("/usr/local/mysql/bin/mysqld-debug", ["/usr/local/mysql/bin/mysqld-debu"..., "--initialize-insecure", "--basedir=/usr/local/mysql", "--datadir=/export/data/mysql/dat"..., "--user=admin"], 0x7ffe74bdcbe8 /* 294 vars */) = 0
28139 brk(NULL)                         = 0x4b50000
28139 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f93d3bf6000
28139 access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
28139 open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
28139 fstat(3, {st_mode=S_IFREG|0644, st_size=16580, ...}) = 0
28139 mmap(NULL, 16580, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f93d3bf1000
28139 close(3)                          = 0
28139 open("/lib64/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
28139 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240l\0\0\0\0\0\0"..., 832) = 832
28139 fstat(3, {st_mode=S_IFREG|0755, st_size=142304, ...}) = 0
28139 mmap(NULL, 2208864, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f93d37ba000
28139 mprotect(0x7f93d37d0000, 2097152, PROT_NONE) = 0
28139 mmap(0x7f93d39d0000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0x7f93d39d0000
28139 mmap(0x7f93d39d2000, 13408, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f93d39d2000
28139 close(3)                          = 0
28139 open("/lib64/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
28139 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\16\0\0\0\0\0\0"..., 832) = 832
28139 fstat(3, {st_mode=S_IFREG|0755, st_size=19520, ...}) = 0
28139 mmap(NULL, 2109744, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f93d35b6000
28139 mprotect(0x7f93d35b9000, 2093056, PROT_NONE) = 0
28139 mmap(0x7f93d37b8000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f93d37b8000
28139 close(3)                          = 0
28139 open("/lib64/libaio.so.1", O_RDONLY|O_CLOEXEC) = 3
28139 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240\5\0\0\0\0\0\0"..., 832) = 832
28139 fstat(3, {st_mode=S_IFREG|0755, st_size=6264, ...}) = 0
28139 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f93d3bf0000
28139 mmap(NULL, 2101328, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f93d33b4000
28139 mprotect(0x7f93d33b5000, 2093056, PROT_NONE) = 0
28139 mmap(0x7f93d35b4000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0x7f93d35b4000
28139 close(3)                          = 0
28139 open("/lib64/libnuma.so.1", O_RDONLY|O_CLOEXEC) = 3
28139 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0004\0\0\0\0\0\0"..., 832) = 832
28139 fstat(3, {st_mode=S_IFREG|0755, st_size=50704, ...}) = 0
28139 mmap(NULL, 2144448, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f93d31a8000
28139 mprotect(0x7f93d31b2000, 2097152, PROT_NONE) = 0
28139 mmap(0x7f93d33b2000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xa000) = 0x7f93d33b2000
28139 close(3)                          = 0
28139 open("/lib64/libcrypt.so.1", O_RDONLY|O_CLOEXEC) = 3
28139 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300\16\0\0\0\0\0\0"..., 832) = 832
28139 fstat(3, {st_mode=S_IFREG|0755, st_size=40816, ...}) = 0
28139 mmap(NULL, 2318912, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f93d2f71000
28139 mprotect(0x7f93d2f79000, 2093056, PROT_NONE) = 0
28139 mmap(0x7f93d3178000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x7f93d3178000
28139 mmap(0x7f93d317a000, 184896, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f93d317a000
28139 close(3)                          = 0
28139 open("/lib64/librt.so.1", O_RDONLY|O_CLOEXEC) = 3
28139 read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300\"\0\0\0\0\0\0"..., 832) = 832
28139 fstat(3, {st_mode=S_IFREG|0755, st_size=44096, ...}) = 0
28139 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f93d3bef000
28139 mmap(NULL, 2128952, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f93d2d69000
28139 mprotect(0x7f93d2d70000, 2093056, PROT_NONE) = 0
28139 mmap(0x7f93d2f6f000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7f93d2f6f000
28139 close(3)                          = 0
28139 open("/lib64/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = 3
28139 read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340\264\5\0\0\0\0\0"..., 832) = 832
28139 fstat(3, {st_mode=S_IFREG|0755, st_size=995840, ...}) = 0
28139 mmap(NULL, 3175456, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f93d2a61000
28139 mprotect(0x7f93d2b4a000, 2097152, PROT_NONE) = 0
28139 mmap(0x7f93d2d4a000, 40960, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xe9000) = 0x7f93d2d4a000
28139 mmap(0x7f93d2d54000, 82976, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f93d2d54000
28139 close(3)                          = 0
28139 open("/lib64/libm.so.6", O_RDONLY|O_CLOEXEC) = 3
28139 read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260T\0\0\0\0\0\0"..., 832) = 832
28139 fstat(3, {st_mode=S_IFREG|0755, st_size=1141560, ...}) = 0
28139 mmap(NULL, 3150168, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f93d275f000
28139 mprotect(0x7f93d2860000, 2093056, PROT_NONE) = 0
28139 mmap(0x7f93d2a5f000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x100000) = 0x7f93d2a5f000
28139 close(3)                          = 0
28139 open("/lib64/libgcc_s.so.1", O_RDONLY|O_CLOEXEC) = 3
28139 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320*\0\0\0\0\0\0"..., 832) = 832
28139 fstat(3, {st_mode=S_IFREG|0755, st_size=88720, ...}) = 0
28139 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f93d3bee000
28139 mmap(NULL, 2184192, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f93d2549000
28139 mprotect(0x7f93d255e000, 2093056, PROT_NONE) = 0
28139 mmap(0x7f93d275d000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14000) = 0x7f93d275d000
28139 close(3)                          = 0
28139 open("/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
28139 read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0 \34\2\0\0\0\0\0"..., 832) = 832
28139 fstat(3, {st_mode=S_IFREG|0755, st_size=2112384, ...}) = 0
28139 mmap(NULL, 3936832, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f93d2187000
28139 mprotect(0x7f93d233e000, 2097152, PROT_NONE) = 0
28139 mmap(0x7f93d253e000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1b7000) = 0x7f93d253e000
28139 mmap(0x7f93d2544000, 16960, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f93d2544000
28139 close(3)                          = 0
28139 open("/lib64/libfreebl3.so", O_RDONLY|O_CLOEXEC) = 3
28139 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240\v\0\0\0\0\0\0"..., 832) = 832
28139 fstat(3, {st_mode=S_IFREG|0755, st_size=11376, ...}) = 0
28139 mmap(NULL, 2105536, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f93d1f84000
28139 mprotect(0x7f93d1f86000, 2093056, PROT_NONE) = 0
28139 mmap(0x7f93d2185000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1000) = 0x7f93d2185000
28139 close(3)                          = 0
28139 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f93d3bed000
28139 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f93d3bec000
28139 mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f93d3bea000
28139 arch_prctl(ARCH_SET_FS, 0x7f93d3bea740) = 0
28139 mprotect(0x7f93d253e000, 16384, PROT_READ) = 0
28139 mprotect(0x7f93d37b8000, 4096, PROT_READ) = 0
28139 mprotect(0x7f93d2185000, 4096, PROT_READ) = 0
28139 mprotect(0x7f93d275d000, 4096, PROT_READ) = 0
28139 mprotect(0x7f93d2a5f000, 4096, PROT_READ) = 0
28139 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f93d3be9000
28139 mprotect(0x7f93d2d4a000, 32768, PROT_READ) = 0
28139 mprotect(0x7f93d39d0000, 4096, PROT_READ) = 0
28139 mprotect(0x7f93d2f6f000, 4096, PROT_READ) = 0
28139 mprotect(0x7f93d3178000, 4096, PROT_READ) = 0
28139 mprotect(0x7f93d33b2000, 4096, PROT_READ) = 0
28139 mprotect(0x7f93d35b4000, 4096, PROT_READ) = 0
28139 mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f93d3be7000
28139 mprotect(0x7f93d3bf7000, 4096, PROT_READ) = 0
28139 munmap(0x7f93d3bf1000, 16580)     = 0
28139 set_tid_address(0x7f93d3beaa10)   = 28139
28139 set_robust_list(0x7f93d3beaa20, 24) = 0
28139 rt_sigaction(SIGRTMIN, {sa_handler=0x7f93d37c0780, sa_mask=[], sa_flags=SA_RESTORER|SA_SIGINFO, sa_restorer=0x7f93d37c9100}, NULL, 8) = 0
28139 rt_sigaction(SIGRT_1, {sa_handler=0x7f93d37c0810, sa_mask=[], sa_flags=SA_RESTORER|SA_RESTART|SA_SIGINFO, sa_restorer=0x7f93d37c9100}, NULL, 8) = 0
28139 rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
28139 getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
28139 brk(NULL)                         = 0x4b50000
28139 brk(0x4b71000)                    = 0x4b71000
28139 open("/proc/self/status", O_RDONLY) = 3
28139 fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
28139 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f93d3bf5000
28139 read(3, "Name:\tmysqld-debug\nUmask:\t0022\nS"..., 1024) = 1024
28139 read(3, "0,00000000,00000000,00000000,000"..., 1024) = 334
28139 close(3)                          = 0
28139 munmap(0x7f93d3bf5000, 4096)      = 0
28139 openat(AT_FDCWD, "/sys/devices/system/node", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 3
28139 getdents(3, /* 11 entries */, 32768) = 360
28139 open("/sys/devices/system/node/node0/meminfo", O_RDONLY) = 4
28139 fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
28139 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f93d3bf5000
28139 read(4, "Node 0 MemTotal:       226242196"..., 4096) = 1175
28139 stat("/etc/sysconfig/64bit_strstr_via_64bit_strstr_sse2_unaligned", 0x7ffda2f609d0) = -1 ENOENT (No such file or directory)
28139 stat("/etc/sysconfig/64bit_strstr_via_64bit_strstr_sse2_unaligned", 0x7ffda2f609d0) = -1 ENOENT (No such file or directory)
28139 read(4, "", 4096)                 = 0
28139 close(4)                          = 0
28139 munmap(0x7f93d3bf5000, 4096)      = 0
28139 getdents(3, /* 0 entries */, 32768) = 0
28139 close(3)                          = 0
28139 sched_getaffinity(0, 512, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59]) = 8
28139 openat(AT_FDCWD, "/sys/devices/system/cpu", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 3
28139 getdents(3, /* 77 entries */, 32768) = 2368
28139 getdents(3, /* 0 entries */, 32768) = 0
28139 close(3)                          = 0
28139 open("/proc/self/status", O_RDONLY) = 3
28139 fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
28139 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f93d3bf5000
28139 read(3, "Name:\tmysqld-debug\nUmask:\t0022\nS"..., 1024) = 1024
28139 read(3, "0,00000000,00000000,00000000,000"..., 1024) = 334
28139 read(3, "", 1024)                 = 0
28139 close(3)                          = 0
28139 munmap(0x7f93d3bf5000, 4096)      = 0
28139 futex(0x7f93d2d6696c, FUTEX_WAKE_PRIVATE, 2147483647) = 0
28139 futex(0x7f93d2d66978, FUTEX_WAKE_PRIVATE, 2147483647) = 0
28139 stat("/etc/my.cnf", {st_mode=S_IFREG|0644, st_size=370, ...}) = 0
28139 open("/etc/my.cnf", O_RDONLY)     = 3
28139 fstat(3, {st_mode=S_IFREG|0644, st_size=370, ...}) = 0
28139 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f93d3bf5000
28139 read(3, "[client]\nsocket=/tmp/mysql.sock\n"..., 4096) = 370
28139 read(3, "", 4096)                 = 0
28139 close(3)                          = 0
28139 munmap(0x7f93d3bf5000, 4096)      = 0
28139 stat("/etc/mysql/my.cnf", 0x7ffda2f5c8d0) = -1 ENOENT (No such file or directory)
28139 stat("/usr/local/mysql/etc/my.cnf", 0x7ffda2f5c8d0) = -1 ENOENT (No such file or directory)
28139 stat("/root/.my.cnf", 0x7ffda2f5c8d0) = -1 ENOENT (No such file or directory)
28139 getrlimit(RLIMIT_NOFILE, {rlim_cur=1073741816, rlim_max=1073741816}) = 0
28139 mmap(NULL, 17179869184, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8fd1f84000
28139 +++ killed by SIGKILL +++


可以看到最后用 mmap 一次分配了 16G 内存,然后就被杀了。

mmap 前调用了 getrlimit , 猜测是 mysql 会根据系统资源限制来分配内存

MySQL 源码分析

MySQL 源码中直接调用 getrlimit 的地方不多,排除了 ndb、innodb_memcached、libevent 之后,只有一处直接调用:

static uint set_max_open_files(uint max_file_limit)
{
  struct rlimit rlimit;
  uint old_cur;
  DBUG_ENTER("set_max_open_files");
  DBUG_PRINT("enter",("files: %u", max_file_limit));

  if (!getrlimit(RLIMIT_NOFILE,&rlimit))
  {
    old_cur= (uint) rlimit.rlim_cur;
    DBUG_PRINT("info", ("rlim_cur: %u  rlim_max: %u",
                        (uint) rlimit.rlim_cur,
                        (uint) rlimit.rlim_max));
    if (rlimit.rlim_cur == (rlim_t) RLIM_INFINITY)
      rlimit.rlim_cur = max_file_limit;
    if (rlimit.rlim_cur >= max_file_limit)
      DBUG_RETURN(rlimit.rlim_cur);                /* purecov: inspected */
    rlimit.rlim_cur= rlimit.rlim_max= max_file_limit;
    if (setrlimit(RLIMIT_NOFILE, &rlimit))
      max_file_limit= old_cur;                        /* Use original value */
    else
    {
      rlimit.rlim_cur= 0;                        /* Safety if next call fails */
      (void) getrlimit(RLIMIT_NOFILE,&rlimit);
      DBUG_PRINT("info", ("rlim_cur: %u", (uint) rlimit.rlim_cur));
      if (rlimit.rlim_cur)                        /* If call didn't fail */
        max_file_limit= (uint) rlimit.rlim_cur;
    }
  }
  DBUG_PRINT("exit",("max_file_limit: %u", max_file_limit));
  DBUG_RETURN(max_file_limit);


其中逻辑是:如果系统的文件打开限制是 RLIM_INFINITY 或者比要设置的 max_file_limit 大,都返回系统的限制。

这个函数也只被直接调用一次:

uint my_set_max_open_files(uint files)
{
  struct st_my_file_info *tmp;
  DBUG_ENTER("my_set_max_open_files");
  DBUG_PRINT("enter",("files: %u  my_file_limit: %u", files, my_file_limit));

  files+= MY_FILE_MIN;
  files= set_max_open_files(MY_MIN(files, OS_FILE_LIMIT)); // 获取最大打开文件数
  if (files <= MY_NFILE)
    DBUG_RETURN(files);

  // 分配内存
  if (!(tmp= (struct st_my_file_info*) my_malloc(key_memory_my_file_info,
                                                 sizeof(*tmp) * files,
                                                 MYF(MY_WME))))
    DBUG_RETURN(MY_NFILE);

  // 初始化
  /* Copy any initialized files */
  memcpy((char*) tmp, (char*) my_file_info,
         sizeof(*tmp) * MY_MIN(my_file_limit, files));
  memset((tmp + my_file_limit), 0,
        MY_MAX((int) (files - my_file_limit), 0) * sizeof(*tmp));
  my_free_open_file_info();                        /* Free if already allocated */
  my_file_info= tmp;
  my_file_limit= files;
  DBUG_PRINT("exit",("files: %u", files));
  DBUG_RETURN(files);
}


原来 MySQL5 会根据最大可打开文件数,提前为每个文件分配和初始化内存,在这个时候就可能分配过多内存,导致 OOM。MySQL8 修复了这个问题。

解决

启动前设置 ulimit

ENTRYPOINT ["ulimit -n 1048576 && /home/admin/start.sh"]

写在后面

在 linux 中,open files设置过大会出现很多问题,应该合理控制open files数量

参考文献

https://learnku.com/articles/34598

作者:京东零售 杨云龙

来源:京东云开发者社区 转载请注明来源

暂无回复。
需要 登录 后方可回复, 如果你还没有账号请点击这里 注册