经验首页 前端设计 程序设计 Java相关 移动开发 数据库/运维 软件/图像 大数据/云计算 其他经验
当前位置:技术经验 » 数据库/运维 » MySQL » 查看文章
记一次MySQL5初始化被kill的问题排查
来源:cnblogs  作者:京东云开发者  时间:2023/10/17 11:02:14  对本文有异议

写在前面

由于测试环境JED申请比较繁琐,所以Eone提供了单机版Mysql供用户使用,近期Eone搭建Mysql5的时候发现莫名被kill了,容器规格是4C8G,磁盘30G

这不科学,之前都是可以的,镜像没变,配置没变,咋就不行了呢,一定不是我的问题,是机器的问题

问题排查

重现

通过多次搭建mysql5进行采样,发现并不是稳定复现,有一些容器是可以正常启动提供服务的,找到被mysql服务被kill的容器日志,发现是MySQL初始化被kill了,

  1. /usr/local/mysql/bin/mysqld --defaults-file=/etc/my.cnf --basedir=/usr/local/mysql --datadir=/export/data/mysql/data --user=admin --initialize-insecure

mysql配置文件:

  1. [client]
  2. socket=/tmp/mysql.sock
  3. [mysqld]
  4. basedir=/usr/local/mysql
  5. datadir=/export/data/mysql/data
  6. port=3306
  7. server_id=1
  8. socket=/tmp/mysql.sock
  9. log-error=/tmp/mysqld.err
  10. pid-file=/tmp/mysqld.pid
  11. skip-host-cache
  12. skip-name-resolve
  13. skip-grant-tables

问题排查

初始化为什么被kill?

手动执行了下初始化的命令,发现直接被kill了

通过dmesg命令发现,貌似oom了

纳尼?怎么用了这么多内存?容器都没这么大内存诶

通过排查,发现MySQL有一个8G的匿名内存,这是哪来的???

将容器规格调整为32G内存,再次尝试,发现成功启动了,但是内存占用貌似不太正常

这不科学,正常的容器内存使用情况是这样的??

MySQL配置不合理?

难道是因为mysql设置的大小不合理?默认不设置innodb_buffer_pool_size按理说应该是128M才对,抱着侥幸心理设置下试下

然而问题依旧,改了配置也没有效果,看来不是配置的问题

真是机器的问题?

对比了一下正常启动的机器,拿着ip找到运维,发现异常的机器是近期添加的云舰系统的机器,之前centos系统宿主机是正常的,新加的云舰系统宿主机都是异常的,看来真是机器的问题,这两台机器有啥差别呢,接下来运维同学开启了排查之路:

1. 首先,看看两台宿主机是不是物理CPU总数不一样

2. 再确认下CentOS是否开启了富容器功能

对比了下两台宿主机的资源配置,发现好像是资源配置的问题

正常的机器配置是这样的

或许真的有可能,修改下open files限制再试下

ulimit -n 1048576

竟然可以了!!!MySQL正常提供服务了,真的是机器的问题

问题分析

strace log分析

  1. 28139 execve("/usr/local/mysql/bin/mysqld-debug", ["/usr/local/mysql/bin/mysqld-debu"..., "--initialize-insecure", "--basedir=/usr/local/mysql", "--datadir=/export/data/mysql/dat"..., "--user=admin"], 0x7ffe74bdcbe8 /* 294 vars */) = 0
  2. 28139 brk(NULL) = 0x4b50000
  3. 28139 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f93d3bf6000
  4. 28139 access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
  5. 28139 open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
  6. 28139 fstat(3, {st_mode=S_IFREG|0644, st_size=16580, ...}) = 0
  7. 28139 mmap(NULL, 16580, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f93d3bf1000
  8. 28139 close(3) = 0
  9. 28139 open("/lib64/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
  10. 28139 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240l\0\0\0\0\0\0"..., 832) = 832
  11. 28139 fstat(3, {st_mode=S_IFREG|0755, st_size=142304, ...}) = 0
  12. 28139 mmap(NULL, 2208864, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f93d37ba000
  13. 28139 mprotect(0x7f93d37d0000, 2097152, PROT_NONE) = 0
  14. 28139 mmap(0x7f93d39d0000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0x7f93d39d0000
  15. 28139 mmap(0x7f93d39d2000, 13408, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f93d39d2000
  16. 28139 close(3) = 0
  17. 28139 open("/lib64/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
  18. 28139 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\16\0\0\0\0\0\0"..., 832) = 832
  19. 28139 fstat(3, {st_mode=S_IFREG|0755, st_size=19520, ...}) = 0
  20. 28139 mmap(NULL, 2109744, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f93d35b6000
  21. 28139 mprotect(0x7f93d35b9000, 2093056, PROT_NONE) = 0
  22. 28139 mmap(0x7f93d37b8000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f93d37b8000
  23. 28139 close(3) = 0
  24. 28139 open("/lib64/libaio.so.1", O_RDONLY|O_CLOEXEC) = 3
  25. 28139 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240\5\0\0\0\0\0\0"..., 832) = 832
  26. 28139 fstat(3, {st_mode=S_IFREG|0755, st_size=6264, ...}) = 0
  27. 28139 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f93d3bf0000
  28. 28139 mmap(NULL, 2101328, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f93d33b4000
  29. 28139 mprotect(0x7f93d33b5000, 2093056, PROT_NONE) = 0
  30. 28139 mmap(0x7f93d35b4000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0x7f93d35b4000
  31. 28139 close(3) = 0
  32. 28139 open("/lib64/libnuma.so.1", O_RDONLY|O_CLOEXEC) = 3
  33. 28139 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0004\0\0\0\0\0\0"..., 832) = 832
  34. 28139 fstat(3, {st_mode=S_IFREG|0755, st_size=50704, ...}) = 0
  35. 28139 mmap(NULL, 2144448, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f93d31a8000
  36. 28139 mprotect(0x7f93d31b2000, 2097152, PROT_NONE) = 0
  37. 28139 mmap(0x7f93d33b2000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xa000) = 0x7f93d33b2000
  38. 28139 close(3) = 0
  39. 28139 open("/lib64/libcrypt.so.1", O_RDONLY|O_CLOEXEC) = 3
  40. 28139 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300\16\0\0\0\0\0\0"..., 832) = 832
  41. 28139 fstat(3, {st_mode=S_IFREG|0755, st_size=40816, ...}) = 0
  42. 28139 mmap(NULL, 2318912, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f93d2f71000
  43. 28139 mprotect(0x7f93d2f79000, 2093056, PROT_NONE) = 0
  44. 28139 mmap(0x7f93d3178000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x7f93d3178000
  45. 28139 mmap(0x7f93d317a000, 184896, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f93d317a000
  46. 28139 close(3) = 0
  47. 28139 open("/lib64/librt.so.1", O_RDONLY|O_CLOEXEC) = 3
  48. 28139 read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300\"\0\0\0\0\0\0"..., 832) = 832
  49. 28139 fstat(3, {st_mode=S_IFREG|0755, st_size=44096, ...}) = 0
  50. 28139 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f93d3bef000
  51. 28139 mmap(NULL, 2128952, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f93d2d69000
  52. 28139 mprotect(0x7f93d2d70000, 2093056, PROT_NONE) = 0
  53. 28139 mmap(0x7f93d2f6f000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7f93d2f6f000
  54. 28139 close(3) = 0
  55. 28139 open("/lib64/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = 3
  56. 28139 read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340\264\5\0\0\0\0\0"..., 832) = 832
  57. 28139 fstat(3, {st_mode=S_IFREG|0755, st_size=995840, ...}) = 0
  58. 28139 mmap(NULL, 3175456, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f93d2a61000
  59. 28139 mprotect(0x7f93d2b4a000, 2097152, PROT_NONE) = 0
  60. 28139 mmap(0x7f93d2d4a000, 40960, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xe9000) = 0x7f93d2d4a000
  61. 28139 mmap(0x7f93d2d54000, 82976, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f93d2d54000
  62. 28139 close(3) = 0
  63. 28139 open("/lib64/libm.so.6", O_RDONLY|O_CLOEXEC) = 3
  64. 28139 read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260T\0\0\0\0\0\0"..., 832) = 832
  65. 28139 fstat(3, {st_mode=S_IFREG|0755, st_size=1141560, ...}) = 0
  66. 28139 mmap(NULL, 3150168, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f93d275f000
  67. 28139 mprotect(0x7f93d2860000, 2093056, PROT_NONE) = 0
  68. 28139 mmap(0x7f93d2a5f000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x100000) = 0x7f93d2a5f000
  69. 28139 close(3) = 0
  70. 28139 open("/lib64/libgcc_s.so.1", O_RDONLY|O_CLOEXEC) = 3
  71. 28139 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320*\0\0\0\0\0\0"..., 832) = 832
  72. 28139 fstat(3, {st_mode=S_IFREG|0755, st_size=88720, ...}) = 0
  73. 28139 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f93d3bee000
  74. 28139 mmap(NULL, 2184192, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f93d2549000
  75. 28139 mprotect(0x7f93d255e000, 2093056, PROT_NONE) = 0
  76. 28139 mmap(0x7f93d275d000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14000) = 0x7f93d275d000
  77. 28139 close(3) = 0
  78. 28139 open("/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
  79. 28139 read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0 \34\2\0\0\0\0\0"..., 832) = 832
  80. 28139 fstat(3, {st_mode=S_IFREG|0755, st_size=2112384, ...}) = 0
  81. 28139 mmap(NULL, 3936832, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f93d2187000
  82. 28139 mprotect(0x7f93d233e000, 2097152, PROT_NONE) = 0
  83. 28139 mmap(0x7f93d253e000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1b7000) = 0x7f93d253e000
  84. 28139 mmap(0x7f93d2544000, 16960, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f93d2544000
  85. 28139 close(3) = 0
  86. 28139 open("/lib64/libfreebl3.so", O_RDONLY|O_CLOEXEC) = 3
  87. 28139 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240\v\0\0\0\0\0\0"..., 832) = 832
  88. 28139 fstat(3, {st_mode=S_IFREG|0755, st_size=11376, ...}) = 0
  89. 28139 mmap(NULL, 2105536, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f93d1f84000
  90. 28139 mprotect(0x7f93d1f86000, 2093056, PROT_NONE) = 0
  91. 28139 mmap(0x7f93d2185000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1000) = 0x7f93d2185000
  92. 28139 close(3) = 0
  93. 28139 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f93d3bed000
  94. 28139 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f93d3bec000
  95. 28139 mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f93d3bea000
  96. 28139 arch_prctl(ARCH_SET_FS, 0x7f93d3bea740) = 0
  97. 28139 mprotect(0x7f93d253e000, 16384, PROT_READ) = 0
  98. 28139 mprotect(0x7f93d37b8000, 4096, PROT_READ) = 0
  99. 28139 mprotect(0x7f93d2185000, 4096, PROT_READ) = 0
  100. 28139 mprotect(0x7f93d275d000, 4096, PROT_READ) = 0
  101. 28139 mprotect(0x7f93d2a5f000, 4096, PROT_READ) = 0
  102. 28139 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f93d3be9000
  103. 28139 mprotect(0x7f93d2d4a000, 32768, PROT_READ) = 0
  104. 28139 mprotect(0x7f93d39d0000, 4096, PROT_READ) = 0
  105. 28139 mprotect(0x7f93d2f6f000, 4096, PROT_READ) = 0
  106. 28139 mprotect(0x7f93d3178000, 4096, PROT_READ) = 0
  107. 28139 mprotect(0x7f93d33b2000, 4096, PROT_READ) = 0
  108. 28139 mprotect(0x7f93d35b4000, 4096, PROT_READ) = 0
  109. 28139 mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f93d3be7000
  110. 28139 mprotect(0x7f93d3bf7000, 4096, PROT_READ) = 0
  111. 28139 munmap(0x7f93d3bf1000, 16580) = 0
  112. 28139 set_tid_address(0x7f93d3beaa10) = 28139
  113. 28139 set_robust_list(0x7f93d3beaa20, 24) = 0
  114. 28139 rt_sigaction(SIGRTMIN, {sa_handler=0x7f93d37c0780, sa_mask=[], sa_flags=SA_RESTORER|SA_SIGINFO, sa_restorer=0x7f93d37c9100}, NULL, 8) = 0
  115. 28139 rt_sigaction(SIGRT_1, {sa_handler=0x7f93d37c0810, sa_mask=[], sa_flags=SA_RESTORER|SA_RESTART|SA_SIGINFO, sa_restorer=0x7f93d37c9100}, NULL, 8) = 0
  116. 28139 rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
  117. 28139 getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
  118. 28139 brk(NULL) = 0x4b50000
  119. 28139 brk(0x4b71000) = 0x4b71000
  120. 28139 open("/proc/self/status", O_RDONLY) = 3
  121. 28139 fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
  122. 28139 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f93d3bf5000
  123. 28139 read(3, "Name:\tmysqld-debug\nUmask:\t0022\nS"..., 1024) = 1024
  124. 28139 read(3, "0,00000000,00000000,00000000,000"..., 1024) = 334
  125. 28139 close(3) = 0
  126. 28139 munmap(0x7f93d3bf5000, 4096) = 0
  127. 28139 openat(AT_FDCWD, "/sys/devices/system/node", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 3
  128. 28139 getdents(3, /* 11 entries */, 32768) = 360
  129. 28139 open("/sys/devices/system/node/node0/meminfo", O_RDONLY) = 4
  130. 28139 fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
  131. 28139 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f93d3bf5000
  132. 28139 read(4, "Node 0 MemTotal: 226242196"..., 4096) = 1175
  133. 28139 stat("/etc/sysconfig/64bit_strstr_via_64bit_strstr_sse2_unaligned", 0x7ffda2f609d0) = -1 ENOENT (No such file or directory)
  134. 28139 stat("/etc/sysconfig/64bit_strstr_via_64bit_strstr_sse2_unaligned", 0x7ffda2f609d0) = -1 ENOENT (No such file or directory)
  135. 28139 read(4, "", 4096) = 0
  136. 28139 close(4) = 0
  137. 28139 munmap(0x7f93d3bf5000, 4096) = 0
  138. 28139 getdents(3, /* 0 entries */, 32768) = 0
  139. 28139 close(3) = 0
  140. 28139 sched_getaffinity(0, 512, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59]) = 8
  141. 28139 openat(AT_FDCWD, "/sys/devices/system/cpu", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 3
  142. 28139 getdents(3, /* 77 entries */, 32768) = 2368
  143. 28139 getdents(3, /* 0 entries */, 32768) = 0
  144. 28139 close(3) = 0
  145. 28139 open("/proc/self/status", O_RDONLY) = 3
  146. 28139 fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
  147. 28139 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f93d3bf5000
  148. 28139 read(3, "Name:\tmysqld-debug\nUmask:\t0022\nS"..., 1024) = 1024
  149. 28139 read(3, "0,00000000,00000000,00000000,000"..., 1024) = 334
  150. 28139 read(3, "", 1024) = 0
  151. 28139 close(3) = 0
  152. 28139 munmap(0x7f93d3bf5000, 4096) = 0
  153. 28139 futex(0x7f93d2d6696c, FUTEX_WAKE_PRIVATE, 2147483647) = 0
  154. 28139 futex(0x7f93d2d66978, FUTEX_WAKE_PRIVATE, 2147483647) = 0
  155. 28139 stat("/etc/my.cnf", {st_mode=S_IFREG|0644, st_size=370, ...}) = 0
  156. 28139 open("/etc/my.cnf", O_RDONLY) = 3
  157. 28139 fstat(3, {st_mode=S_IFREG|0644, st_size=370, ...}) = 0
  158. 28139 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f93d3bf5000
  159. 28139 read(3, "[client]\nsocket=/tmp/mysql.sock\n"..., 4096) = 370
  160. 28139 read(3, "", 4096) = 0
  161. 28139 close(3) = 0
  162. 28139 munmap(0x7f93d3bf5000, 4096) = 0
  163. 28139 stat("/etc/mysql/my.cnf", 0x7ffda2f5c8d0) = -1 ENOENT (No such file or directory)
  164. 28139 stat("/usr/local/mysql/etc/my.cnf", 0x7ffda2f5c8d0) = -1 ENOENT (No such file or directory)
  165. 28139 stat("/root/.my.cnf", 0x7ffda2f5c8d0) = -1 ENOENT (No such file or directory)
  166. 28139 getrlimit(RLIMIT_NOFILE, {rlim_cur=1073741816, rlim_max=1073741816}) = 0
  167. 28139 mmap(NULL, 17179869184, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8fd1f84000
  168. 28139 +++ killed by SIGKILL +++

可以看到最后用 mmap 一次分配了 16G 内存,然后就被杀了。

mmap 前调用了 getrlimit , 猜测是 mysql 会根据系统资源限制来分配内存

MySQL源码分析

MySQL 源码中直接调用 getrlimit 的地方不多,排除了 ndb、innodb_memcached、libevent 之后,只有一处直接调用:

  1. static uint set_max_open_files(uint max_file_limit)
  2. {
  3. struct rlimit rlimit;
  4. uint old_cur;
  5. DBUG_ENTER("set_max_open_files");
  6. DBUG_PRINT("enter",("files: %u", max_file_limit));
  7. if (!getrlimit(RLIMIT_NOFILE,&rlimit))
  8. {
  9. old_cur= (uint) rlimit.rlim_cur;
  10. DBUG_PRINT("info", ("rlim_cur: %u rlim_max: %u",
  11. (uint) rlimit.rlim_cur,
  12. (uint) rlimit.rlim_max));
  13. if (rlimit.rlim_cur == (rlim_t) RLIM_INFINITY)
  14. rlimit.rlim_cur = max_file_limit;
  15. if (rlimit.rlim_cur >= max_file_limit)
  16. DBUG_RETURN(rlimit.rlim_cur); /* purecov: inspected */
  17. rlimit.rlim_cur= rlimit.rlim_max= max_file_limit;
  18. if (setrlimit(RLIMIT_NOFILE, &rlimit))
  19. max_file_limit= old_cur; /* Use original value */
  20. else
  21. {
  22. rlimit.rlim_cur= 0; /* Safety if next call fails */
  23. (void) getrlimit(RLIMIT_NOFILE,&rlimit);
  24. DBUG_PRINT("info", ("rlim_cur: %u", (uint) rlimit.rlim_cur));
  25. if (rlimit.rlim_cur) /* If call didn't fail */
  26. max_file_limit= (uint) rlimit.rlim_cur;
  27. }
  28. }
  29. DBUG_PRINT("exit",("max_file_limit: %u", max_file_limit));
  30. DBUG_RETURN(max_file_limit);

其中逻辑是:如果系统的文件打开限制是 RLIM_INFINITY 或者比要设置的 max_file_limit 大,都返回系统的限制。

这个函数也只被直接调用一次:

  1. uint my_set_max_open_files(uint files)
  2. {
  3. struct st_my_file_info *tmp;
  4. DBUG_ENTER("my_set_max_open_files");
  5. DBUG_PRINT("enter",("files: %u my_file_limit: %u", files, my_file_limit));
  6. files+= MY_FILE_MIN;
  7. files= set_max_open_files(MY_MIN(files, OS_FILE_LIMIT)); // 获取最大打开文件数
  8. if (files <= MY_NFILE)
  9. DBUG_RETURN(files);
  10. // 分配内存
  11. if (!(tmp= (struct st_my_file_info*) my_malloc(key_memory_my_file_info,
  12. sizeof(*tmp) * files,
  13. MYF(MY_WME))))
  14. DBUG_RETURN(MY_NFILE);
  15. // 初始化
  16. /* Copy any initialized files */
  17. memcpy((char*) tmp, (char*) my_file_info,
  18. sizeof(*tmp) * MY_MIN(my_file_limit, files));
  19. memset((tmp + my_file_limit), 0,
  20. MY_MAX((int) (files - my_file_limit), 0) * sizeof(*tmp));
  21. my_free_open_file_info(); /* Free if already allocated */
  22. my_file_info= tmp;
  23. my_file_limit= files;
  24. DBUG_PRINT("exit",("files: %u", files));
  25. DBUG_RETURN(files);
  26. }

原来 MySQL5 会根据最大可打开文件数,提前为每个文件分配和初始化内存,在这个时候就可能分配过多内存,导致 OOM。MySQL8修复了这个问题。

解决

启动前设置ulimit

  1. ENTRYPOINT ["ulimit -n 1048576 && /home/admin/start.sh"]

写在后面

在linux中,open files设置过大会出现很多问题,应该合理控制open files数量

参考文献

https://learnku.com/articles/34598

作者:京东零售 杨云龙

来源:京东云开发者社区 转载请注明来源

原文链接:https://www.cnblogs.com/Jcloud/p/17769006.html

 友情链接:直通硅谷  点职佳  北美留学生论坛

本站QQ群:前端 618073944 | Java 606181507 | Python 626812652 | C/C++ 612253063 | 微信 634508462 | 苹果 692586424 | C#/.net 182808419 | PHP 305140648 | 运维 608723728

W3xue 的所有内容仅供测试,对任何法律问题及风险不承担任何责任。通过使用本站内容随之而来的风险与本站无关。
关于我们  |  意见建议  |  捐助我们  |  报错有奖  |  广告合作、友情链接(目前9元/月)请联系QQ:27243702 沸活量
皖ICP备17017327号-2 皖公网安备34020702000426号