{"id":1676,"date":"2024-11-07T08:35:34","date_gmt":"2024-11-07T00:35:34","guid":{"rendered":"http:\/\/blue.yn.cn\/?p=1676"},"modified":"2024-11-09T19:16:14","modified_gmt":"2024-11-09T11:16:14","slug":"promql%e8%bf%9b%e9%98%b6","status":"publish","type":"post","link":"http:\/\/blue.yn.cn\/?p=1676","title":{"rendered":"PromQL\u8fdb\u9636"},"content":{"rendered":"<h3>PromQL\u8d44\u6e90\u76d1\u63a7\u6848\u4f8b<\/h3>\n<h4>\u76d1\u63a7\u7cfb\u7edf\u8d44\u6e90\u65b9\u6cd5\u8bba<\/h4>\n<ul>\n<li>1.<code>google<\/code>\u7684\u56db\u4e2a\u9ec4\u91d1\u6307\u6807\u7740\u773c\u70b9\u5728\u670d\u52a1\u76d1\u63a7\uff0c\u8fd9\u56db\u4e2a\u6307\u6807\u5206\u522b\u662f\u5ef6\u8fdf\u3001\u6d41\u91cf\u3001\u9519\u8bef\u3001\u9971\u548c\u5ea6\u3002<\/li>\n<li>2.<code>red<\/code>\u65b9\u6cd5\u4e3b\u8981\u7740\u773c\u70b9\u5728\u7528\u6237\u4f53\u9a8c\uff0c\u5b83\u57fa\u4e8e<code>google<\/code>\u7684\u9ec4\u91d1\u6307\u6807\uff0c\u7ec6\u5316\u4e86\u4e09\u4e2a\u5173\u952e\u6307\u6807\uff1a\u8bf7\u6c42\u7387\u3001\u9519\u8bef\u6570\u3001\u8bf7\u6c42\u5904\u7406\u65f6\u95f4<\/li>\n<li>3.<code>use<\/code>\u65b9\u6cd5\u4e3b\u8981\u7740\u773c\u70b9\u5728\u7cfb\u7edf\u8d44\u6e90\u7684\u4f7f\u7528i\u60c5\u51b5\uff0c\u8fd9\u4e09\u4e2a\u6307\u6807\u5206\u522b\u662f\u4f7f\u7528\u7387\u3001\u9971\u548c\u5ea6\u3001\u9519\u8bef\u3002<\/li>\n<\/ul>\n<h4>\u4f7f\u7528use\u76d1\u63a7\u7cfb\u7edf\u8d44\u6e90<\/h4>\n<p>\u5728\u76d1\u63a7<code>cpu<\/code>\u3001\u5185\u5b58\u3001\u78c1\u76d8\u548c\u7f51\u7edc\u7b49\u7cfb\u7edf\u8d44\u6e90\u65f6\uff0c\u4f7f\u7528<code>use<\/code>\u7684\u65b9\u6cd5\u8bba\u662f\u6700\u4e3a\u5408\u9002\u7684\uff0c\u5b83\u8981\u6c42\u6211\u4eec\u9488\u5bf9\u6bcf\u4e2a\u8d44\u6e90\uff08\u4e5f\u5c31\u662f<code>cpu<\/code>\u3001\u5185\u5b58\u3001\u7f51\u7edc\u7b49\uff09\u5206\u522b\u68c0\u67e5\u4e09\u4e2a\u5173\u952e\u65b9\u9762\uff1a\u4f7f\u7528\u7387\u3001\u9971\u548c\u5ea6\u548c\u9519\u8bef\u3002<\/p>\n<ul>\n<li><strong>\u4f7f\u7528\u7387<\/strong>\uff1a\u8868\u793a\u8d44\u6e90\u5728\u5de5\u4f5c\u65f6\u5360\u7528\u7684\u5e73\u5747\u65f6\u95f4\uff0c\u4f8b\u5982\uff0c<code>cpu<\/code>\u5728\u8fc7\u53bb\u768460\u79d2\u5185\u670930\u79d2\u88ab\u7528\u6765\u6267\u884c\u6307\u4ee4\uff0c\u90a3\u4e48\u6211\u4eec\u53ef\u4ee5\u8bf4\u8fd9\u4e2a<code>cpu<\/code>\u7684\u4f7f\u7528\u7387\u662f50%<\/li>\n<li><strong>\u9971\u548c\u5ea6<\/strong>\uff1a\u6307\u660e\u8d44\u6e90\u5df2\u7ecf\u5230\u8fbe\u6216\u63a5\u8fd1\u5176\u5904\u7406\u80fd\u529b\u6781\u9650\u7684\u7a0b\u5ea6\uff0c\u8868\u660e\u65e0\u6cd5\u518d\u5904\u7406\u66f4\u591a\u7684\u5de5\u4f5c\u91cf\u3002\u8fd9\u901a\u5e38\u901a\u8fc7\u961f\u5217\u957f\u5ea6\u7b49\u6307\u6807\u6765\u8861\u91cf\u3002<\/li>\n<li><strong>\u9519\u8bef<\/strong>\uff1a\u8bb0\u5f55\u4e86\u8d44\u6e90\u5728\u8fd0\u884c\u8fc7\u7a0b\u4e2d\u53d1\u751f\u7684\u9519\u8bef\u4e8b\u4ef6\u7684\u6b21\u6570\u3002<\/li>\n<\/ul>\n<h3>cpu\u76d1\u63a7\u6848\u4f8b\u5b9e\u8df5<\/h3>\n<h4>cpu\u91cd\u70b9\u76d1\u63a7\u7684\u7ef4\u5ea6<\/h4>\n<p>\u5bf9\u5e94<code>cpu<\/code>\u8d44\u6e90\u7684\u76d1\u63a7\uff0c\u91c7\u7528<code>use<\/code>\u65b9\u6cd5\u8bba\u901a\u5e38\u5173\u6ce8\u4ee5\u4e0b\u51e0\u70b9\uff1a<\/p>\n<ul>\n<li><code>cpu<\/code>\u4f7f\u7528\u7387\uff1a\u8861\u91cf<code>cpu<\/code>\u5728\u4e00\u6bb5\u65f6\u95f4\u5185\u5904\u4e8e\u6d3b\u8dc3\u72b6\u6001\u7684\u767e\u5206\u6bd4\u3002\u9ad8\u4f7f\u7528\u7387\u53ef\u80fd\u8868\u660e\u7cfb\u7edf\u6b63\u5728\u5bc6\u96c6\u6267\u884c\u4efb\u52a1\u3002<\/li>\n<li><code>cpu<\/code>\u9971\u548c\u5ea6\uff1a\u53cd\u6620\u4e86\u5728\u67d0\u4e00\u65f6\u95f4\u70b9\u7b49\u5f85<code>cpu<\/code>\u65f6\u95f4\u7684\u8fdb\u7a0b\u6570\u91cf\u3002\u9ad8\u9971\u548c\u5ea6\u53ef\u80fd\u610f\u5473\u7740\u6709\u5f88\u591a\u4efb\u52a1\u5728\u4e89\u593a<code>cpu<\/code>\u8d44\u6e90\u3002<\/li>\n<li>\u9519\u8bef\uff0c\u901a\u5e38\u5bf9<code>cpu<\/code>\u4e0d\u592a\u5f71\u54cd<\/li>\n<\/ul>\n<h4>\u8ba1\u7b97cpu\u5e73\u5747\u4f7f\u7528\u7387<\/h4>\n<p>\u8ba1\u7b97<code>cpu<\/code>\u5e73\u5747\u4f7f\u7528\u7387\u7684\u516c\u5f0f\uff1a\uff081-<code>cpu<\/code>\u7a7a\u95f2\u7387\uff09*100=<code>cpu<\/code>\u4f7f\u7528\u7387<\/p>\n<ul>\n<li>\n<p>\u83b7\u53d6\u6bcf\u4e2a<code>cpu<\/code>\u6a21\u5f0f\u7684\u53d8\u5316\u7387\uff0c\u4f7f\u7528<code>irate<\/code>\u51fd\u6570\u7075\u654f\u5ea6\u66f4\u9ad8\uff0c\u53cd\u6620\u66f4\u63a5\u8fd1\u5b9e\u9645\u4f7f\u7528\u7387<br \/>\n<code>irate(node_cpu_seconds_total[1m])<\/code><\/p>\n<\/li>\n<li>\n<p>\u67e5\u8be2<code>cpu<\/code>\u7684\u7a7a\u95f2\u65f6\u95f4\u6bd4\u7387\uff0c\u5373<code>cpu<\/code>\u5904\u4e8e\u7a7a\u95f2\u72b6\u6001\u7684\u65f6\u95f4\u6bd4\u4f8b<br \/>\n<code>irate(node_cpu_seconds_total{mode=&quot;idle&quot;}[1m])<\/code><\/p>\n<\/li>\n<li>\n<p>\u7cfb\u7edf\u53ef\u80fd\u4f1a\u6709\u591a\u4e2a<code>cpu<\/code>\u6838\u5fc3\uff0c\u56e0\u6b64\u6211\u4eec\u9700\u8981\u901a\u8fc7<code>avg<\/code>\u805a\u5408\u51fd\u6570\u6765\u8ba1\u7b97<code>cpu<\/code>\u7684\u5e73\u5747\u7a7a\u95f2\u7387\uff0c\u6700\u540e\u6309\u7167\u5b9e\u4f8b\u8fdb\u884c\u5206\u7ec4\u7edf\u8ba1\u3002<br \/>\n<code>avg(irate(node_cpu_seconds_total{mode=&quot;idle&quot;}[1m])) by (instance)<\/code><\/p>\n<\/li>\n<li>\n<p>\u4f7f\u75281\u51cf\u53bb\u6bcf\u4e2a\u6838\u5fc3\u7684\u5e73\u5747\u7a7a\u95f2\u7387\uff0c\u5f97\u5230\u5e73\u5747\u4f7f\u7528\u7387\uff0c\u7136\u540e\u5c06\u6b64\u503c\u4e58\u4ee5100\uff0c\u8f6c\u6362\u4e3a\u767e\u5206\u6bd4\uff0c\u6700\u7ec8\u5f97\u51fa<code>cpu<\/code>\u7684\u6574\u4f53\u5e73\u5747\u4f7f\u7528\u7387<br \/>\n<code>(1-avg(irate(node_cpu_seconds_total{mode=&quot;idle&quot;}[1m])) by (instance)) * 100<\/code><\/p>\n<\/li>\n<\/ul>\n<h4>\u8ba1\u7b97cpu\u9971\u548c\u5ea6<\/h4>\n<p>\u8bc4\u4f30\u4e3b\u673a\u7684<code>cpu<\/code>\u9971\u548c\u5ea6\uff0c\u6700\u5e38\u7528\u7684\u65b9\u6cd5\u662f\u8ddf\u8e2a\u7cfb\u7edf\u7684\u5e73\u5747\u8d1f\u8f7d\u3002\u901a\u5e38\u60c5\u51b5\u4e0b\uff0c\u5982\u679c\u8fd9\u4e2a\u8d1f\u8f7d\u7684\u503c\u4f4e\u4e8e<code>cpu<\/code>\u6838\u5fc3\u7684\u6570\u91cf\uff0c\u53ef\u4ee5\u8ba4\u4e3a\u7cfb\u7edf\u8fd0\u884c\u6b63\u5e38\u3002\u7136\u800c\uff0c\u5f53\u5e73\u5747\u8d1f\u8f7d\u6301\u7eed\u9ad8\u4e8e<code>cpu<\/code>\u6838\u5fc3\u6570\u91cf\u7684\u4e24\u500d\u65f6\uff0c\u8fd9\u901a\u5e38\u662f<code>cpu<\/code>\u5df2\u7ecf\u975e\u5e38\u9971\u548c\u4e86\u3002<\/p>\n<p>\u8ba1\u7b97<code>cpu<\/code>\u9971\u548c\u5ea6\u7684\u516c\u5f0f\uff1a1\u5206\u949f\u5e73\u5747\u8d1f\u8f7d\/(<code>cpu<\/code>\u6838\u5fc3\u6570<em>2)<\/em>100=<code>cpu<\/code>\u9971\u548c\u5ea6\u7684\u767e\u5206\u6bd4\uff0c\u5f53\u5927\u4e8e80%\u5219\u8ba4\u4e3a<code>cpu<\/code>\u5df2\u7ecf\u63a5\u8fd1\u5904\u7406\u80fd\u529b\u7684\u6781\u9650\u3002<\/p>\n<ul>\n<li>\n<p>\u83b7\u53d61\u5206\u949f\u5e73\u5747\u8d1f\u8f7d\u503c\uff0c\u7136\u540e\u6309\u7167\u5b9e\u4f8b\u548c<code>job<\/code>\u8fdb\u884c\u5206\u7ec4\u8ba1\u7b97<br \/>\n<code>sum(node_load1) by (instance,job)<\/code><\/p>\n<\/li>\n<li>\n<p>\u83b7\u53d6<code>cpu<\/code>\u7684\u6838\u5fc3\u6570\uff0c\u7136\u540e\u4e58\u4ee52\uff0c\u6309\u5b9e\u4f8b\u8fdb\u884c\u5206\u7ec4\u8ba1\u7b97\u6bcf\u4e2a\u5b9e\u4f8b\u7684\u6838\u5fc3\u6570<br \/>\n<code>count(node_cpu_seconds_total{mode=&quot;idle&quot;}) by (instance,job) * 2<\/code><\/p>\n<\/li>\n<li>\n<p>\u8ba1\u7b97<code>cpu<\/code>\u9971\u548c\u5ea6\u767e\u5206\u6bd4\uff0c\u5982\u679c\u8fd9\u4e2a\u767e\u5206\u6bd4\u5927\u4e8e80%\uff0c<\/p>\n<\/li>\n<\/ul>\n<pre><code class=\"language-bash\">sum(node_load1) by (instance,job) \n\/\n(count(node_cpu_seconds_total{mode=&quot;idle&quot;}) by (instance,job) * 2) * 100<\/code><\/pre>\n<h4>\u914d\u7f6e<code>cpu<\/code>\u544a\u8b66\u89c4\u5219<\/h4>\n<ol>\n<li>\u5b9a\u4e49<code>cpu<\/code>\u544a\u8b66\u89c4\u5219\u7ec4\uff0c\u7136\u540e\u6ce8\u5165\u4e24\u6761\u89c4\u5219<\/li>\n<\/ol>\n<pre><code class=\"language-bash\">cat node_rules.yml\ngroups:\n- name: CPU\u544a\u8b66\u89c4\u5219\n  rules:\n  - alert: \u8282\u70b9CPU\u4f7f\u7528\u7387\u8d85\u8fc780%\n    expr: ( 1 - avg(irate(node_cpu_seconds_total{mode=&quot;idle&quot;}[1m])) by (instance,job) ) * 100 &gt; 80\n    for: 1m\n    labels:\n      severity: warning\n    annotations:\n      summary: &quot;\u4e3b\u673aCPU\u5229\u7528\u7387\u8fc7\u9ad8\uff0c\u5b9e\u4f8b\uff1a{{ $labels.instance }}, \u4efb\u52a1:{{ $labels.job }}&quot;\n      description: &quot;\u8be5\u5b9e\u4f8b\u7684CPU\u4f7f\u7528\u7387\u5927\u4e8e80%\uff0c\u5f53\u524d\u4f7f\u7528\u7387\uff1a{{ $value }}%\u3002\u53ef\u80fd\u5b58\u5728CPU\u8d44\u6e90\u7d27\u5f20\u60c5\u51b5\u3002&quot;\n\n  - alert: \u8282\u70b9CPU\u9971\u548c\u5ea6\u8fc7\u9ad8\n    expr: sum(node_load1) by (instance,job) \/ (count(node_cpu_seconds_total{mode=&quot;idle&quot;}) by (instance,job) * 2) * 100 &gt; 80\n    for: 2m\n    labels:\n      severity: critical\n    annotations:\n      summary: &quot;CPU\u9971\u548c\u5ea6\u8fc7\u9ad8\uff0c\u5b9e\u4f8b\uff1a{{ $labels.instance }}, \u4efb\u52a1:{{ $labels.job }}&quot;\n      description: &quot;\u8be5\u5b9e\u4f8b\u76841\u5206\u949f\u5e73\u5747CPU\u8d1f\u8f7d\u8d85\u8fc7\u4e86\u6838\u5fc3\u6570\u7684\u4e24\u500d\uff0c\u5df2\u7ecf\u6301\u7eed2\u5206\u949f\uff0c\u5f53\u524dCPU\u9971\u548c\u5ea6\uff1a{{ $value }}%\u3002\u9700\u8981\u7acb\u5373\u68c0\u67e5\u7cfb\u7edf\u8d1f\u8f7d\u60c5\u51b5\u3002&quot;<\/code><\/pre>\n<ol start=\"2\">\n<li>\n<p>\u68c0\u67e5\u544a\u8b66\u89c4\u5219\u914d\u7f6e\u662f\u5426\u6b63\u786e<\/p>\n<pre><code class=\"language-bash\">\/etc\/prometheus\/promtool check rules \/etc\/prometheus\/rules\/node_rules.yml\nChecking \/etc\/prometheus\/rules\/node_rules.yml\nSUCCESS: 2 rules found<\/code><\/pre>\n<\/li>\n<li>\n<p>\u52a0\u8f7d\u914d\u7f6e\u6587\u4ef6<\/p>\n<\/li>\n<\/ol>\n<pre><code class=\"language-bash\"># \u914d\u7f6e\u6587\u4ef6\u6dfb\u52a0\u544a\u8b66\u6587\u4ef6\u8def\u5f84\ncat prometheus.yml\nrule_files:\n  - &quot;\/etc\/prometheus\/rules\/*.yml&quot;\n\n# \u52a0\u8f7d\u914d\u7f6e\u6587\u4ef6\ncurl -X POST http:\/\/localhost:9090\/-\/reload<\/code><\/pre>\n<ol start=\"4\">\n<li><code>prometheus alerts<\/code>\u9875\u9762\u6548\u679c<\/li>\n<\/ol>\n<p><img decoding=\"async\" src=\"\/wp-content\/uploads\/2024\/11\/image-1731074439617.png\" alt=\"file\" \/><\/p>\n<h3>\u5185\u5b58\u76d1\u63a7\u6848\u4f8b\u5b9e\u8df5<\/h3>\n<h4>\u5185\u5b58\u91cd\u70b9\u76d1\u63a7\u7684\u7ef4\u5ea6<\/h4>\n<p>\u5bf9\u4e8e\u5185\u5b58\u8d44\u6e90\u7684\u76d1\u63a7\uff0c\u91c7\u7528<code>use<\/code>\u65b9\u6cd5\u8bba\u901a\u5e38\u5173\u6ce8\u4ee5\u4e0b\u51e0\u70b9\uff1a<\/p>\n<ul>\n<li>1.\u5185\u5b58\u4f7f\u7528\u7387\uff1a\u8861\u91cf\u7cfb\u7edf\u5185\u5b58\u88ab\u4f7f\u7528\u7684\u6210\u90fd<\/li>\n<li>2.\u5185\u5b58\u9971\u548c\u5ea6\uff1a\u8868\u660e\u7cfb\u7edf\u5185\u5b58\u7684\u8d1f\u8f7d\u7a0b\u5ea6\u3002\u5185\u5b58\u9971\u548c\u5ea6\uff0c\u901a\u5e38\u76d1\u63a7<code>swap<\/code>\u7684\u4f7f\u7528\u7387\u6765\u8bc4\u4f30<\/li>\n<li>3.\u5185\u5b58\u9519\u8bef\uff1a\u901a\u5e38\u8f83\u5c11\u53d1\u751f\u3002<\/li>\n<\/ul>\n<h4>\u8ba1\u7b97\u5185\u5b58\u4f7f\u7528\u7387<\/h4>\n<p>\u8ba1\u7b97\u5185\u5b58\u4f7f\u7528\u7387\u767e\u5206\u6bd4\u516c\u5f0f\uff1a\uff08\u603b\u5185\u5b58-\u53ef\u7528\u5185\u5b58\uff09\/\u603b\u5185\u5b58*100=\u5185\u5b58\u4f7f\u7528\u7387<br \/>\n<code>(node_memory_MemTotal_bytes-node_memory_MemAvailable_bytes)\/node_memory_MemTotal_bytes*100<\/code><\/p>\n<h4>\u8ba1\u7b97\u5185\u5b58\u9971\u548c\u5ea6<\/h4>\n<p>\u8ba1\u7b97\u5185\u5b58\u9971\u548c\u5ea6\u767e\u5206\u6bd4\u516c\u5f0f\uff1a\uff08\u603b<code>swap<\/code>-\u672a\u4f7f\u7528\u7684<code>swap<\/code>\uff09\/\u603b<code>swap<\/code>*100<br \/>\n<code>(node_memory_SwapTotal_bytes-node_memory_SwapFree_bytes)\/node_memory_SwapTotal_bytes*100<\/code><\/p>\n<h4>\u914d\u7f6e\u5185\u5b58\u544a\u8b66\u89c4\u5219<\/h4>\n<ol>\n<li>\u5b9a\u4e49\u5185\u5b58\u544a\u8b66\u89c4\u5219\u7ec4\uff0c\u7136\u540e\u6ce8\u5165\u4e24\u6761\u89c4\u5219<\/li>\n<\/ol>\n<pre><code class=\"language-bash\">cat \/etc\/prometheus\/rules\/node_rules.yml\ngroups:\n- name: \u5185\u5b58\u544a\u8b66\u89c4\u5219\n  rules:\n  - alert: \u8282\u70b9\u5185\u5b58\u4e0d\u8db3\n    expr: (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes)\/ node_memory_MemTotal_bytes * 100 &gt; 80\n    for: 2m\n    labels:\n      severity: warning\n    annotations:\n      summary: &quot;\u4e3b\u673a\u5185\u5b58\u4f7f\u7528\u7387\u8f83\u9ad8, \u5b9e\u4f8b:{{ $labels.instance }}, \u4efb\u52a1:{{ $labels.job }}&quot;\n      description: &quot;\u8be5\u5b9e\u4f8b\u7684\u5185\u5b58\u4f7f\u7528\u7387\u6301\u7eed2\u5206\u949f\u9ad8\u4e8e80%\uff0c\u5f53\u524d\u4f7f\u7528\u7387\u7387\uff1a{{ $value}}%&quot;\n\n  - alert: \u5185\u5b58\u9971\u548c\u5ea6\u2fbc\n    expr: ( 1 - node_memory_SwapFree_bytes \/ node_memory_SwapTotal_bytes) * 100 &gt; 10\n    for: 2m\n    labels:\n      severity: warning\n    annotations:\n      summary: &quot;\u4e3b\u673a\u5185\u5b58\u5185\u5b58\u9971\u548c\u5ea6\u9ad8, \u5b9e\u4f8b:{{ $labels.instance }}, \u4efb\u52a1:{{ $labels.job }}&quot;\n      description: &quot;SWAP\u5185\u5b58\u4f7f\u7528\u7387\u5df2\u8fde\u7eed2\u5206\u949f\u8d85\u8fc710%\uff0c\u8868\u660e\u5185\u5b58\u9971\u548c\u5ea6\u8fc7\u9ad8\uff0c\u5f53\u524dSWAP\u4f7f\u7528\u7387\u4e3a\uff1a{{ $value }}%\u3002&quot;<\/code><\/pre>\n<ol start=\"2\">\n<li>\n<p>\u6d4b\u8bd5\u89c4\u5219\u914d\u7f6e\u6587\u4ef6\uff0c\u7136\u540e\u91cd\u65b0\u52a0\u8f7d<code>prometheus<\/code>\u914d\u7f6e\u6587\u4ef6<\/p>\n<\/li>\n<li>\n<p><code>prometheus alerts<\/code>\u9875\u9762\u6548\u679c<\/p>\n<\/li>\n<\/ol>\n<p><img decoding=\"async\" src=\"\/wp-content\/uploads\/2024\/11\/image-1731078599543.png\" alt=\"file\" \/><\/p>\n<h3>\u78c1\u76d8\u76d1\u63a7\u6848\u4f8b\u5b9e\u8df5<\/h3>\n<h4>\u78c1\u76d8\u91cd\u70b9\u76d1\u63a7\u7684\u7ef4\u5ea6<\/h4>\n<p>\u5bf9\u4e8e\u78c1\u76d8\uff0c\u6211\u4eec\u4e0d\u5149\u9700\u8981\u5ea6\u91cf\u78c1\u76d8\u4f7f\u7528\u7387\uff0c\u751a\u81f3\u8fd8\u9700\u8981\u5ea6\u91cf\u78c1\u76d8<code>io<\/code>\u8bfb\u5199\u541e\u5410\u91cf\u3001\u4ee5\u53ca\u78c1\u76d8<code>iops<\/code><\/p>\n<ul>\n<li>1.\u78c1\u76d8\u4f7f\u7528\u7387\uff1a\u8861\u91cf\u78c1\u76d8\u7a7a\u95f4\u4f7f\u7528\u7387\u3001<code>inode<\/code>\u4f7f\u7528\u7387<\/li>\n<li>2.\u78c1\u76d8\u5ef6\u8fdf\uff1a\u6709\u4e9b\u78c1\u76d8\u660e\u786e\u6807\u6ce8\u4e86\u5ef6\u8fdf\uff0c\u6211\u4eec\u53ef\u4ee5\u76d1\u63a7\u78c1\u76d8\u7684\u8bfb\u5199\u5ef6\u8fdf\uff0c\u800c\u540e\u8fdb\u884c\u5224\u5b9a\uff1b<\/li>\n<li>3.\u78c1\u76d8\u9971\u548c\u5ea6\uff1a\u901a\u8fc7\u76d1\u63a7<code>i\/o<\/code>\u541e\u5410\u91cf\u548c<code>iops<\/code>\uff0c\u53ef\u4ee5\u8861\u91cf\u78c1\u76d8\u5728\u7e41\u5fd9\u65f6\u6bb5\u7684\u9971\u548c\u5ea6\u60c5\u51b5\u3002<\/li>\n<\/ul>\n<p>\u5e38\u89c4\u78c1\u76d8\u7684\u6bcf\u79d2\u7684<code>iops<\/code>\u80fd\u529b\u548c<code>io<\/code>\u541e\u5410\u91cf\uff0c\u6709\u6807\u51c6\u540e\uff0c\u5c31\u597d\u57fa\u4e8e\u6807\u51c6\u8fdb\u884c\u89e6\u53d1\u5668\u8bbe\u5b9a\uff0c\u4f8b\u5982\u5f53<code>iops<\/code>\u6bcf\u79d2\u8bfb\u5199\u6b21\u6570\u7684\u9971\u548c\u5ea6\u8d85\u8fc780%\u5219\u9700\u8981\u5173\u6ce8\u4e86<\/p>\n<table>\n<thead>\n<tr>\n<th>\u6027\u80fd\u7c7b\u522b<\/th>\n<th>ESSD\u4e91\u76d8\uff08PL3\uff09<\/th>\n<th>SSD\u4e91\u76d8<\/th>\n<th>\u9ad8\u6548\u4e91\u76d8<\/th>\n<th>\u666e\u901a\u4e91\u76d8<\/th>\n<\/tr>\n<\/thead>\n<tbody>\n<tr>\n<td>iops\uff08\u6bcf\u79d2\u80fd\u5b8c\u6210\u7684\u8bfb\u5199\u6b21\u6570\uff09<\/td>\n<td>1000000\/s<\/td>\n<td>\u6700\u592725000\/s\u3001\u6700\u5c0f2400\/s<\/td>\n<td>\u6700\u59275000\/s\u3001\u6700\u5c0f1960\/s<\/td>\n<td>\u6700\u592780~150\/s<\/td>\n<\/tr>\n<tr>\n<td>io\u541e\u5410\u91cf\uff08\u6bcf\u79d2\u4f20\u8f93\u5927\u5c0f\uff09<\/td>\n<td>\u6700\u59274000MB\/s<\/td>\n<td>\u6700\u5927300MB\/s\u3001\u6700\u5c0f130MB\/s<\/td>\n<td>\u6700\u5927140MB\/s\u3001\u6700\u5c0f103MB\/s<\/td>\n<td>30MB\/s~40MB\/s<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h4>\u8ba1\u7b97\u78c1\u76d8\u7a7a\u95f4\u4f7f\u7528\u7387<\/h4>\n<p>\u8ba1\u7b97\u78c1\u76d8\u7a7a\u95f4\u4f7f\u7528\u7387\uff1a\uff08\u603b\u7684\u78c1\u76d8\u7a7a\u95f4-\u53ef\u7528\u7684\u78c1\u76d8\u7a7a\u95f4=\u5df2\u7528\u7684\u78c1\u76d8\u7a7a\u95f4\uff09\/\u603b\u7684\u78c1\u76d8\u7a7a\u95f4<em>100<br \/>\n\u8ba1\u7b97<code>inode<\/code>\u7a7a\u95f4\u4f7f\u7528\u7387\uff1a\uff08\u603b\u7684<code>inode<\/code>\u6570-\u53ef\u7528\u7684<code>inode<\/code>\u6570\uff09\/\u603b\u7684<code>inode<\/code>\u6570<\/em>100<\/p>\n<ol>\n<li>\n<p>\u8ba1\u7b97\u4e0d\u540c\u5206\u533a\u7684\u78c1\u76d8\u7a7a\u95f4\u4f7f\u7528\u7387\uff08\u6392\u9664<code>tmpfs<\/code>\u7684\u5206\u533a\uff09<br \/>\n<code>(node_filesystem_size_bytes{device!=&quot;tmpfs&quot;}-node_filesystem_avail_bytes{device!=&quot;tmpfs&quot;})\/node_filesystem_size_bytes{device!=&quot;tmpfs&quot;}*100<\/code><\/p>\n<\/li>\n<li>\n<p>\u8ba1\u7b97<code>inode<\/code>\u4f7f\u7528\u7387\uff08\u6392\u9664<code>tmpfs<\/code>\u7684\u5206\u533a\uff09<br \/>\n<code>(node_filesystem_files{device!=&quot;tmpfs&quot;}-node_filesystem_files_free{device!=&quot;tmpfs&quot;})\/node_filesystem_files{device!=&quot;tmpfs&quot;}*100<\/code><\/p>\n<\/li>\n<\/ol>\n<h4>\u8ba1\u7b97\u78c1\u76d8io\u541e\u5410\u91cf<\/h4>\n<p><code>i\/o\u541e\u5410\u91cf<\/code>\uff1a\u8861\u91cf\u5728\u5355\u4f4d\u65f6\u95f4\u5185\u78c1\u76d8\u80fd\u591f\u8bfb\u5199\u591a\u5c11\u6570\u636e\u91cf\uff0c\u901a\u5e38\u4ee5<code>MB\/s<\/code>\uff08\u5146\u5b57\u8282\u6bcf\u79d2\uff09\u4e3a\u5355\u4f4d<br \/>\n<code>i\/o<\/code>\u541e\u5410\u91cf\u9971\u548c\u5ea6\uff08%\uff09=\uff08\u5f53\u524d<code>i\/o<\/code>\u541e\u5410\u91cf\/\u78c1\u76d8\u7684\u6700\u5927<code>i\/o<\/code>\u541e\u5410\u91cf\u80fd\u529b\uff09*100<\/p>\n<ol>\n<li>\n<p>\u6a21\u62df<code>io<\/code>\u5199\u5165\u541e\u5410\u91cf\uff0c\u5e76\u9650\u5236\u5199\u5165<code>io<\/code>\u6700\u5927<code>20MB\/s<\/code><br \/>\n<code>dd if=\/dev\/zero bs=1M count=10000 | pv -L 20M &gt; \/tmp\/bigdata<\/code><\/p>\n<\/li>\n<li>\n<p>\u83b7\u53d6\u5f53\u524d\u78c1\u76d8\u6700\u8fd11\u5206\u949f\u7684\uff0c\u5199\u5165\u541e\u5410\u91cf\u6700\u5927\u7684\u503c\uff0c\u7136\u540e\u6309\u5b9e\u4f8b\u8fdb\u884c\u5206\u7ec4\u8ba1\u7b97\uff0c\u6700\u540e\u5c06\u7ed3\u679c\u8f6c\u4e3a<code>MB<\/code><br \/>\n<code>round(max(irate(node_disk_written_bytes_total[1m])) by (instance,job)\/1024\/1024)<\/code><\/p>\n<\/li>\n<li>\n<p>\u5f53\u524d<code>io<\/code>\u5199\u5165\u541e\u5410\u91cf\u9664\u4ee5\u78c1\u76d8\u6700\u5927\u5199\u5165<code>io<\/code>\u541e\u5410\u91cf\uff0c\u4e58\u4ee5100\uff0c\u83b7\u53d6<code>io<\/code>\u541e\u5410\u91cf\u7684\u9971\u548c\u5ea6\u767e\u5206\u6bd4<\/p>\n<\/li>\n<\/ol>\n<pre><code class=\"language-bash\">(max(rate(node_disk_written_bytes_total[1m])) by (instance,job) \/ 1024 \/1024) \/ 30 * 100\n\n# \u544a\u8b66(\u5f53IO\u5199\u2f0a\u541e\u5410\u2f24\u4e8e80%\u5219\u89e6\u53d1\u544a\u8b66)\nround(max(rate(node_disk_written_bytes_total[1m])) by (instance,job) \/ 1024\/1024 \/ 30 * 100) &gt; 80<\/code><\/pre>\n<ol start=\"4\">\n<li>\u6a21\u62df<code>io<\/code>\u8bfb\u53d6\u541e\u5410\u91cf\uff0c\u5e76\u9650\u5236\u8bfb\u53d6<code>io<\/code>\u6700\u5927<code>20MB\/s<\/code><\/li>\n<\/ol>\n<pre><code class=\"language-bash\"># yum install pv -y\n# pv -L 20M \/tmp\/bigdata &gt; \/dev\/null<\/code><\/pre>\n<ol start=\"5\">\n<li>\n<p>\u83b7\u53d6\u5f53\u524d\u78c1\u76d8\u6700\u8fd11\u5206\u949f\u8bfb\u53d6\u541e\u5410\u91cf\u6700\u5927\u7684\u503c\uff0c\u7136\u540e\u6309\u5b9e\u4f8b\u8fdb\u884c\u5206\u7ec4\u8ba1\u7b97\uff0c\u6700\u540e\u5c06\u7ed3\u679c\u8f6c\u4e3a<code>MB<\/code><br \/>\n<code>round(max(irate(node_disk_read_bytes_total[1m])) by (instance,job)\/1024\/1024)<\/code><\/p>\n<\/li>\n<li>\n<p>\u4f7f\u7528\u5f53\u524d<code>io<\/code>\u8bfb\u53d6\u541e\u5410\u91cf\u9664\u4ee5\u78c1\u76d8\u6700\u5927\u8bfb\u53d6<code>io<\/code>\u541e\u5410\u91cf\uff0c\u4e58\u4ee5100\uff0c\u83b7\u53d6<code>io<\/code>\u541e\u5410\u91cf\u7684\u9971\u548c\u5ea6\u767e\u5206\u6bd4<\/p>\n<\/li>\n<\/ol>\n<pre><code class=\"language-bash\">max(irate(node_disk_read_bytes_total[1m])) by (instance,job) \/ 1024 \/1024 \/30 * 100\n\n# \u544a\u8b66(\u5f53IO\u5199\u2f0a\u541e\u5410\u2f24\u4e8e80%\u5219\u89e6\u53d1\u544a\u8b66)\nround(max(irate(node_disk_read_bytes_total[1m])) by (instance,job) \/ 1024 \/1024 \/30 * 100 ) &gt; 80<\/code><\/pre>\n<h4>\u8ba1\u7b97\u78c1\u76d8iops<\/h4>\n<p><code>iops<\/code>\uff1a\u8861\u91cf\u5728\u5355\u4f4d\u65f6\u95f4\u5185\u78c1\u76d8\u80fd\u591f\u8bfb\u5199\u64cd\u4f5c\u7684\u6b21\u6570\u3002<br \/>\n<code>iops<\/code>\u9971\u548c\u5ea6\uff08%\uff09\uff1a\uff08\u5f53\u524d<code>iops<\/code>\/\u78c1\u76d8\u7684\u6700\u5927<code>iops<\/code>\u80fd\u529b\uff09*100<\/p>\n<ol>\n<li>\n<p>\u6a21\u62df<code>io<\/code>\u5199\u5165\u541e\u5410\u91cf\uff0c\u4e5f\u76f8\u5f53\u4e8e\u6709\u5927\u91cf<code>iops<\/code><br \/>\n<code>dd if=\/dev\/zero bs=1M count=10000 | pv -L 100M &gt; \/tmp\/bigdata<\/code><\/p>\n<\/li>\n<li>\n<p>\u83b7\u53d6\u5f53\u524d\u78c1\u76d8\u6700\u8fd11\u5206\u949f\u7684<code>iops<\/code>\u5199\u5165\u6b21\u6570\u7684\u6700\u5927\u503c\uff0c\u7136\u540e\u6309\u5b9e\u4f8b\u8fdb\u884c\u5206\u7ec4\u8ba1\u7b97<br \/>\n<code>max(irate(node_disk_writes_completed_total[1m])) by (instance,job)<\/code><\/p>\n<\/li>\n<li>\n<p>\u4f7f\u7528\u5f53\u524d<code>iops<\/code>\u5199\u5165\u6b21\u6570\u9664\u4ee5\u78c1\u76d8\u6700\u5927<code>iops<\/code>\uff0c\u4e58\u4ee5100\uff0c\u83b7\u53d6<code>iops<\/code>\u5199\u5165\u7684\u9971\u548c\u5ea6\u767e\u5206\u6bd4<\/p>\n<\/li>\n<\/ol>\n<pre><code class=\"language-bash\">max(irate(node_disk_writes_completed_total[1m])) by (instance,job) \/ 120 *100\n\n# \u544a\u8b66(\u5f53IOPS\u9971\u548c\u5ea6\u2f24\u4e8e60%\u5219\u89e6\u53d1\u544a\u8b66)\nround(max(irate(node_disk_writes_completed_total[1m])) by (instance,job) \/120 * 100) &gt; 60<\/code><\/pre>\n<ol start=\"4\">\n<li>\u6a21\u62df<code>io<\/code>\u8bfb\u53d6\uff0c\u4e5f\u76f8\u5f53\u4e8e\u6709\u5927\u91cf<code>ios<\/code><\/li>\n<\/ol>\n<pre><code class=\"language-bash\"># yum install pv -y\n# pv -L 50M \/tmp\/bigdata &gt; \/dev\/null<\/code><\/pre>\n<ol start=\"5\">\n<li>\n<p>\u83b7\u53d6\u5f53\u524d\u78c1\u76d8\u6700\u8fd11\u5206\u949f\u8bfb\u53d6<code>iops<\/code>\u6700\u5927\u6b21\u6570\u7684\u503c\uff0c\u7136\u540e\u6309\u5b9e\u4f8b\u8fdb\u884c\u5206\u7ec4\u8ba1\u7b97<br \/>\n<code>max(irate(node_disk_reads_completed_total[1m])) by (instance,job)<\/code><\/p>\n<\/li>\n<li>\n<p>\u4f7f\u7528\u5f53\u524d<code>io<\/code>\u8bfb\u53d6\u541e\u5410\u91cf\u9664\u4ee5\u78c1\u76d8\u6700\u5927\u8bfb\u53d6<code>io<\/code>\u541e\u5410\u91cf\uff0c\u4e58\u4ee5100\uff0c\u83b7\u53d6<code>iops<\/code>\u8bfb\u53d6\u7684\u9971\u548c\u5ea6\u767e\u5206\u6bd4\u3002<\/p>\n<\/li>\n<\/ol>\n<pre><code class=\"language-bash\">max(irate(node_disk_reads_completed_total[1m])) by (instance,job) \/ 120 * 100\n# \u544a\u8b66(\u5f53IOPS\u8bfb\u53d6\u2f24\u4e8e60%\u5219\u89e6\u53d1\u544a\u8b66)\n\nround(max(irate(node_disk_reads_completed_total[1m])) by (instance,job) \/ 120 * 100) &gt; 60<\/code><\/pre>\n<h4>\u914d\u7f6e\u78c1\u76d8\u544a\u8b66\u89c4\u5219<\/h4>\n<ol>\n<li>\u7f16\u5199\u78c1\u76d8\u544a\u8b66\u89c4\u5219\u6587\u4ef6<\/li>\n<\/ol>\n<pre><code class=\"language-yaml\">cat \/etc\/prometheus\/rules\/node_rules.yml\ngroups:\n- name: \u78c1\u76d8\u544a\u8b66\u89c4\u5219\n  rules:\n  - alert: \u78c1\u76d8\u7a7a\u95f4\u544a\u6025\n    expr: ( node_filesystem_size_bytes{device!=&quot;tmpfs&quot;} - node_filesystem_avail_bytes{device!=&quot;tmpfs&quot;} ) \/ node_file                                                                      system_size_bytes{device!=&quot;tmpfs&quot;} * 100 &gt; 70\n    for: 1m\n    labels:\n      severity: critical\n    annotations:\n      summary: &quot;\u5b9e\u4f8b {{ $labels.instance }} \u78c1\u76d8 {{ $labels.mountpoint }}\u5206\u533a\u7a7a\u95f4\u4e0d\u8db3&quot;\n      description: &quot;\u5b9e\u4f8b {{ $labels.instance }} \u78c1\u76d8 {{ $labels.mountpoint}} \u5206\u533a\u7a7a\u95f4\u4f7f\u7528\u7387\u5df2\u8d85\u8fc7 70%\uff0c\u5f53\u524d\u4f7f\u7528\u7387\u4e3a                                                                       {{ $value }}%\uff0c\u8bf7\u53ca\u65f6\u5904\u7406\u3002&quot;\n\n  - alert: \u78c1\u76d8Inode\u7a7a\u95f4\u544a\u6025\n    expr: (node_filesystem_files{device!=&quot;tmpfs&quot;} - node_filesystem_files_free{device!=&quot;tmpfs&quot;} ) \/ node_filesystem_                                                                      files{device!=&quot;tmpfs&quot;} * 100 &gt; 70\n    for: 1m\n    labels:\n      severity: critical\n    annotations:\n      summary: &quot;\u5b9e\u4f8b {{ $labels.instance }} \u78c1\u76d8 {{ $labels.mountpoint }}\u5206\u533aInode\u7a7a\u95f4\u4e0d\u8db3&quot;\n      description: &quot;\u5b9e\u4f8b {{ $labels.instance }} \u78c1\u76d8 {{ $labels.mountpoint}} \u5206\u533a\u7684Inode\u7a7a\u95f4\u4f7f\u7528\u7387\u5df2\u8d85\u8fc7 70%\uff0c\u5f53\u524d\u4f7f                                                                      \u7528\u7387\u4e3a {{ $value }}%\uff0c\u8bf7\u53ca\u65f6\u5904\u7406\u3002&quot;\n\n  - alert: \u78c1\u76d8IOPS\u5199\u5165\u8f83\u9ad8\n    #expr: sum(rate(node_disk_writes_completed_total[1m])) by (instance,job) \/ 5000 * 100 &gt;60\n    #round\u51fd\u6570\u53ef\u4ee5\u5bf9\u503c\u8fdb\u884c\u56db\u820d\u4e94\u5165\n    expr: round(max(irate(node_disk_writes_completed_total[1m])) by (instance,job) \/ 5000 * 100) &gt; 60\n    for: 1m\n    labels:\n      severity: critical\n    annotations:\n      summary: &quot;\u5b9e\u4f8b {{ $labels.instance }} IOPS\u6bcf\u79d2\u5199\u5165\u6b21\u6570\u8d85\u8fc73000\u6b21\/s&quot;\n      description:\n        \u76ee\u524d\u78c1\u76d8IOPS\u5199\u5165\u9971\u548c\u5ea6\u662f {{ $value }}%\n        \u76ee\u524d\u78c1\u76d8IOPS\u6bcf\u79d2\u5199\u5165\u6700\u5927 {{ printf `max(rate(node_disk_writes_completed_total{instance=&quot;%s&quot;,job=&quot;%s&quot;}[1m]))`                                                                       $labels.instance $labels.job | query | first | value | printf &quot;%.2f&quot; }} \u6b21\/s\n\n  - alert: \u78c1\u76d8IOPS\u8bfb\u53d6\u8f83\u9ad8\n    expr: round(max(irate(node_disk_reads_completed_total[1m])) by (instance,job) \/ 5000 * 100) &gt; 60\n    for: 1m\n    labels:\n      severity: critical\n    annotations:\n      summary: &quot;\u5b9e\u4f8b {{ $labels.instance }} IOPS\u6bcf\u79d2\u8bfb\u53d6\u6b21\u6570\u8d85\u8fc73000\u6b21\/s&quot;\n      description:\n        \u76ee\u524d\u78c1\u76d8IOPS\u8bfb\u53d6\u9971\u548c\u5ea6\u662f {{ $value }}%\n        \u76ee\u524d\u78c1\u76d8IOPS\u6bcf\u79d2\u8bfb\u53d6\u6700\u5927 {{ printf `max(rate(node_disk_reads_completed_total{instance=&quot;%s&quot;,job=&quot;%s&quot;}[1m]))`                                                                       $labels.instance $labels.job | query | first | value | printf &quot;%.2f&quot; }} \u6b21\/s\n\n  - alert: \u78c1\u76d8IO\u5199\u5165\u541e\u5410\u8f83\u9ad8\n    expr: round(max(rate(node_disk_written_bytes_total[1m])) by (instance,job) \/ 1024 \/1024 \/ 140 * 100) &gt; 60\n    for: 1m\n    labels:\n      severity: critical\n    annotations:\n      summary: &quot;\u5b9e\u4f8b {{ $labels.instance }} \u78c1\u76d8IO\u5199\u2f0a\u6bcf\u79d2\u8d85\u8fc7\u6700\u592784MB\/s&quot;\n      description:\n        \u76ee\u524d\u78c1\u76d8IO\u5199\u5165\u541e\u5410\u91cf\u7684\u9971\u548c\u5ea6\u662f {{ $value }}%\u3002\n        \u76ee\u524d\u78c1\u76d8IO\u5199\u5165\u541e\u5410\u91cf\u6bcf\u79d2\u6700\u5927\u662f {{ printf `max(rate(node_disk_written_bytes_total{instance=&quot;%s&quot;,job=&quot;%s&quot;}[1m]                                                                      )) \/1024\/1024` $labels.instance $labels.job | query | first | value | printf &quot;%.2f&quot; }}MB\/s\n\n  - alert: \u78c1\u76d8IO\u8bfb\u53d6\u541e\u5410\u8f83\u9ad8\n    expr: round(max(rate(node_disk_read_bytes_total[1m])) by (instance,job) \/ 1024 \/1024 \/140 * 100 ) &gt; 60\n    for: 1m\n    labels:\n      severity: critical\n    annotations:\n      summary: &quot;\u5b9e\u4f8b {{ $labels.instance }} \u78c1\u76d8IO\u8bfb\u53d6\u6bcf\u79d2\u8d85\u8fc7\u6700\u592784MB\/s&quot;\n      description:\n        \u76ee\u524d\u78c1\u76d8IO\u8bfb\u53d6\u541e\u5410\u91cf\u7684\u9971\u548c\u5ea6\u662f {{ $value }}%\u3002\n        \u76ee\u524d\u78c1\u76d8IO\u8bfb\u53d6\u541e\u5410\u91cf\u6bcf\u79d2\u6700\u5927\u662f {{ printf `max(rate(node_disk_read_bytes_total{instance=&quot;%s&quot;,job=&quot;%s&quot;}[1m]))                                                                       \/1024\/1024` $labels.instance $labels.job | query | first | value | printf &quot;%.2f&quot; }}MB\/s<\/code><\/pre>\n<ol start=\"2\">\n<li>\u68c0\u67e5\u544a\u8b66\u89c4\u5219\u6587\u4ef6<\/li>\n<\/ol>\n<p><img decoding=\"async\" src=\"\/wp-content\/uploads\/2024\/11\/image-1731129564829.png\" alt=\"file\" \/><\/p>\n<h3>\u7f51\u7edc\u76d1\u63a7\u6848\u4f8b\u5b9e\u8df5<\/h3>\n<h4>\u7f51\u7edc\u91cd\u70b9\u76d1\u63a7\u7684\u7ef4\u5ea6<\/h4>\n<p>\u5bf9\u4e8e\u7f51\u7edc\uff0c\u901a\u5e38\u4f7f\u7528<code>use<\/code>\u65b9\u6cd5\u76d1\u63a7\u5982\u4e0b\u51e0\u4e2a\u7ef4\u5ea6\uff1a<\/p>\n<ul>\n<li>1.\u7f51\u7edc\u4f20\u8f93\u901f\u7387\uff1a\u5373\u5f53\u524d\u7f51\u7edc\u6bcf\u79d2\u4f20\u8f93\u7684\u5e26\u5bbd<\/li>\n<li>2.\u7f51\u7edc\u9971\u548c\u5ea6\uff1a\u5bf9\u4e8e\u7f51\u7edc\u9971\u548c\u5ea6\uff0c\u901a\u5e38\u67e5\u770b\u7f51\u7edc\u662f\u5426\u4e22\u5305\u4e25\u91cd\uff0c\u5982\u679c\u6301\u7eed\u6709\u4e22\u5305\u60c5\u51b5\u5219\u4efb\u52a1\u76ee\u524d\u7f51\u7edc\u6bd4\u8f83\u9971\u548c<\/li>\n<li>3.\u7f51\u7edc\u9519\u8bef\u7387\uff1a\u53ef\u4ee5\u901a\u8fc7\u76d1\u63a7\u7f51\u7edc\u53d1\u9001\u548c\u63a5\u6536\u7684\u9519\u8bef\u6765\u83b7\u5f97<\/li>\n<\/ul>\n<h4>\u8ba1\u7b97\u7f51\u7edc\u4f20\u8f93\u901f\u7387<\/h4>\n<p>\u7f51\u7edc\u4f20\u8f93\u901f\u7387= <code>irate<\/code>\uff08\u7f51\u7edc\u63a5\u53e3\u53d1\u9001\/\u63a5\u6536\u7684\u603b\u6570\u636e\u91cf\uff09*8\/1024\/1024=<code>Mbps<\/code><\/p>\n<ol>\n<li>\u6a21\u62df\u5e26\u5bbd<\/li>\n<\/ol>\n<pre><code class=\"language-bash\"># 1\u3001\u5728\u4e24\u4e2a\u6d4b\u8bd5\u7684\u8282\u70b9\u4e0a\u5b89\u88c5\uff1ayum install iperf -y\n# 2\u3001\u670d\u52a1\u7aef\u8fd0\u2f8f\u5e76\u6307\u5b9a\u7aef\u2f1d\uff1aiperf -s -p 9999\n# 3\u3001\u5ba2\u6237\u7aef\u6a21\u62df\u5e26\u5bbd\u53d1\u9001\u547d\u4ee4\uff0c-b\u6307\u5b9a\u53d1\u9001\u2f24\u2f29\uff0c-t\u6307\u5b9a\u53d1\u9001\u6301\u7eed\u65f6\u2ed3\uff1aiperf -c &lt;ip&gt; -p 9999 -b 300M -t 60<\/code><\/pre>\n<ol start=\"2\">\n<li>\n<p>\u83b7\u53d6\u6700\u5927\u7684\u4e0b\u8f7d\u5e26\u5bbd\uff0c\u4e5f\u5c31\u662f\u6bcf\u79d2\u80fd\u63a5\u6536\u591a\u5c11<code>Mbps<\/code><br \/>\n<code>max(irate(node_network_receive_bytes_total{device!~&quot;ethA.*|ethB.*|lo|bond0&quot;}[1m])*8\/1024\/1024) by (instance,job,device)<\/code><\/p>\n<\/li>\n<li>\n<p>\u83b7\u53d6\u6700\u5927\u7684\u4e0a\u4f20\u5e26\u5bbd\uff0c\u4e5f\u5c31\u662f\u6bcf\u79d2\u80fd\u4f20\u8f93\u591a\u5c11<code>Mbps<\/code><br \/>\n<code>max(irate(node_network_transmit_bytes_total{device!~&quot;ethA.*|ethB.*|lo|bond0&quot;}[1m])*8\/1024\/1024) by (instance,job,device)<\/code><\/p>\n<\/li>\n<li>\n<p>\u5047\u8bbe\u516c\u53f8\u5e26\u5bbd\u662f<code>50Mbps<\/code>\uff0c\u6211\u5e0c\u671b\u8fbe\u5230<code>40Mbps<\/code>\u65f6\u5219\u89e6\u53d1\u544a\u8b66\uff0c\u8ba1\u7b97\u516c\u5f0f\uff1a\u7f51\u7edc\u6bcf\u79d2\u4f20\u8f93\u5e26\u5bbd\/\u7f51\u7edc\u6700\u5927\u5e26\u5bbd * 100 &gt;= 80<\/p>\n<\/li>\n<\/ol>\n<pre><code class=\"language-bash\"># \u4e0b\u8f7d\nmax(irate(node_network_receive_bytes_total[1m]) * 8 \/ 1024 \/ 1024) by (instance,job,device) \/ 50 * 100 &gt;= 80\n\n# \u4e0a\u4f20\nmax(irate(node_network_transmit_bytes_total[1m]) * 8 \/ 1024 \/ 1024) by (instance,job,device) \/ 50 * 100 &gt;= 80<\/code><\/pre>\n<h4>\u8ba1\u7b97\u7f51\u7edc\u8fde\u63a5\u6570\u4f7f\u7528\u7387<\/h4>\n<p>\u7f51\u7edc\u8fde\u63a5\u7387 = \u5f53\u524d\u6d3b\u8dc3<code>tcp<\/code>\u8fde\u63a5\u6570 \/ \u7cfb\u7edf\u5185\u6838\u6700\u5927\u652f\u6301\u7684<code>tcp<\/code>\u8fde\u63a5\u6570 * 100<\/p>\n<pre><code class=\"language-bash\">(node_nf_conntrack_entries \/ node_nf_conntrack_entries_limit) * 100\n\n# \u544a\u8b66\n(node_nf_conntrack_entries \/ node_nf_conntrack_entries_limit) * 100 &gt; 80<\/code><\/pre>\n<h4>\u914d\u7f6e\u7f51\u7edc\u544a\u8b66\u89c4\u5219<\/h4>\n<ol>\n<li>\u914d\u7f6e\u7f51\u7edc\u544a\u8b66\u89c4\u5219<\/li>\n<\/ol>\n<pre><code class=\"language-bash\">cat \/etc\/prometheus\/rules\/node_rules.yml\n- name: \u7f51\u7edc\u7edc\u544a\u8b66\u89c4\u5219\n  rules:\n  - alert: \u7f51\u7edc\u4e0b\u8f7d\u5e26\u5bbd\u5f02\u5e38\n    expr: max(irate(node_network_receive_bytes_total[1m]) * 8 \/ 1024 \/ 1024) by (instance,job,device) \/ 1000 * 100 &gt;= 80\n    for: 1m\n    labels:\n      severity: critical\n    annotations:\n      summary: &quot;\u5b9e\u4f8b {{ $labels.instance }} \u7684 {{ $labels.device }} \u63a5\u53e3\u4e0b\u8f7d\u6d41\u91cf\u5df2\u7ecf\u8d85\u8fc7800Mbps&quot;\n      description:\n        \u76ee\u524d\u4e0b\u8f7d\u5e26\u5bbd\u5df2\u7ecf\u8fbe\u5230 {{ printf `(irate(node_network_receive_bytes_total{instance=&quot;%s&quot;,job=&quot;%s&quot;,device=&quot;%s&quot;}[1m]) * 8 \/ 1024 \/ 1024)` $labels.instance $labels.job $labels.device | query | first | value | printf &quot;%.2f&quot;}} Mbps\/s\n        \u76ee\u524d\u4e0b\u8f7d\u5e26\u5bbd\u4f7f\u7528\u7387\u5728 {{ $value }}%\n\n  - alert: \u7f51\u7edc\u4e0a\u4f20\u5e26\u5bbd\u5f02\u5e38\n    expr: max(irate(node_network_transmit_bytes_total[1m]) * 8 \/ 1024 \/ 1024) by (instance,job,device) \/ 1000 * 100 &gt;= 80\n    for: 1m\n    labels:\n      severity: critical\n    annotations:\n      summary: &quot;\u5b9e\u4f8b {{ $labels.instance }} \u7684 {{ $labels.device }} \u63a5\u53e3\u4e0a\u4f20\u6d41\u91cf\u5df2\u7ecf\u8d85\u8fc7800Mbps&quot;\n      description:\n        \u76ee\u524d\u4e0a\u4f20\u5e26\u5bbd\u5df2\u7ecf\u8fbe\u5230 {{ printf `(irate(node_network_transmit_bytes_total{instance=&quot;%s&quot;,job=&quot;%s&quot;,device=&quot;%s&quot;}[1m]) * 8 \/ 1024 \/ 1024)` $labels.instance $labels.job $labels.device | query | first | value | printf &quot;%.2f&quot;}} Mbps\/s\n        \u76ee\u524d\u4e0a\u4f20\u5e26\u5bbd\u4f7f\u7528\u7387\u5728 {{ $value }}%\n\n  - alert: \u7f51\u7edcTCP\u8fde\u63a5\u6570\u5f02\u5e38\n    expr: node_nf_conntrack_entries \/ node_nf_conntrack_entries_limit * 100 &gt; 80\n    for: 1m\n    labels:\n      severity: critical\n    annotations:\n      summary: &quot;\u5b9e\u4f8b {{ $labels.instance }} \u7684 tcp\u8fde\u63a5\u6570\u8d85\u8fc780%&quot;\n      description:\n        \u76ee\u524dTCP\u8fde\u63a5\u6570\u662f {{ printf `node_nf_conntrack_entries{instance=&quot;%s&quot;,job=&quot;%s&quot;}` $labels.instance $labels.job | query | first | value | printf &quot;%.2f&quot; }}\n        \u76ee\u524dTCP\u8fde\u63a5\u4f7f\u7528\u7387\u662f {{ $value }}%<\/code><\/pre>\n<ol start=\"2\">\n<li><code>prometheus web<\/code>\u754c\u9762\u67e5\u770b\u544a\u8b66<\/li>\n<\/ol>\n<p><img decoding=\"async\" src=\"\/wp-content\/uploads\/2024\/11\/image-1731150963005.png\" alt=\"file\" \/><\/p>\n","protected":false},"excerpt":{"rendered":"<p>PromQL\u8d44\u6e90\u76d1\u63a7\u6848\u4f8b \u76d1\u63a7\u7cfb\u7edf\u8d44\u6e90\u65b9\u6cd5\u8bba 1.google\u7684\u56db\u4e2a\u9ec4\u91d1\u6307\u6807\u7740\u773c\u70b9\u5728\u670d\u52a1\u76d1\u63a7\uff0c\u8fd9\u56db\u4e2a\u6307\u6807\u5206\u522b\u662f&#8230; &raquo; <a class=\"read-more-link\" href=\"http:\/\/blue.yn.cn\/?p=1676\">\u9605\u8bfb\u5168\u6587<\/a><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[20],"tags":[],"class_list":["post-1676","post","type-post","status-publish","format-standard","hentry","category-prometheus"],"_links":{"self":[{"href":"http:\/\/blue.yn.cn\/index.php?rest_route=\/wp\/v2\/posts\/1676","targetHints":{"allow":["GET"]}}],"collection":[{"href":"http:\/\/blue.yn.cn\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/blue.yn.cn\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/blue.yn.cn\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/blue.yn.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=1676"}],"version-history":[{"count":9,"href":"http:\/\/blue.yn.cn\/index.php?rest_route=\/wp\/v2\/posts\/1676\/revisions"}],"predecessor-version":[{"id":1689,"href":"http:\/\/blue.yn.cn\/index.php?rest_route=\/wp\/v2\/posts\/1676\/revisions\/1689"}],"wp:attachment":[{"href":"http:\/\/blue.yn.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=1676"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/blue.yn.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=1676"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/blue.yn.cn\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=1676"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}