通过Zabbix去监控K8S集群

一、环境准备和说明

首先你确保已经部署好了一套K8S集群，然后我们需要在zabbix-server端安装部署python3，默认Cent OS系统中使用的python2，在监控服务端节点上安装python3，如果还没有部署，那么请参考下面的文章进行部署。

[root@zabbix-server ~]# wget https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz

[root@zabbix-server ~]# yum -y install zlib zlib-devel bzip2 bzip2-devel ncurses ncurses-devel readline readline-devel openssl openssl-devel openssl-static xz lzma xz-devel sqlite sqlite-devel gdbm gdbm-devel tk tk-devel libffi libffi-devel gcc make glibc-kernheaders

[root@zabbix-server ~]# mkdir /usr/local/python3

#解压至当前目录下即可
[root@zabbix-server ~]# tar -zxvf Python-3.7.4.tgz

#编译安装
[root@zabbix-server ~]# cd Python-3.7.4
[root@zabbix-server Python-3.7.4]# ./configure --prefix=/usr/local/python3 --with-ssl --enable-shared CFLAGS=-fPIC
[root@zabbix-server Python-3.7.4]# make && make install

[root@zabbix-server ~]# ln -sv /usr/local/python3/bin/python3.7 /usr/bin/python3
[root@zabbix-server ~]# ln -sv /usr/local/python3/bin/pip3.7 /usr/bin/pip3

[root@zabbix-server ~]# find / -name libpython3.7m.so.1.0
/root/Python-3.7.4/libpython3.7m.so.1.0
/usr/local/python3/lib/libpython3.7m.so.1.0

[root@zabbix-server ~]# echo "/usr/local/python3/lib " > /etc/ld.so.conf.d/python3.7.conf
[root@zabbix-server ~]# ldconfig
[root@zabbix-server ~]# ldd /usr/local/python3/bin/python3.7

#安装yum源,zabbix服务端一般都会安装了yum源，如果没有的则需要执行
[root@zabbix-server ~]# rpm -Uvh https://repo.zabbix.com/zabbix/5.0/rhel/7/x86_64/zabbix-release-5.0-1.el7.noarch.rpm

#更新yum仓库
[root@zabbix-server ~]# yum repolist

#安装zabbix-sender
[root@zabbix-server ~]# yum -y install zabbix-sender

#检测zabbix-sender命令可用
[root@zabbix-server ~]# zabbix_sender -V
zabbix_sender (Zabbix) 5.0.7

二、获取k8s的api地址

• 查看K8S的api地址的命令如下，在k8s的master节点执行
• 注意获取到的地址确保外部可以访问

[root@k8s-master1 ~]# kubectl cluster-info

三、Master节点获取token

一般我们在部署K8S集群中，因为K8S安全性要求很高的，我们都是通过HTTPS的方式去访问的，同时我们也会创建很多的访问的token进行授权访问，这里我们就不再进行创建token了，我这里直接使用之前部署K8S集群安装Dashboard时生成的token用作zabbix获取数据的凭证。

注意：你也可以单独创建一个token用来供zabbix去采集数据。

[root@k8s-master1 ~]# kubectl describe secret -n kube-system $(kubectl get secret  -n kube-system | grep dashboard-admin-user | awk '{print $1}')

• 注意：如果token权限不对，那么后面执行脚本去采集信息会出现下图的报错信息：

四、创建脚本并修改脚本

• 在zabbix-server端创建

[root@zabbix-server ~]# cd /etc/zabbix/zabbix_agentd.d/
[root@zabbix-server zabbix_agentd.d]# mkdir k8s
[root@zabbix-server zabbix_agentd.d]# cd k8s/
[root@zabbix-server k8s]# vim get_k8s.py
#!/usr/bin/python3
import json
import datetime
import sys
import os
import math
import urllib3
import requests
import re
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

zabbix_sender = "/usr/bin/zabbix_sender"
zabbix_server = "zabbix服务端IP地址"
clientname = "k8s_master"


#k8s的api地址
url = "https://xx.xx.xx.xx.:6443"

#k8s的token
token = "token黏贴至此"

def get_result(api_name):
   headers = {"Authorization":"Bearer "+token}
   json_data = requests.get(url+api_name,headers=headers,verify=False)
   return json_data.json()

#时间转换,把所有服务的创建时间变成运行时间
def Trantime(time_data):
    time = re.match(r"(\d+)-(\d+)-(\d+).*?(\d+):(\d+):(\d+)",time_data)
    total_send = round((datetime.datetime.now()-datetime.datetime(int(time.group(1)),int(time.group(2)),int(time.group(3)),int(time.group(4)),int(time.group(5)),int(time.group(6)))).total_seconds())
    return str(math.floor(total_send/86400)).split(".")[0]+"d"+str(math.floor((total_send%86400)/3600)).split(".")[0]+"h"   

#把所有的单位转换成b
def Tranunit(unit_data):
    unit = re.match(r"(\d+)(.*)",unit_data)
    value = int(unit.group(1))
    if unit.group(2) == "K":
       return value*1000
    elif unit.group(2) == "Ki":
       return value*1024
    elif unit.group(2) == "M":
       return value*1000*1000
    elif unit.group(2) == "Mi":
       return value*1024*1024
    elif unit.group(2) == "G":
       return value*1000*1000*1000
    elif unit.group(2) == "Gi":
        return value*1024*1024*1024
    elif unit.group(2) == "n":
        return math.ceil(value/1000/1000)
    elif unit.group(2) == "m":
        return value
    else:
        return value

#对数据进行zabbix_sender发送
def send_data(data,dis_key,key):
    result = json.dumps({"data":data},sort_keys=True,ensure_ascii=False)
    cmd = "{0} -z {1} -s {2} -k {3} -o '{4}'>/dev/null".format(zabbix_sender,zabbix_server,clientname,dis_key,result)
    os.system(cmd)
    for i in data:
        for value in i.keys():
         if value !="{#NAME}":
           cmd = "{0} -z {1} -s {2} -k {3}[{4}.{5}] -o '{6}'>/dev/null".format(zabbix_sender,zabbix_server,clientname,key,i["{#NAME}"],i[value][0],i[value][1])
           os.system(cmd)
           print(cmd)     
#获取所有k8s节点的信息
def get_node():
    get_nodes = []
    node_result = get_result("/api/v1/nodes")
    node_use_result = get_result("/apis/metrics.k8s.io/v1beta1/nodes")
    for j in node_use_result.get("items"):
          for i in node_result.get("items"):
            if i.get("metadata").get("name") == j.get("metadata").get("name"):
              data={"{#NAME}": i.get("metadata").get("name"),
                               "{#STATUS}": ["status",i.get("status").get("conditions")[-1].get("type") if i.get("status").get("conditions")[-1].get("status") == "True" else "NotReady"],
                                "{#IP}": ["ip",i.get("status").get("addresses")[0].get("address")],
                                "{#KUBELET_VERSION}": ["version",i.get("status").get("nodeInfo").get("kubeletVersion")],
                                "{#OS_IMAGE}": ["os_image",i.get("status").get("nodeInfo").get("osImage")],
                                 "{#CPU}": ["cpu",i.get("status").get("capacity").get("cpu")],
                                 "{#MEMORY}": ["memory",Tranunit(i.get("status").get("capacity").get("memory"))],
                                 "{#LIMIT_STORAGE}": ["storage",Tranunit(i.get("status").get("capacity").get("ephemeral-storage"))],
                                 "{#RUNTIME}":["runtime",Trantime(i.get("metadata").get("creationTimestamp"))],
                                 "{#USECPU}":["usecpu",Tranunit(j.get("usage").get("cpu"))],
                                 "{#USEMEMORY}":["usememory",Tranunit(j.get("usage").get("memory"))]
                                 }
       
          get_nodes.append(data)
    send_data(get_nodes,"getnode","node")

#获取k8s组件的健康信息
def get_health():
    get_healths = []
    health_result = get_result("/api/v1/componentstatuses") 
    for i in health_result.get("items"):
        data = {} 
        data = {"{#NAME}": i.get("metadata").get("name"),
                "{#STATUS}": ["status",i.get("conditions")[0].get("type")],
                "{#MESSAGE}":["message",i.get("conditions")[0].get("message")]                 
                }
        get_healths.append(data)
    send_data(get_healths,"gethealth","health")

#获取k8s的所有pod
def get_pod(): 
    get_pods = []
    pod_result = get_result("/api/v1/pods")
    for i in pod_result.get("items"):
           data = {"{#NAME}":i.get("metadata").get("name"),
                   "{#RUNTIME}":["runtime",Trantime(i.get("metadata").get("creationTimestamp"))],  
                   "{#STATUS}":["status",i.get("status").get("phase")],
                   "{#RESTARTCOUNT}":["restartcount",i.get("status").get("containerStatuses")[0].get("restartCount")]
                      }
            
           get_pods.append(data)
    send_data(get_pods,"getpod","pod")
        
if __name__ == "__main__":
   cmd = "{0}()".format(sys.argv[1])
   eval(cmd)
   print("采集完成")

#安装python所需模块
[root@zabbix-server k8s]# pip3 install urllib3
[root@zabbix-server k8s]# pip3 install requests
[root@zabbix-server k8s]# pip3 install eval

#为脚本添加执行权限
[root@zabbix-server k8s]# chmod +x get_k8s.py

#执行脚本，检查脚本是否可以正常采集到数据
[r[root@zabbix-server k8s]# ./get_k8s.py get_pod

[root@zabbix-server ~]# cd /usr/lib/zabbix/externalscripts/
[root@zabbix-server externalscripts]# vim get_k8s
#!/usr/bin/bash
/usr/bin/python3 /etc/zabbix/zabbix_agentd.d/k8s/get_k8s.py get_pod
/usr/bin/python3 /etc/zabbix/zabbix_agentd.d/k8s/get_k8s.py get_health
/usr/bin/python3 /etc/zabbix/zabbix_agentd.d/k8s/get_k8s.py get_node

五、创建主机并套用脚本

• 注意：主机名称必须与上面采集数据py脚本中定义的client文章来源(Source)：浅时光博客name的值保文章来源(Source)：浅时光博客持一致

• 将K8S监控的zabbix模板文件上传至zabbix服务端，通过web界面导入即可

抱歉！为防机器采集，此处内容已被隐藏，请输入验证码查看内容

验证码：

请关注“浅时光博客”官方微信公众号，回复关键字“5384”，获取验证码。

【注】用手机微信扫描右侧二维码都可以关注“浅时光博客”官方微信公众号。

六、查看获取到的数据

七、查看监控告警信息

• 如果我们的集群中有Pod状态异常或者Pod重启等，都会触发告警
• 下面为触发告警的展示样式

参考文章

https://zhuanlan.zhihu.com/p/149911655