ZooKeeper 监控
1.docker-compose.yaml配置文件增加metrics端口
environment: ZOO_CFG_EXTRA: 'metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider metricsProvider.httpPort=7000'2.rules/zookeeper.yaml文件
cd /data/prometheus/conf
cd /data/prometheus/confcat >rules/zookeeper.yaml <<\EOFgroups:- name: zookeeper rules: - alert: create too many znodes expr: znode_count > 1000000 for: 1m labels: severity: warning annotations: summary: "Instance {{ $labels.instance }} create too many znodes" description: "{{ $labels.instance }} of job {{$labels.job}} create too many znodes: [{{ $value }}]." - alert: create too many connections expr: num_alive_connections > 50 # suppose we use the default maxClientCnxns: 60 for: 1m labels: severity: warning annotations: summary: "Instance {{ $labels.instance }} create too many connections" description: "{{ $labels.instance }} of job {{$labels.job}} create too many connections: [{{ $value }}]." - alert: znode total occupied memory is too big expr: approximate_data_size /1024 /1024 > 1 * 1024 # more than 1024 MB(1 GB) for: 1m labels: severity: warning annotations: summary: "Instance {{ $labels.instance }} znode total occupied memory is too big" description: "{{ $labels.instance }} of job {{$labels.job}} znode total occupied memory is too big: [{{ $value }}] MB." - alert: set too many watch expr: watch_count > 10000 for: 1m labels: severity: warning annotations: summary: "Instance {{ $labels.instance }} set too many watch" description: "{{ $labels.instance }} of job {{$labels.job}} set too many watch: [{{ $value }}]." - alert: a leader election happens expr: increase(election_time_count[5m]) > 0 for: 1m labels: severity: warning annotations: summary: "Instance {{ $labels.instance }} a leader election happens" description: "{{ $labels.instance }} of job {{$labels.job}} a leader election happens: [{{ $value }}]." - alert: open too many files expr: open_file_descriptor_count > 300 for: 1m labels: severity: warning annotations: summary: "Instance {{ $labels.instance }} open too many files" description: "{{ $labels.instance }} of job {{$labels.job}} open too many files: [{{ $value }}]." - alert: fsync time is too long expr: rate(fsynctime_sum[1m]) > 100 for: 1m labels: severity: warning annotations: summary: "Instance {{ $labels.instance }} fsync time is too long" description: "{{ $labels.instance }} of job {{$labels.job}} fsync time is too long: [{{ $value }}]." - alert: take snapshot time is too long expr: rate(snapshottime_sum[5m]) > 100 for: 1m labels: severity: warning annotations: summary: "Instance {{ $labels.instance }} take snapshot time is too long" description: "{{ $labels.instance }} of job {{$labels.job}} take snapshot time is too long: [{{ $value }}]." - alert: avg latency is too high expr: avg_latency > 100 for: 1m labels: severity: warning annotations: summary: "Instance {{ $labels.instance }} avg latency is too high" description: "{{ $labels.instance }} of job {{$labels.job}} avg latency is too high: [{{ $value }}]." - alert: JvmMemoryFillingUp expr: jvm_memory_bytes_used / jvm_memory_bytes_max{area="heap"} > 0.8 for: 5m labels: severity: warning annotations: summary: "JVM memory filling up (instance {{ $labels.instance }})" description: "JVM memory is filling up (> 80%)\n labels: {{ $labels }} value = {{ $value }}\n"EOF3.prometheus配置
# 进入prometheus配置目录cd /data/prometheus/conf# 配置需要监控的ZooKeeper实例连接地址cat > targets/zookeeper/dev.yaml <<\EOF- labels: env: sz-prom-dev targets: - '192.168.102.232:7000'EOFcat > targets/zookeeper/sit.yaml <<\EOF- labels: env: sz-prom-sit targets: - '192.168.102.231:7000'EOFcat > targets/zookeeper/uat.yaml <<\EOF- labels: env: sz-prom-uat targets: - '192.168.102.237:7000'EOFcat > targets/zookeeper/staging.yaml <<\EOF- labels: env: sz-prom-staging targets: - '192.168.102.213:7000' - '192.168.102.214:7000' - '192.168.102.219:7000'EOF# 添加prometheus job配置cat >> prometheus.yml <<\EOF - job_name: zookeeper file_sd_configs: - files: - /etc/prometheus/targets/zookeeper/*.yamlEOF# 重载prometheus配置
curl -vv --basic -u 'admin:meng@2022' -X POST http://192.168.10.19:9090/-/reload
