Alerts

alert: POD_CPU_IS_HIGH
expr: sum by(container, pod, namespace) (rate(container_cpu_usage_seconds_total{container!=""}[5m])) * 100 > 90
for: 1m
labels:
  severity: critical
annotations:
  description: Container {{ $labels.container }} CPU usage inside POD {{ $labels.pod}} is high in {{ $labels.namespace}}
  summary: POD {{ $labels.pod}} CPU Usage is high in {{ $labels.namespace}}

alert: POD_MEMORY_USAGE_IS_HIGH
expr: (sum by(container, pod, namespace) (container_memory_working_set_bytes{container!=""}) / sum by(container, pod, namespace) (container_spec_memory_limit_bytes > 0) * 100) > 80
for: 1m
labels:
  severity: critical
annotations:
  description: |-
    Container Memory usage is above 80%
      VALUE = {{ $value }}
      LABELS = {{ $labels }}
  summary: Container {{ $labels.container }} Memory usage inside POD {{ $labels.pod}} is high in {{ $labels.namespace}}

alert: NODE_CPU_IS_HIGH
expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
for: 1m
labels:
  severity: critical
annotations:
  description: node {{ $labels.kubernetes_node }} cpu is high
  summary: node cpu is greater than 80 precent

alert: NODE_DISK_SPACE_IS_LOW
expr: (100 * ((node_filesystem_avail_bytes{fstype!="rootfs",mountpoint="/"}) / (node_filesystem_size_bytes{fstype!="rootfs",mountpoint="/"}))) < 10
for: 1m
labels:
  severity: critical
annotations:
  description: node {{ $labels.node }} disk space is only {{ printf "%0.2f" $value }}% free.
  summary: node disk space remaining is less than 10 percent

alert: NODE_DOWN
expr: up{component="kubernetes-nodes"} == 0
for: 3m
labels:
  severity: warning
annotations:
  description: '{{ $labels.job }} job failed to scrape instance {{ $labels.instance }} for more than 3 minutes. Node Seems to be down'
  summary: Node {{ $labels.kubernetes_node }} is down

alert: NODE_MEMORY_LESS_THAN_10%
expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10
for: 1m
labels:
  severity: critical
annotations:
  description: node {{ $labels.kubernetes_node }} memory left is low
  summary: node memory left is lesser than 10 precent

alert: prometheus-job-down
expr: up{job="prometheus"} == 0
for: 1m
labels:
  severity: warning
annotations:
  description: Default Prometheus Job is Down LABELS = {{ $labels }}
  summary: The Default Prometheus Job is Down (job {{ $labels.job}})