kube-state-metrics Agent Skills & AI Rules

rule_files:
  - ../../examples/prometheus-alerting-rules/alerts.yaml

evaluation_interval: 1m

tests:
  # Test long-lasting invalid configuration should trigger alert
  - interval: 1m
    input_series:
      # T=0: 3 shards with correct configuration
      # T=9m: shard #0 was incorrectly redeployed with --total-shards=1
      - series: 'kube_state_metrics_total_shards{job="kube-state-metrics",pod="kube-state-metrics-shard-0-0"}'
        values: '3x9 1x21'
      - series: 'kube_state_metrics_total_shards{job="kube-state-metrics",pod="kube-state-metrics-shard-1-0"}'
        values: '3x30'
      - series: 'kube_state_metrics_total_shards{job="kube-state-metrics",pod="kube-state-metrics-shard-2-0"}'
        values: '3x30'
    alert_rule_test:
      - eval_time: 25m
        alertname: KubeStateMetricsShardingMismatch
        exp_alerts:
          - exp_labels:
              severity: critical
            exp_annotations:
              summary: "kube-state-metrics sharding is misconfigured."
              description: "kube-state-metrics pods are running with different --total-shards configuration, some Kubernetes objects may be exposed multiple times or not exposed at all."

  # Test deployment from one shard to multiple shards should not trigger alert
  - interval: 1m
    input_series:
      # T=0: 1 shard with correct configuration
      # T=9m: two more shards were deployed, all shards were correctly configured
      - series: 'kube_state_metrics_total_shards{job="kube-state-metrics",pod="kube-state-metrics-shard-0-0"}'
        values: '1x9 3x21'
      - series: 'kube_state_metrics_total_shards{job="kube-state-metrics",pod="kube-state-metrics-shard-1-0"}'
        values: '_x9 3x21'
      - series: 'kube_state_metrics_total_shards{job="kube-state-metrics",pod="kube-state-metrics-shard-2-0"}'
        values: '_x9 3x21'
      - series: 'kube_state_metrics_shard_ordinal{job="kube-state-metrics",pod="kube-state-metrics-shard-0-0",shard_ordinal="0"}'
        values: '0x30'
      - series: 'kube_state_metrics_shard_ordinal{job="kube-state-metrics",pod="kube-state-metrics-shard-1-0",shard_ordinal="1"}'
        values: '_x9 1x21'
      - series: 'kube_state_metrics_shard_ordinal{job="kube-state-metrics",pod="kube-state-metrics-shard-2-0",shard_ordinal="2"}'
        values: '_x9 2x21'
    alert_rule_test: []

  # Test misconfigured single shard deployment
  - interval: 1m
    input_series:
      - series: 'kube_state_metrics_total_shards{job="kube-state-metrics",pod="kube-state-metrics-shard-0-0"}'
        values: '3x30'
      - series: 'kube_state_metrics_shard_ordinal{job="kube-state-metrics",pod="kube-state-metrics-shard-0-0",shard_ordinal="0"}'
        values: '0x30'
    alert_rule_test:
      - eval_time: 25m
        alertname: KubeStateMetricsShardsMissing
        exp_alerts:
          - exp_labels:
              severity: critical
            exp_annotations:
              summary: "kube-state-metrics shards are missing."
              description: "kube-state-metrics shards are missing, some Kubernetes objects are not being exposed."

  # Test misconfigured single shard deployment
  - interval: 1m
    input_series:
      - series: 'kube_state_metrics_total_shards{job="kube-state-metrics",pod="kube-state-metrics-shard-0-0"}'
        values: '3x30'
      - series: 'kube_state_metrics_shard_ordinal{job="kube-state-metrics",pod="kube-state-metrics-shard-0-0",shard_ordinal="0"}'
        values: '0x30'
    alert_rule_test:
      - eval_time: 25m
        alertname: KubeStateMetricsShardsMissing
        exp_alerts:
          - exp_labels:
              severity: critical
            exp_annotations:
              summary: "kube-state-metrics shards are missing."
              description: "kube-state-metrics shards are missing, some Kubernetes objects are not being exposed."
    promql_expr_test:
      # Test that missing shards are #2 (0b100) and #1 (0b010)
      - expr: |
          2^max(kube_state_metrics_total_shards{job="kube-state-metrics"}) - 1
            -
          sum( 2 ^ max by (shard_ordinal) (kube_state_metrics_shard_ordinal{job="kube-state-metrics"}) )
        eval_time: 25m
        exp_samples:
          - value: 0b110