Мониторинг ресурсов с помощю Prometheus + Grafana

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

# Sample config for Prometheus.

global:
  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
  # scrape_timeout is set to the global default (10s).

  # Attach these labels to any time series or alerts when communicating with
  # external systems (federation, remote storage, Alertmanager).
  external_labels:
      monitor: 'example'

# Alertmanager configuration
alerting:
  alertmanagers:
  - static_configs:
    - targets: ['localhost:9093']

# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
  # - "first_rules.yml"
  # - "second_rules.yml"

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: 'prometheus'

    # Override the global default and scrape targets from this job every 5 seconds.
    scrape_interval: 5s
    scrape_timeout: 5s

    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.

    static_configs:
      - targets: ['localhost:9090']

  - job_name: node
    # If prometheus-node-exporter is installed, grab stats about the local
    # machine by default.
    static_configs:
      - targets: ['remote_host_ip:9100']

  - job_name: process
    # Prometheus process exporter
    static_configs:
      - targets: ['remote_host_ip:9256']

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

# TLS and basic authentication configuration example.
#
# Additionally, a certificate and a key file are needed.
# tls_server_config:
#   cert_file: server.crt
#   key_file: server.key

http_server_config:
  # Enable HTTP/2 support. Note that HTTP/2 is only supported with TLS.
  # This can not be changed on the fly.
  # http2: false

# Usernames and passwords required to connect to Prometheus.
# Passwords are hashed with bcrypt: https://github.com/prometheus/exporter-toolkit/blob/master/docs/web-configuration.md#about-bcrypt
basic_auth_users:
  # password 'web'
  web: $2y$10$bIprcxc.jAZYHkGlVujX6OR891UNNyQ7/PZDGSTU9eyQcN5j3sw.y
  # password 'grafana'
  grafana: $2y$10$RpURPsp/cG2.gd4j2PPj/uxo.sm/D18.Dsi92Y8Lh3iwE4IebAfuG

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

version: '3.2'
services:
  prometheus:
    image: prom/prometheus:v2.30.3
    container_name: prometheus
    restart: always
    ports:
    - 9090:9090
    command:
    - --config.file=/etc/prometheus/prometheus.yml
    - --web.config.file=/etc/prometheus/web-config.yml
    - --storage.tsdb.path=/var/lib/prometheus/metrics2/
    volumes:
    - ./config/prometheus.yml:/etc/prometheus/prometheus.yml:ro
    - ./config/web-config.yml:/etc/prometheus/web-config.yml:ro
    - ./metrics:/var/lib/prometheus/metrics2:rw

  grafana:
    image: grafana/grafana:8.2.2
    container_name: grafana
    restart: always
    ports:
    - 3000:3000

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108

#!/bin/bash

install_path='/usr/bin/'

if [ $(whoami) != 'root' ]; then
  echo "The script must be run as root user"
  exit 0
fi

node_arch=$(ls node_exporter-*)
proc_arch=$(ls process-exporter-*)

if [ -z $node_arch ]; then
  echo "Node exporter archive not found"
  exit 0
fi

if [ -z $node_arch ]; then
  echo "Process exporter archive not found"
  exit 0
fi

#
# Install binaries
#
echo "Extract node exporter binary..."
tar --transform 's/.*\///' --transform 'flags=r;s/node_exporter/prometheus-node-exporter/' --show-transformed-names -zxvf $node_arch -C $install_path ${node_arch%.tar.gz}/node_exporter
error=$?

if [ $error -ne 0 ]; then
  echo "Error extracting node exporter binary"
  exit $error
fi

echo "Extract process exporter binary..."
tar --transform 's/.*\///' --transform 'flags=r;s/process-exporter/prometheus-process-exporter/' --show-transformed-names -zxvf $proc_arch -C $install_path ${proc_arch%.tar.gz}/process-exporter
error=$?

if [ $error -ne 0 ]; then
  echo "Error extracting process exporter binary"
  exit $error
fi

#
# Node exporter setup
#
echo "Create prometheus group..."
groupadd --system prometheus

echo "Create prometheus user..."
useradd --system -g prometheus -s /bin/false prometheus

echo "Make node exporter service configuration..."
cat > /etc/systemd/system/node-exporter.service << EOF
[Unit]
Description=Prometheus Node Exporter
After=network.target

[Service]
Type=simple
Restart=always
User=prometheus
Group=prometheus
ExecStart=${install_path}prometheus-node-exporter --web.listen-address=:9100

[Install]
WantedBy=multi-user.target
EOF

echo "Enable node exporter service..."
systemctl enable node-exporter.service

echo "Start node exporter service..."
systemctl start node-exporter.service

#
# Process exporter setup
#
echo "Make process exporter configuration..."
cat > /etc/process-exporter.yml << EOF
process_names:
  - name: "{{.Comm}}"
    cmdline:
    - '.+'
EOF

echo "Make process exporter service configuration..."
cat > /etc/systemd/system/process-exporter.service << EOF
[Unit]
Description= Prometheus Process Exporter
After=network.target

[Service]
User=root
Type=simple
ExecStart=${install_path}prometheus-process-exporter --config.path /etc/process-exporter.yml --web.listen-address=:9256
KillMode=process
Restart=always

[Install]
WantedBy=multi-user.target
EOF

echo "Enable process exporter service..."
systemctl enable process-exporter.service

echo "Start process exporter service..."
systemctl start process-exporter.service