Storage Server の常時稼働監視基盤: Prometheus・Loki・Promtail と Quadlet による構築記録

  nvme0n1 (1.8T)  → LVM → / (100G)
nvme1n1 (3.6T)  → LUKS → /mnt/data (archive, backups, object storage)
  

  /opt/containers/systemd/
├── rootful/
│   ├── node-exporter.container
│   ├── promtail.container
│   └── smartctl-exporter.container
└── rootless/
    ├── loki.container
    ├── minio.container
    ├── mktxp.container
    └── prometheus.container
  

  global:
  scrape_interval: 30s
  evaluation_interval: 30s

scrape_configs:
  - job_name: prometheus
    static_configs:
      - targets:
          - storage.home.arpa:9090

  - job_name: node-exporter
    static_configs:
      - targets:
          - storage.home.arpa:9100
          - desktop.home.arpa:9100
          - compute.home.arpa:9100

  - job_name: loki
    static_configs:
      - targets:
          - storage.home.arpa:3100

  - job_name: promtail
    static_configs:
      - targets:
          - storage.home.arpa:9080

  - job_name: smartctl
    scrape_interval: 5m
    static_configs:
      - targets:
          - storage.home.arpa:9633
          - compute.home.arpa:9633

  - job_name: mikrotik
    static_configs:
      - targets:
          - storage.home.arpa:49090

  - job_name: minio
    metrics_path: /minio/v2/metrics/cluster
    static_configs:
      - targets:
          - storage.home.arpa:9000
  

  # Mac (Homebrew)
brew install node_exporter
sudo brew services start node_exporter
# 外部 scrape 用に listen アドレスを変更
sudo sed -i '' 's/--web.listen-address=127\.0\.0\.1:9100/--web.listen-address=0.0.0.0:9100/' \
  /Library/LaunchDaemons/homebrew.mxcl.node_exporter.plist
  

  server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /promtail/positions.yaml

clients:
  - url: http://storage.home.arpa:3100/loki/api/v1/push

scrape_configs:
  # --- Linux ファイルログ ---
  - job_name: security-files
    static_configs:
      - labels:
          job: security
          host: storage.home.arpa
          __path__: /var/log/alternatives.log*
      - labels:
          job: security
          host: storage.home.arpa
          __path__: /var/log/apport.log*

  - job_name: apt-files
    static_configs:
      - labels:
          job: apt
          host: storage.home.arpa
          __path__: /var/log/apt/history.log
      - labels:
          job: apt
          host: storage.home.arpa
          __path__: /var/log/apt/term.log
      - labels:
          job: apt
          host: storage.home.arpa
          __path__: /var/log/dpkg.log

  # --- systemd journal ---
  - job_name: kernel-journal
    journal:
      path: /var/log/journal
      max_age: 168h
      labels:
        job: kernel
        host: storage.home.arpa
    relabel_configs:
      - source_labels: ["__journal__hostname"]
        target_label: host
      - source_labels: ["__journal__syslog_identifier"]
        target_label: ident
      - source_labels: ["__journal_priority"]
        target_label: priority

  - job_name: systemd-units
    journal:
      path: /var/log/journal
      max_age: 168h
      labels:
        job: systemd
        host: storage.home.arpa
    relabel_configs:
      - source_labels: ["__journal__systemd_unit"]
        target_label: unit
      - source_labels: ["__journal__hostname"]
        target_label: host
      - source_labels: ["__journal__uid"]
        target_label: uid

  # --- ネットワーク機器 syslog (RouterOS / Netgear WiFi) ---
  - job_name: network-syslog
    syslog:
      listen_address: 0.0.0.0:1514
      listen_protocol: udp
      syslog_format: rfc3164
      use_incoming_timestamp: false
      label_structured_data: true
    relabel_configs:
      - source_labels: ["__syslog_message_hostname"]
        target_label: host
      - source_labels: ["__syslog_message_app_name"]
        target_label: app
      - source_labels: ["__syslog_severity"]
        target_label: severity
      - source_labels: ["__syslog_facility"]
        target_label: facility
  

  /system logging action add name=promtail target=remote remote=10.10.10.3 remote-port=1514 bsd-syslog=yes
/system logging add topics=firewall action=promtail
/system logging add topics=system,info action=promtail
  

  /ip firewall filter set [find builtin=no && dynamic=no] log=yes log-prefix="FW: "
  

  [edge.home.arpa]
hostname = edge.home.arpa
username = metrics
password = (API user password)
port = 8728
use_ssl = False
verify_ssl = False
timeout = 5
allow_duplicates = False
  

  {job="network-syslog"} | line_format "{{.time}} | {{.host}} | {{.severity}} | {{.line}}"