node-test:
  hosts:
    10.10.10.11: { nodename: node-test-1 }
    10.10.10.12: { nodename: node-test-2 }
    10.10.10.13: { nodename: node-test-3 }
  vars:
    node_cluster: node-test

#nodename:                # [实例] # 节点实例标识，如缺失则使用现有主机名，可选，无默认值
node_cluster: nodes       # [集群] # 节点集群标识，如缺失则使用默认值'nodes'，可选
nodename_overwrite: true          # 用 nodename 覆盖节点的主机名吗？
nodename_exchange: false          # 在剧本主机之间交换 nodename 吗？
node_id_from_pg: true             # 如果可行，是否借用 postgres 身份作为节点身份？

node-test:
  hosts:
    10.10.10.11: { nodename: node-test-1 }
    10.10.10.12: { nodename: node-test-2 }
    10.10.10.13: { nodename: node-test-3 }
  vars:
    node_cluster: node-test

node_load1{cls="pg-meta", ins="pg-meta-1", ip="10.10.10.10", job="nodes"}
node_load1{cls="pg-test", ins="pg-test-1", ip="10.10.10.11", job="nodes"}
node_load1{cls="pg-test", ins="pg-test-2", ip="10.10.10.12", job="nodes"}
node_load1{cls="pg-test", ins="pg-test-3", ip="10.10.10.13", job="nodes"}

#nodename:                # [实例] # 节点实例标识，如缺失则使用现有主机名，可选，无默认值
node_cluster: nodes       # [集群] # 节点集群标识，如缺失则使用默认值'nodes'，可选
nodename_overwrite: true          # 用 nodename 覆盖节点的主机名吗？
nodename_exchange: false          # 在剧本主机之间交换 nodename 吗？
node_id_from_pg: true             # 如果可行，是否借用 postgres 身份作为节点身份？

node_write_etc_hosts: true        # modify `/etc/hosts` on target node?
node_default_etc_hosts:           # static dns records in `/etc/hosts`
  - "${admin_ip} h.pigsty a.pigsty p.pigsty g.pigsty"
node_etc_hosts: []                # extra static dns records in `/etc/hosts`
node_dns_method: add              # how to handle dns servers: add,none,overwrite
node_dns_servers: ['${admin_ip}'] # dynamic nameserver in `/etc/resolv.conf`
node_dns_options:                 # dns resolv options in `/etc/resolv.conf`
  - options single-request-reopen timeout:1

["${admin_ip} h.pigsty a.pigsty p.pigsty g.pigsty"]

- "options single-request-reopen timeout:1"

node_repo_modules: local          # upstream repo to be added on node, local by default.
node_repo_remove: true            # remove existing repo on node?
node_packages: [ ]                # packages to be installed current nodes
#node_default_packages:           # default packages to be installed on all nodes

- lz4,unzip,bzip2,pv,jq,git,ncdu,make,patch,bash,lsof,wget,uuid,tuned,nvme-cli,numactl,sysstat,iotop,htop,rsync,tcpdump
- python3,python3-pip,socat,lrzsz,net-tools,ipvsadm,telnet,ca-certificates,openssl,keepalived,etcd,haproxy,chrony
- zlib,yum,audit,bind-utils,readline,vim-minimal,node_exporter,grubby,openssh-server,openssh-clients

- lz4,unzip,bzip2,pv,jq,git,ncdu,make,patch,bash,lsof,wget,uuid,tuned,nvme-cli,numactl,sysstat,iotop,htop,rsync,tcpdump
- python3,python3-pip,socat,lrzsz,net-tools,ipvsadm,telnet,ca-certificates,openssl,keepalived,etcd,haproxy,chrony
- zlib1g,acl,dnsutils,libreadline-dev,vim-tiny,node-exporter,openssh-server,openssh-client

node_disable_firewall: true       # disable node firewall? true by default
node_disable_selinux: true        # disable node selinux? true by default
node_disable_numa: false          # disable node numa, reboot required
node_disable_swap: false          # disable node swap, use with caution
node_static_network: true         # preserve dns resolver settings after reboot
node_disk_prefetch: false         # setup disk prefetch on HDD to increase performance
node_kernel_modules: [ softdog, br_netfilter, ip_vs, ip_vs_rr, ip_vs_wrr, ip_vs_sh ]
node_hugepage_count: 0            # number of 2MB hugepage, take precedence over ratio
node_hugepage_ratio: 0            # node mem hugepage ratio, 0 disable it by default
node_overcommit_ratio: 0          # node mem overcommit ratio, 0 disable it by default
node_tune: oltp                   # node tuned profile: none,oltp,olap,crit,tiny
node_sysctl_params: { }           # sysctl parameters in k:v format in addition to tuned

node_kernel_modules: [ softdog, br_netfilter, ip_vs, ip_vs_rr, ip_vs_wrr, ip_vs_sh ]

node_data: /data                  # node main data directory, `/data` by default
node_admin_enabled: true          # create a admin user on target node?
node_admin_uid: 88                # uid and gid for node admin user
node_admin_username: dba          # name of node admin user, `dba` by default
node_admin_ssh_exchange: true     # exchange admin ssh key among node cluster
node_admin_pk_current: true       # add current user's ssh pk to admin authorized_keys
node_admin_pk_list: []            # ssh public keys to be added to admin user
node_aliases: {}                  # extra shell aliases to be added, k:v dict

node_aliases:
  g:   git
  d:   docker

alias g="git"
alias d="docker"

node_timezone: ''                 # 设置节点时区，空字符串表示跳过
node_ntp_enabled: true            # 启用chronyd时间同步服务？
node_ntp_servers:                 # `/etc/chrony.conf`中的ntp服务器
  - pool pool.ntp.org iburst
node_crontab_overwrite: true      # 覆盖还是追加到`/etc/crontab`？
node_crontab: [ ]                 # `/etc/crontab`中的crontab条目

node_ntp_servers: [ 'pool ${admin_ip} iburst' ]

node_crontab: 
  - '00 01 * * * postgres /pg/bin/pg-backup full' ] # make a full backup every 1am

vip_enabled: false                # enable vip on this node cluster?
# vip_address:         [IDENTITY] # node vip address in ipv4 format, required if vip is enabled
# vip_vrid:            [IDENTITY] # required, integer, 1-254, should be unique among same VLAN
vip_role: backup                  # optional, `master/backup`, backup by default, use as init role
vip_preempt: false                # optional, `true/false`, false by default, enable vip preemption
vip_interface: eth0               # node vip network interface to listen, `eth0` by default
vip_dns_suffix: ''                # node vip dns name suffix, empty string by default
vip_exporter_port: 9650           # keepalived exporter listen port, 9650 by default

haproxy_enabled: true             # 在此节点上启用haproxy？
haproxy_clean: false              # 清理所有现有的haproxy配置？
haproxy_reload: true              # 配置后重新加载haproxy？
haproxy_auth_enabled: true        # 为haproxy管理页面启用身份验证
haproxy_admin_username: admin     # haproxy管理用户名，默认为`admin`
haproxy_admin_password: pigsty    # haproxy管理密码，默认为`pigsty`
haproxy_exporter_port: 9101       # haproxy管理/导出端口，默认为9101
haproxy_client_timeout: 24h       # 客户端连接超时，默认为24小时
haproxy_server_timeout: 24h       # 服务器端连接超时，默认为24小时
haproxy_services: []              # 需要在节点上暴露的haproxy服务列表

haproxy_services:                   # list of haproxy service

  # expose pg-test read only replicas
  - name: pg-test-ro                # [REQUIRED] service name, unique
    port: 5440                      # [REQUIRED] service port, unique
    ip: "*"                         # [OPTIONAL] service listen addr, "*" by default
    protocol: tcp                   # [OPTIONAL] service protocol, 'tcp' by default
    balance: leastconn              # [OPTIONAL] load balance algorithm, roundrobin by default (or leastconn)
    maxconn: 20000                  # [OPTIONAL] max allowed front-end connection, 20000 by default
    default: 'inter 3s fastinter 1s downinter 5s rise 3 fall 3 on-marked-down shutdown-sessions slowstart 30s maxconn 3000 maxqueue 128 weight 100'
    options:
      - option httpchk
      - option http-keep-alive
      - http-check send meth OPTIONS uri /read-only
      - http-check expect status 200
    servers:
      - { name: pg-test-1 ,ip: 10.10.10.11 , port: 5432 , options: check port 8008 , backup: true }
      - { name: pg-test-2 ,ip: 10.10.10.12 , port: 5432 , options: check port 8008 }
      - { name: pg-test-3 ,ip: 10.10.10.13 , port: 5432 , options: check port 8008 }

node_exporter_enabled: true       # setup node_exporter on this node?
node_exporter_port: 9100          # node exporter listen port, 9100 by default
node_exporter_options: '--no-collector.softnet --no-collector.nvme --collector.tcpstat --collector.processes'

promtail_enabled: true            # enable promtail logging collector?
promtail_clean: false             # purge existing promtail status file during init?
promtail_port: 9080               # promtail listen port, 9080 by default
promtail_positions: /var/log/positions.yaml # promtail position status file path

node-id       ：生成节点身份标识
node_name     ：设置主机名
node_hosts    ：配置 /etc/hosts 记录
node_resolv   ：配置 DNS 解析器 /etc/resolv.conf
node_firewall ：设置防火墙 & selinux
node_ca       ：添加并信任CA证书
node_repo     ：添加上游软件仓库
node_pkg      ：安装 rpm/deb 软件包
node_feature  ：配置 numa、grub、静态网络等特性
node_kernel   ：配置操作系统内核模块
node_tune     ：配置 tuned 调优模板
node_sysctl   ：设置额外的 sysctl 参数
node_profile  ：写入 /etc/profile.d/node.sh
node_ulimit   ：配置资源限制
node_data     ：配置数据目录
node_admin    ：配置管理员用户和ssh密钥
node_timezone ：配置时区
node_ntp      ：配置 NTP 服务器/客户端
node_crontab  ：添加/覆盖 crontab 定时任务
node_vip      ：为节点集群设置可选的 L2 VIP
haproxy       ：在节点上设置 haproxy 以暴露服务
monitor       ：配置节点监控：node_exporter & promtail

register       : 从 prometheus & nginx 中移除节点注册信息
  - prometheus : 移除已注册的 prometheus 监控目标
  - nginx      : 移除用于 haproxy 管理界面的 nginx 代理记录
vip            : 移除节点的 keepalived 与 L2 VIP（如果启用 VIP）
haproxy        : 移除 haproxy 负载均衡器
node_exporter  : 移除节点监控：Node Exporter
vip_exporter   : 移除 keepalived_exporter （如果启用 VIP）
promtail       : 移除 loki 日志代理 promtail
profile        : 移除 /etc/profile.d/node.sh 环境配置文件

# ./node.yml -l <cls|ip|group>        # 向 Pigsty 中添加节点的实际剧本
# bin/node-add <selector|ip...>       # 向 Pigsty 中添加节点
bin/node-add node-test                # 初始化节点集群 'node-test'
bin/node-add 10.10.10.10              # 初始化节点  '10.10.10.10'

# ./node-rm.yml -l <cls|ip|group>    # 从 pigsty 中移除节点的实际剧本
# bin/node-rm <cls|ip|selector> ...  # 从 pigsty 中移除节点
bin/node-rm node-test                # 移除节点集群 'node-test'
bin/node-rm 10.10.10.10              # 移除节点 '10.10.10.10'

node.yml -t node_admin -k -K -e ansible_user=<另一个管理员>   # 为另一个管理员输入 ssh/sudo 密码以完成此任务

proxy:
  hosts:
    10.10.10.29: { nodename: proxy-1 } # 您可以显式指定初始的 VIP 角色：MASTER / BACKUP
    10.10.10.30: { nodename: proxy-2 } # , vip_role: master }
  vars:
    node_cluster: proxy
    vip_enabled: true
    vip_vrid: 128
    vip_address: 10.10.10.99
    vip_interface: eth1

./node.yml -l proxy -t node_vip     # 首次启用 VIP 
./node.yml -l proxy -t vip_refresh  # 刷新 vip 配置（例如指定 master）

# 在节点上安装监控软件：node_exporter, promtail，分别收集指标，收集日志。
./node.yml -t node_repo,node_pkg -e '{"node_packages_default":[],"node_packages":["node_exporter", "promtail"]}'
./node.yml -t node_exporter,node_register  # 配置 node_exporter 监控组件，并将其注册到 Prometheus 中
./node.yml -t promtail                     # 如果你需要收集节点日志，额外执行此任务即可

# Play
./node.yml -t node                            # 完成节点主体初始化（haproxy，监控除外）
./node.yml -t haproxy                         # 在节点上设置 haproxy
./node.yml -t monitor                         # 配置节点监控：node_exporter & promtail （以及可选的 keepalived_exporter）
./node.yml -t node_vip                        # 为没启用过 VIP 的集群安装、配置、启用L2 VIP
./node.yml -t vip_config,vip_reload           # 刷新节点L2 VIP配置
./node.yml -t haproxy_config,haproxy_reload   # 刷新节点上的服务定义
./node.yml -t register_prometheus             # 重新将节点注册到 Prometheus 中
./node.yml -t register_nginx                  # 重新将节点 haproxy 管控界面注册到 Nginx 中

# Task
./node.yml -t node-id        # 生成节点身份标识
./node.yml -t node_name      # 设置主机名
./node.yml -t node_hosts     # 配置节点 /etc/hosts 记录
./node.yml -t node_resolv    # 配置节点 DNS 解析器 /etc/resolv.conf
./node.yml -t node_firewall  # 配置防火墙 & selinux
./node.yml -t node_ca        # 配置节点的CA证书
./node.yml -t node_repo      # 配置节点上游软件仓库
./node.yml -t node_pkg       # 在节点上安装 yum 软件包
./node.yml -t node_feature   # 配置 numa、grub、静态网络等特性
./node.yml -t node_kernel    # 配置操作系统内核模块
./node.yml -t node_tune      # 配置 tuned 调优模板
./node.yml -t node_sysctl    # 设置额外的 sysctl 参数
./node.yml -t node_profile   # 配置节点环境变量：/etc/profile.d/node.sh
./node.yml -t node_ulimit    # 配置节点资源限制
./node.yml -t node_data      # 配置节点首要数据目录
./node.yml -t node_admin     # 配置管理员用户和ssh密钥
./node.yml -t node_timezone  # 配置节点时区
./node.yml -t node_ntp       # 配置节点 NTP 服务器/客户端
./node.yml -t node_crontab   # 添加/覆盖 crontab 定时任务
./node.yml -t node_vip       # 为节点集群设置可选的 L2 VIP

################################################################
#                         Node Alert                           #
################################################################
- name: node-alert
  rules:

    #==============================================================#
    #                          Aliveness                           #
    #==============================================================#
    # node exporter is dead indicate node is down
    - alert: NodeDown
      expr: node_up < 1
      for: 1m
      labels: { level: 0, severity: CRIT, category: node }
      annotations:
        summary: "CRIT NodeDown {{ $labels.ins }}@{{ $labels.instance }}"
        description: |
          node_up[ins={{ $labels.ins }}, instance={{ $labels.instance }}] = {{ $value }} < 1
          http://g.pigsty/d/node-instance?var-ins={{ $labels.ins }}          

    # haproxy the load balancer
    - alert: HaproxyDown
      expr: haproxy_up < 1
      for: 1m
      labels: { level: 0, severity: CRIT, category: node }
      annotations:
        summary: "CRIT HaproxyDown {{ $labels.ins }}@{{ $labels.instance }}"
        description: |
          haproxy_up[ins={{ $labels.ins }}, instance={{ $labels.instance }}] = {{ $value }} < 1
          http://g.pigsty/d/node-haproxy?var-ins={{ $labels.ins }}          

    # promtail the logging agent
    - alert: PromtailDown
      expr: promtail_up < 1
      for: 1m
      labels: { level: 1, severity: WARN, category: node }
      annotations:
        summary: "WARN PromtailDown {{ $labels.ins }}@{{ $labels.instance }}"
        description: |
          promtail_up[ins={{ $labels.ins }}, instance={{ $labels.instance }}] = {{ $value }} < 1
          http://g.pigsty/d/node-instance?var-ins={{ $labels.ins }}          

    # docker the container engine
    - alert: DockerDown
      expr: docker_up < 1
      for: 1m
      labels: { level: 1, severity: WARN, category: node }
      annotations:
        summary: "WARN DockerDown {{ $labels.ins }}@{{ $labels.instance }}"
        description: |
          docker_up[ins={{ $labels.ins }}, instance={{ $labels.instance }}] = {{ $value }} < 1
          http://g.pigsty/d/node-instance?var-ins={{ $labels.ins }}          

    # keepalived daemon
    - alert: KeepalivedDown
      expr: keepalived_up < 1
      for: 1m
      labels: { level: 1, severity: WARN, category: node }
      annotations:
        summary: "WARN KeepalivedDown {{ $labels.ins }}@{{ $labels.instance }}"
        description: |
          keepalived_up[ins={{ $labels.ins }}, instance={{ $labels.instance }}] = {{ $value }} < 1
          http://g.pigsty/d/node-instance?var-ins={{ $labels.ins }}          



    #==============================================================#
    #                          Node : CPU                          #
    #==============================================================#
    # cpu usage high : 1m avg cpu usage > 70% for 3m
    - alert: NodeCpuHigh
      expr: node:ins:cpu_usage_1m > 0.70
      for: 1m
      labels: { level: 1, severity: WARN, category: node }
      annotations:
        summary: 'WARN NodeCpuHigh {{ $labels.ins }}@{{ $labels.instance }} {{ $value  | printf "%.2f" }}'
        description: |
          node:ins:cpu_usage[ins={{ $labels.ins }}] = {{ $value  | printf "%.2f" }} > 70%          

    # OPTIONAL: one core high
    # OPTIONAL: throttled
    # OPTIONAL: frequency
    # OPTIONAL: steal

    #==============================================================#
    #                       Node : Schedule                        #
    #==============================================================#
    # node load high : 1m avg standard load > 100% for 3m
    - alert: NodeLoadHigh
      expr: node:ins:stdload1 > 1
      for: 1m
      labels: { level: 1, severity: WARN, category: node }
      annotations:
        summary: 'WARN NodeLoadHigh {{ $labels.ins }}@{{ $labels.instance }} {{ $value  | printf "%.2f" }}'
        description: |
          node:ins:stdload1[ins={{ $labels.ins }}] = {{ $value  | printf "%.2f" }} > 100%          


    #==============================================================#
    #                        Node : Memory                         #
    #==============================================================#
    # available memory < 10%
    - alert: NodeOutOfMem
      expr: node:ins:mem_avail < 0.10
      for: 1m
      labels: { level: 1, severity: WARN, category: node }
      annotations:
        summary: 'WARN NodeOutOfMem {{ $labels.ins }}@{{ $labels.instance }} {{ $value  | printf "%.2f" }}'
        description: |
          node:ins:mem_avail[ins={{ $labels.ins }}] = {{ $value  | printf "%.2f" }} < 10%          

    # commit ratio > 90%
    #- alert: NodeMemCommitRatioHigh
    #  expr: node:ins:mem_commit_ratio > 0.90
    #  for: 1m
    #  labels: { level: 1, severity: WARN, category: node }
    #  annotations:
    #    summary: 'WARN NodeMemCommitRatioHigh {{ $labels.ins }}@{{ $labels.instance }} {{ $value  | printf "%.2f" }}'
    #    description: |
    #      node:ins:mem_commit_ratio[ins={{ $labels.ins }}] = {{ $value  | printf "%.2f" }} > 90%

    # OPTIONAL: EDAC Errors

    #==============================================================#
    #                        Node : Swap                           #
    #==============================================================#
    # swap usage > 1%
    - alert: NodeMemSwapped
      expr: node:ins:swap_usage > 0.01
      for: 5m
      labels: { level: 1, severity: WARN, category: node }
      annotations:
        summary: 'WARN NodeMemSwapped {{ $labels.ins }}@{{ $labels.instance }} {{ $value  | printf "%.2f" }}'
        description: |
          node:ins:swap_usage[ins={{ $labels.ins }}] = {{ $value  | printf "%.2f" }} > 1%          

    #==============================================================#
    #                     Node : File System                       #
    #==============================================================#

    # filesystem usage > 90%
    - alert: NodeFsSpaceFull
      expr: node:fs:space_usage > 0.90
      for: 1m
      labels: { level: 1, severity: WARN, category: node }
      annotations:
        summary: 'WARN NodeFsSpaceFull {{ $labels.ins }}@{{ $labels.instance }} {{ $value  | printf "%.2f" }}'
        description: |
          node:fs:space_usage[ins={{ $labels.ins }}] = {{ $value  | printf "%.2f" }} > 90%          

    # inode usage > 90%
    - alert: NodeFsFilesFull
      expr: node:fs:inode_usage > 0.90
      for: 1m
      labels: { level: 1, severity: WARN, category: node }
      annotations:
        summary: 'WARN NodeFsFilesFull {{ $labels.ins }}@{{ $labels.instance }} {{ $value  | printf "%.2f" }}'
        description: |
          node:fs:inode_usage[ins={{ $labels.ins }}] = {{ $value  | printf "%.2f" }} > 90%          

    # file descriptor usage > 90%
    - alert: NodeFdFull
      expr: node:ins:fd_usage > 0.90
      for: 1m
      labels: { level: 1, severity: WARN, category: node }
      annotations:
        summary: 'WARN NodeFdFull {{ $labels.ins }}@{{ $labels.instance }} {{ $value  | printf "%.2f" }}'
        description: |
          node:ins:fd_usage[ins={{ $labels.ins }}] = {{ $value  | printf "%.2f" }} > 90%          

    # OPTIONAL: space predict 1d
    # OPTIONAL: filesystem read-only
    # OPTIONAL: fast release on disk space

    #==============================================================#
    #                          Node : Disk                         #
    #==============================================================#
    # read latency > 32ms (typical on pci-e ssd: 100µs)
    - alert: NodeDiskSlow
      expr: node:dev:disk_read_rt_1m{device="dfa"} > 0.032 or node:dev:disk_write_rt_1m{device="dfa"} > 0.032
      for: 1m
      labels: { level: 2, severity: INFO, category: node }
      annotations:
        summary: 'INFO NodeReadSlow {{ $labels.ins }}@{{ $labels.instance }} {{ $value  | printf "%.6f" }}'
        description: |
          node:dev:disk_read_rt_1m[ins={{ $labels.ins }}] = {{ $value  | printf "%.6f" }} > 32ms          

    # OPTIONAL: raid card failure
    # OPTIONAL: read/write traffic high
    # OPTIONAL: read/write latency high

    #==============================================================#
    #                        Node : Network                        #
    #==============================================================#
    # OPTIONAL: unusual network traffic
    # OPTIONAL: interface saturation high

    #==============================================================#
    #                        Node : Protocol                       #
    #==============================================================#

    # rate(node:ins:tcp_error[1m]) > 1
    - alert: NodeTcpErrHigh
      expr: rate(node:ins:tcp_error[1m]) > 1
      for: 1m
      labels: { level: 1, severity: WARN, category: node }
      annotations:
        summary: 'WARN NodeTcpErrHigh {{ $labels.ins }}@{{ $labels.instance }} {{ $value  | printf "%.2f" }}'
        description: |
          rate(node:ins:tcp_error{ins={{ $labels.ins }}}[1m]) = {{ $value  | printf "%.2f" }} > 1          

    # node:ins:tcp_retrans_ratio1m > 1e-4
    - alert: NodeTcpRetransHigh
      expr: node:ins:tcp_retrans_ratio1m > 1e-2
      for: 1m
      labels: { level: 2, severity: INFO, category: node }
      annotations:
        summary: 'INFO NodeTcpRetransHigh {{ $labels.ins }}@{{ $labels.instance }} {{ $value  | printf "%.6f" }}'
        description: |
          node:ins:tcp_retrans_ratio1m[ins={{ $labels.ins }}] = {{ $value  | printf "%.6f" }} > 1%          

    # OPTIONAL: tcp conn high
    # OPTIONAL: udp traffic high
    # OPTIONAL: conn track

    #==============================================================#
    #                          Node : Time                         #
    #==============================================================#

    - alert: NodeTimeDrift
      expr: node_timex_sync_status != 1
      for: 1m
      labels: { level: 1, severity: WARN, category: node }
      annotations:
        summary: 'WARN NodeTimeDrift {{ $labels.ins }}@{{ $labels.instance }}'
        description: |
          node_timex_status[ins={{ $labels.ins }}]) = {{ $value | printf "%.6f" }} != 0 or
          node_timex_sync_status[ins={{ $labels.ins }}]) = {{ $value | printf "%.6f" }} != 1          


    # time drift > 64ms
    # - alert: NodeTimeDrift
    #   expr: node:ins:time_drift > 0.064
    #   for: 1m
    #   labels: { level: 1, severity: WARN, category: node }
    #   annotations:
    #     summary: 'WARN NodeTimeDrift {{ $labels.ins }}@{{ $labels.instance }}'
    #     description: |
    #       abs(node_timex_offset_seconds)[ins={{ $labels.ins }}]) = {{ $value | printf "%.6f" }} > 64ms