Thanos

# macOS
brew install thanos

# external_labels 是必须的
cat <<YAML > prometheus.yaml
global:
  external_labels:
    cluster: test
    replica: svr-1
  scrape_interval: 15s

scrape_configs:
  - job_name: "prometheus"
    static_configs:
    - targets: ["localhost:9090"]

  - job_name: "node"
    static_configs:
    - targets: ["localhost:9100"]
YAML

# 默认参数
# http://localhost:9090
prometheus \
  --config.file="prometheus.yml" \
  --storage.tsdb.path=data/ \
  --storage.tsdb.max-block-duration=2h \
  --storage.tsdb.min-block-duration=2h \
  --web.enable-lifecycle \
  --web.enable-admin-api

cat <<YAML > thanos-store.yaml
type: S3
config:
  bucket: "metrics"
  endpoint: "minio.cluster.internal"
  insecure: true
  signature_version2: false
  access_key: "key"
  secret_key: "secretsecret"
YAML
# Sidecar 上传 chunk 暴露 prometheus 为 StoreAPI
# 默认 http://localhost:10902 grpc://localhost:10901
thanos sidecar \
  --tsdb.path="./data" \
  --prometheus.url=http://localhost:9090 \
  --objstore.config-file=thanos-store.yaml \
  --http-address="0.0.0.0:10902" \
  --grpc-address="0.0.0.0:10901"

# 暴露 S3 为 StoreAPI
# 修改端口避免冲突
thanos store \
    --http-address=0.0.0.0:10912 \
    --grpc-address=0.0.0.0:10911 \
    --data-dir="./store-cache" \
    --objstore.config-file=thanos-store.yaml

# 查询
# 修改端口避免冲突
# 查询 UI http://localhost:11902/graph
# --store 为后端
thanos query \
    --http-address=0.0.0.0:11902 \
    --grpc-address=0.0.0.0:11901 \
    --store=localhost:10901 \
    --store=localhost:10911


# 压缩
# --data-dir cache blocks and process compactions
# --wait 等待新的任务，默认执行完成后退出
thanos compact \
  --http-address=0.0.0.0:12902 \
  --data-dir=thanos-compact \
  --objstore.config-file=thanos-store.yaml \
  --wait

# 查看 bucket 分布情况
# http://localhost:10902/
thanos tools bucket web --objstore.config-file=thanos-bucket.yaml

# 查看 bucket
thanos tools bucket ls
# 以 table 显示 bucket 信息
# | ULID | FROM | UNTIL | RANGE | UNTIL-DOWN | #SERIES | #SAMPLES | #CHUNKS | COMP-LEVEL | COMP-FAILED | LABELS | RESOLUTION | SOURCE |
thanos tools bucket inspect

# 标记删除或不压缩
# tools bucket mark --id=ID --marker=MARKER --details=DETAILS
# Marker - deletion-mark.json,no-compact-mark.json
# mark 或删除在 web 上不会刷新，需要停止从开
thanos tools bucket mark --id 01EJD9PS4P3MJMF3TGJGTJTE25 --marker deletion-mark.json --details 'useless bucket'

# 立即清除被 mark bucket
# 默认由 compactor 来清理
thanos tools bucket cleanup

# out.txt
thanos tools bucket inspect --log.level error --objstore.config-file=thanos-store.yaml > out.txt
# ids.txt
cat out.txt | grep -P '\d\d-2021' | cut -d '|' -f 2  | tr -d ' '| sed 's/^/--id=/' > ids.txt
# filter & delete
# 批量标记删除 - 需要有 compact 实际删除 - 或者执行 compact --delete-delay=0
cat out.txt | grep -P '(01|02|03|04|05|06|07|08)-2021' | cut -d '|' -f 2  | tr -d ' '| sed 's/^/--id=/' | xargs thanos tools bucket mark --marker deletion-mark.json --details 'delete' --objstore.config-file=thanos-store.yaml
cat ids.txt | xargs thanos tools bucket mark --marker deletion-mark.json --details 'delete' --objstore.config-file=thanos-store.yaml

# 文件缓存
type: IN-MEMORY
config:
  max_size: 0
  max_size_items: 2048
  # 缓存时效
  validity: 6h

---
type: MEMCACHED
config:
  addresses: []
  timeout: 0s
  max_idle_connections: 0
  max_async_concurrency: 0
  max_async_buffer_size: 0
  max_get_multi_concurrency: 0
  max_item_size: 0
  max_get_multi_batch_size: 0
  dns_provider_update_interval: 0s
  expiration: 0s

---
# S3 常用配置
type: S3
config:
  bucket: ''
  endpoint: ''
  # HTTP 还是 HTTPS
  insecure: true
  access_key: ''
  secret_key: ''
  # 部分 S3 实现需要设置为 false
  signature_version2: false
  # multipart 上传的单块 大小 - 部分 S3 实现需要修改
  # 0 -> 默认 128m
  part_size: 0

---
# S3 完整配置
type: S3
config:
  bucket: ''
  endpoint: ''
  region: ''
  access_key: ''
  insecure: false
  signature_version2: false
  secret_key: ''
  put_user_metadata: {}
  http_config:
    idle_conn_timeout: 1m30s
    response_header_timeout: 2m
    insecure_skip_verify: false
  trace:
    enable: false
  part_size: 134217728
  sse_config:
    type: ''
    kms_key_id: ''
    kms_encryption_context: {}
    encryption_key: ''

---
# 阿里云 OSS
type: ALIYUNOSS
config:
  endpoint: ''
  bucket: ''
  access_key_id: ''
  access_key_secret: ''

---
# 腾讯云 COS
type: COS
config:
  bucket: ''
  region: ''
  app_id: ''
  secret_key: ''
  secret_id: ''

type: JAEGER
config:
  service_name: ''
  disabled: false
  rpc_metrics: false
  tags: ''
  sampler_type: ''
  sampler_param: 0
  sampler_manager_host_port: ''
  sampler_max_operations: 0
  sampler_refresh_interval: 0s
  reporter_max_queue_size: 0
  reporter_flush_interval: 0s
  reporter_log_spans: false
  endpoint: ''
  user: ''
  password: ''
  agent_host: ''
  agent_port: 0

# --delete-delay=0 - 立即删除 - 默认 48h
thanos compact \
  --http-address=0.0.0.0:12902 \
  --data-dir=thanos-compact \
  --objstore.config-file=thanos-store.yaml \
  --wait \
  --compact.concurrency=32 \
  --retention.resolution-raw=30d \
  --retention.resolution-5m=90d \
  --retention.resolution-1h=180d \
  --delete-delay=0

thanos sidecar \
  --tsdb.path=/var/lib/prometheus/data/ \
  --prometheus.url=http://localhost:9090 \
  --objstore.config-file=thanos-bucket.yaml \
  --shipper.upload-compacted

thanos sidecar --prometheus.url=http://localhost:9090

# 5m 使用下采样会有 gap
avg(irate(node_cpu_seconds_total{mode="system"}[5m])) by (instance) *100
# 10m 则没有问题
avg(irate(node_cpu_seconds_total{mode="system"}[10m])) by (instance) *100

# grafana
avg(irate(node_cpu_seconds_total{mode="system"}[$__interval])) by (instance) *100

ls */meta.json | xargs -n 1 -I {}  sh -c "dirname {} | tr -d '\n';echo -n ' ';jq -r '.maxTime/1000 | todate' {}" > /tmp/dump.txt
# 强制删除
cat /tmp/dump.txt | grep '2021-' | awk '{print $1}' | sudo xargs -I {} rm -r {}