new exporter version

This commit is contained in:
a.pivkin 2025-12-17 10:02:26 +03:00
parent 91cc8ffe61
commit 59d2508e7f
12 changed files with 394 additions and 58 deletions

12
go_exporter/Dockerfile Normal file
View File

@ -0,0 +1,12 @@
FROM ubuntu:22.04
RUN apt-get update && apt-get install -y \
librbd-dev \
librados-dev \
&& rm -rf /var/lib/apt/lists/*
RUN mkdir /app
WORKDIR /app
COPY ./rbd-exporter /app/rbd-exporter
EXPOSE 9040
CMD [ "./rbd-exporter" ]

View File

@ -1,10 +1,9 @@
package main
import (
"encoding/json"
"fmt"
"github.com/ceph/go-ceph/rados"
"github.com/ceph/go-ceph/rbd"
pipe "gopkg.in/pipe.v2"
)
func PoolFactory(cephConn CephConnection, poolName string) (Pool, error) {
@ -21,42 +20,44 @@ func PoolFactory(cephConn CephConnection, poolName string) (Pool, error) {
if err != nil {
return Pool{}, fmt.Errorf("Couldn't get list of rbds %w", err)
}
for _, v := range imageList {
stat, err := RBDFacroty(poolName, v)
for _, rbdname := range imageList {
stat, err := RBDFacroty(ioctx, rbdname)
if err != nil {
fmt.Errorf("Coundn't get stat from disk %s %w", v, err)
logger.Errorf("Coundn't get stat from disk %s %w", rbdname, err)
panic(err)
}
// fmt.Println(stat)
rbdlist = append(rbdlist, stat)
}
return Pool{
Name: poolName,
hasRBD: len(imageList) != 0,
RBDlist: rbdlist,
}, nil
}
func RBDFacroty(poolname string, rbdname string) (RBD, error) {
func RBDFacroty(ioctx *rados.IOContext, rbdname string) (RBD, error) {
defer func() {
if v := recover(); v != nil {
fmt.Println("Shit happened")
}
}()
rbdPath := fmt.Sprintf("%s/%s", poolname, rbdname)
args := []string{"du", "--format", "json", rbdPath}
p := pipe.Line(
pipe.Exec("rbd", args...),
)
output, err := pipe.CombinedOutput(p)
image := rbd.GetImage(ioctx,rbdname)
err := image.Open()
if err != nil {
fmt.Errorf("Error in processing RBD %v", err)
panic(err)
}
var rbdImage RBDUsage
if err := json.Unmarshal(output, &rbdImage); err != nil {
fmt.Errorf("Error in unmarshaling %w", err)
defer image.Close()
info,err := image.Stat()
if err != nil {
return RBD{},fmt.Errorf("Couldn't get stats for image %s %w",rbdname,err)
}
return RBD{
Name: rbdImage.Images[0].Name,
Id: rbdImage.Images[0].Id,
RequestedSize: rbdImage.Images[0].RequestedSize,
UsedSize: rbdImage.Images[0].UsedSize,
Name: rbdname,
ImageInfo: *info,
},nil
}

151
go_exporter/deploy.yaml Normal file
View File

@ -0,0 +1,151 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: export-deploy
namespace: rook-ceph
labels:
app: export-deploy
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "9040"
prometheus.io/path: "/metrics"
spec:
replicas: 1
selector:
matchLabels:
app: export-box
template:
metadata:
labels:
app: export-box
spec:
containers:
- name: export
image: serviceplant/goexp:0.0.30
ports:
- containerPort: 9040
name: metrics
command:
- /bin/bash
- -c
- |
# Replicate the script from toolbox.sh inline so the ceph image
# can be run directly, instead of requiring the rook toolbox
CEPH_CONFIG="/etc/ceph/ceph.conf"
MON_CONFIG="/etc/rook/mon-endpoints"
KEYRING_FILE="/etc/ceph/keyring"
# create a ceph config file in its default location so ceph/rados tools can be used
# without specifying any arguments
write_endpoints() {
endpoints=$(cat ${MON_CONFIG})
# filter out the mon names
# external cluster can have numbers or hyphens in mon names, handling them in regex
# shellcheck disable=SC2001
mon_endpoints=$(echo "${endpoints}"| sed 's/[a-z0-9_-]\+=//g')
DATE=$(date)
echo "$DATE writing mon endpoints to ${CEPH_CONFIG}: ${endpoints}"
cat <<EOF > ${CEPH_CONFIG}
[global]
mon_host = ${mon_endpoints}
[client.admin]
keyring = ${KEYRING_FILE}
EOF
}
# watch the endpoints config file and update if the mon endpoints ever change
watch_endpoints() {
# get the timestamp for the target of the soft link
real_path=$(realpath ${MON_CONFIG})
initial_time=$(stat -c %Z "${real_path}")
while true; do
echo "I am watching!!!"
real_path=$(realpath ${MON_CONFIG})
latest_time=$(stat -c %Z "${real_path}")
if [[ "${latest_time}" != "${initial_time}" ]]; then
write_endpoints
initial_time=${latest_time}
fi
sleep 10
done
}
# read the secret from an env var (for backward compatibility), or from the secret file
ceph_secret=${ROOK_CEPH_SECRET}
if [[ "$ceph_secret" == "" ]]; then
ceph_secret=$(cat /var/lib/rook-ceph-mon/secret.keyring)
fi
# create the keyring file
cat <<EOF > ${KEYRING_FILE}
[${ROOK_CEPH_USERNAME}]
key = ${ceph_secret}
EOF
# write the initial config file
write_endpoints
# continuously update the mon endpoints if they fail over
exec /app/rbd-exporter --keyring /etc/ceph/keyring &
watch_endpoints
imagePullPolicy: IfNotPresent
tty: true
securityContext:
runAsNonRoot: true
runAsUser: 2016
runAsGroup: 2016
capabilities:
drop: ["ALL"]
env:
- name: ROOK_CEPH_USERNAME
valueFrom:
secretKeyRef:
name: rook-ceph-mon
key: ceph-username
volumeMounts:
- mountPath: /etc/ceph
name: ceph-config
- name: mon-endpoint-volume
mountPath: /etc/rook
- name: ceph-admin-secret
mountPath: /var/lib/rook-ceph-mon
readOnly: true
volumes:
- name: ceph-admin-secret
secret:
secretName: rook-ceph-mon
optional: false
items:
- key: ceph-secret
path: secret.keyring
- name: mon-endpoint-volume
configMap:
name: rook-ceph-mon-endpoints
items:
- key: data
path: mon-endpoints
- name: ceph-config
emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
name: export-service
namespace: rook-ceph
labels:
svc_app: export
rook_cluster: rook-ceph
spec:
selector:
app: export-box
ports:
- name: metrics
port: 9040
targetPort: 9040
protocol: TCP
type: ClusterIP

View File

@ -1,8 +1,13 @@
package main
import "flag"
import (
"flag"
"fmt"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
)
var params Params
// This func runs even before main()
func init() {
@ -13,4 +18,21 @@ func init() {
params.config = *config_file
params.keyring = *_keyring
logger = loggerInit()
logger.Info("Setting up logger is complete successfully")
logger.Info("Registering prom metrics")
}
// Here I initialize logger and set some custom settings
func loggerInit() *zap.SugaredLogger {
config := zap.NewDevelopmentConfig()
config.EncoderConfig.TimeKey = "timestamp"
config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
logger, err := config.Build()
if err != nil {
panic(fmt.Sprintf("Logger set up failed: %v", err))
}
defer logger.Sync()
return logger.Sugar()
}

View File

@ -4,10 +4,19 @@ go 1.25.5
require (
github.com/ceph/go-ceph v0.36.0
gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544
github.com/prometheus/client_golang v1.23.2
go.uber.org/zap v1.27.1
)
require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.66.1 // indirect
github.com/prometheus/procfs v0.16.1 // indirect
go.uber.org/multierr v1.10.0 // indirect
go.yaml.in/yaml/v2 v2.4.2 // indirect
golang.org/x/sys v0.36.0 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
google.golang.org/protobuf v1.36.8 // indirect
)

View File

@ -1,23 +1,49 @@
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/ceph/go-ceph v0.36.0 h1:IDE4vEF+4fmjve+CPjD1WStgfQ+Lh6vD+9PMUI712KI=
github.com/ceph/go-ceph v0.36.0/go.mod h1:fGCbndVDLuHW7q2954d6y+tgPFOBnRLqJRe2YXyngw4=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gofrs/uuid/v5 v5.3.2 h1:2jfO8j3XgSwlz/wHqemAEugfnTlikAYHhnqQ8Xh4fE0=
github.com/gofrs/uuid/v5 v5.3.2/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8=
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc=
go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544 h1:WJH1qsOB4/zb/li+zLMn0vaAUJ5FqPv6HYLI3aQVg1k=
gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544/go.mod h1:UhTeH/yXCK/KY7TX24mqPkaQ7gZeqmWd/8SSS8B3aHw=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@ -3,22 +3,30 @@ package main
import (
"errors"
"fmt"
"net/http"
"os"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"go.uber.org/zap"
)
func main() {
var result []Pool = []Pool{}
cephConn, err := connect()
if err != nil {
fmt.Println(err)
if wrapped := errors.Unwrap(err); wrapped != nil {
fmt.Println(wrapped)
}
os.Exit(1)
}
defer cephConn.conn.Shutdown()
fmt.Println("Successfully connected")
var params Params
var logger *zap.SugaredLogger
func getMetrics(cephConn CephConnection) {
metrics := InitMetrics()
prometheus.MustRegister(
metrics.total_rbd_requested_size_per_pool,
)
ticker := time.NewTicker(2 * time.Second)
defer ticker.Stop()
for range ticker.C {
var result []Pool = []Pool{}
poolList, err := cephConn.conn.ListPools()
if err != nil {
errors.New("Cannot get list of pools")
@ -28,6 +36,44 @@ func main() {
x, _ := PoolFactory(cephConn, v)
result = append(result, x)
}
fmt.Println(result)
for _,v := range result {
if !v.hasRBD {continue}
FillMetrics(v,metrics)
}
fmt.Println("====================")
// fmt.Println(result)
}
}
func main() {
cephConn, err := connect()
if err != nil {
fmt.Println(err)
if wrapped := errors.Unwrap(err); wrapped != nil {
fmt.Println(wrapped)
}
os.Exit(1)
}
defer cephConn.conn.Shutdown()
logger.Info("Successfully connected")
http.Handle("/metrics", promhttp.Handler())
// HTTP runs in separate thread cuz it blocks futher execution of main
go func() {
logger.Info("Starting http server")
// Here I check for errors if HTTP fails
if err := http.ListenAndServe(":9040", nil); err != nil {
logger.Fatalf("HTTP server failed to start %v", err)
}
logger.Info("HTTP server started")
}()
go getMetrics(cephConn)
select {}
}

42
go_exporter/metrics.go Normal file
View File

@ -0,0 +1,42 @@
package main
import (
"github.com/prometheus/client_golang/prometheus"
)
func InitMetrics() *Metrics {
m := &Metrics{
total_rbd_requested_size_per_pool: prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "total_rbd_requested_size_per_pool",
Help: "total size of all requested RBDs in a specific pool",
},
[]string{"poolname"},
),
// total_rbd_requested_size: prometheus.NewGauge(
// prometheus.GaugeOpts{
// Name: "total_rbd_requested_size",
// Help: "total size of all RBDs in the cluster",
// },
// ),
}
return m
}
func FillMetrics(pool Pool,metrics *Metrics) {
var totalSizePerPool uint64 = 0
metrics.total_rbd_requested_size_per_pool.Reset()
logger.Debugf("Processing pool %s",pool.Name)
for _,v := range pool.RBDlist {
totalSizePerPool += uint64(v.getSize())
}
logger.Debugf("Total size of RBDs in pool %s is %d",pool.Name,totalSizePerPool)
metrics.total_rbd_requested_size_per_pool.WithLabelValues(pool.Name,
).Set(
float64(totalSizePerPool))
}

BIN
go_exporter/rbd-exporter Executable file

Binary file not shown.

View File

@ -0,0 +1,20 @@
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: export-monitor
namespace: rook-ceph
labels:
team: rook
spec:
namespaceSelector:
matchNames:
- rook-ceph
selector:
matchLabels:
svc_app: export
rook_cluster: rook-ceph
endpoints:
- port: metrics
interval: 30s
path: /metrics
honorLabels: true

View File

@ -1,6 +1,10 @@
package main
import "github.com/ceph/go-ceph/rados"
import (
"github.com/ceph/go-ceph/rados"
"github.com/ceph/go-ceph/rbd"
"github.com/prometheus/client_golang/prometheus"
)
type Params struct {
config string
@ -17,14 +21,12 @@ type iRBD interface {
}
type RBDUsage struct {
Images []RBD `json:"images"`
Images []RBD
}
type RBD struct {
Name string `json:"name"`
Id string `json:"id"`
RequestedSize uint64 `json:"provisioned_size"`
UsedSize int64 `json:"used_size"`
Name string
rbd.ImageInfo
}
func (r RBD) getName() string {
@ -32,11 +34,16 @@ func (r RBD) getName() string {
}
func (r RBD) getSize() int64 {
return r.UsedSize
return int64(r.Size)
}
type Pool struct {
Name string
hasRBD bool
RBDlist []iRBD
// RBDlist []string
}
type Metrics struct {
total_rbd_requested_size_per_pool *prometheus.GaugeVec
total_rbd_requested_size prometheus.Gauge
}

View File

@ -5,7 +5,7 @@ COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN CGO_ENABLED=0 GOOS=linux go build -o exporter .
RUN CGO_ENABLED=0 GOOS=linux go build -o rbd-exporter .
FROM ubuntu:22.04
RUN apt-get update && apt-get install -y \
@ -14,7 +14,7 @@ RUN apt-get update && apt-get install -y \
WORKDIR /home/
COPY --from=builder /app/exporter .
COPY --from=builder /app/rbd-exporter .
EXPOSE 9040
CMD [ "./exporter" ]
CMD [ "./rbd-exporter" ]