lolkek

ceph_socket/go.mod

@@ -0,0 +1,5 @@
+module main.go
+
+go 1.25.4
+
+require gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544 // indirect

ceph_socket/go.sum

@@ -0,0 +1,2 @@
+gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544 h1:WJH1qsOB4/zb/li+zLMn0vaAUJ5FqPv6HYLI3aQVg1k=
+gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544/go.mod h1:UhTeH/yXCK/KY7TX24mqPkaQ7gZeqmWd/8SSS8B3aHw=

ceph_socket/main.go

@@ -0,0 +1,22 @@
+package main
+
+import (
+	"fmt"
+	"log"
+
+	"gopkg.in/pipe.v2"
+)
+
+func main() {
+	args := []string{"25"}
+	p := pipe.Line(
+		pipe.ReadFile("go.mod"),
+		pipe.Exec("grep",args...),
+	)
+
+	output, err := pipe.CombinedOutput(p)
+	if err != nil {
+		log.Fatal("Shit happened")
+	}
+	fmt.Println(string(output))
+}

go_exporter/Dockerfile

@@ -7,9 +7,12 @@ RUN go mod download
 COPY . .
 RUN CGO_ENABLED=0 GOOS=linux go build -o exporter .
 
-FROM alpine:latest
+FROM ubuntu:22.04
-RUN apk --no-cache add ca-certificates
+RUN apt-get update && apt-get install -y \
-WORKDIR /root/
+    ceph-common \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /home/
 
 COPY --from=builder /app/exporter .
 EXPOSE 9040

go_exporter/deploy.yaml

@@ -1,21 +1,131 @@
-apiVersion: v1
+apiVersion: apps/v1
-kind: Pod
+kind: Deployment
 metadata:
-  name: export
+  name: export-deploy
   namespace: rook-ceph
   labels:
-    app: export
+    app: export-deploy
-  annotations:
-    prometheus.io/scrape: "true"
-    prometheus.io/port: "9040"
-    prometheus.io/path: "/metrics" 
 spec:
-  containers:
+  replicas: 1
-  - name: export
+  selector:
-    image: serviceplant/goexp:0.0.5
+    matchLabels:
-    ports:
+      app: export-box
-      - containerPort: 9040
+  template:
-        name: metrics
+    metadata:
+      labels:
+        app: export-box
+    spec:
+      containers:
+      - name: export
+        image: serviceplant/goexp:0.0.23
+        ports:
+          - containerPort: 9040
+            name: metrics
+        command:
+          - /bin/bash
+          - -c
+          - |
+            # Replicate the script from toolbox.sh inline so the ceph image
+            # can be run directly, instead of requiring the rook toolbox
+            CEPH_CONFIG="/etc/ceph/ceph.conf"
+            MON_CONFIG="/etc/rook/mon-endpoints"
+            KEYRING_FILE="/etc/ceph/keyring"
+
+            # create a ceph config file in its default location so ceph/rados tools can be used
+            # without specifying any arguments
+            write_endpoints() {
+              endpoints=$(cat ${MON_CONFIG})
+
+              # filter out the mon names
+              # external cluster can have numbers or hyphens in mon names, handling them in regex
+              # shellcheck disable=SC2001
+              mon_endpoints=$(echo "${endpoints}"| sed 's/[a-z0-9_-]\+=//g')
+
+              DATE=$(date)
+              echo "$DATE writing mon endpoints to ${CEPH_CONFIG}: ${endpoints}"
+                cat <<EOF > ${CEPH_CONFIG}
+            [global]
+            mon_host = ${mon_endpoints}
+
+            [client.admin]
+            keyring = ${KEYRING_FILE}
+            EOF
+            }
+
+            # watch the endpoints config file and update if the mon endpoints ever change
+            watch_endpoints() {
+              # get the timestamp for the target of the soft link
+              real_path=$(realpath ${MON_CONFIG})
+              initial_time=$(stat -c %Z "${real_path}")
+              while true; do
+                real_path=$(realpath ${MON_CONFIG})
+                latest_time=$(stat -c %Z "${real_path}")
+
+                if [[ "${latest_time}" != "${initial_time}" ]]; then
+                  write_endpoints
+                  initial_time=${latest_time}
+                fi
+
+                sleep 10
+              done
+            }
+
+            # read the secret from an env var (for backward compatibility), or from the secret file
+            ceph_secret=${ROOK_CEPH_SECRET}
+            if [[ "$ceph_secret" == "" ]]; then
+              ceph_secret=$(cat /var/lib/rook-ceph-mon/secret.keyring)
+            fi
+
+            # create the keyring file
+            cat <<EOF > ${KEYRING_FILE}
+            [${ROOK_CEPH_USERNAME}]
+            key = ${ceph_secret}
+            EOF
+
+            # write the initial config file
+            write_endpoints
+
+            # continuously update the mon endpoints if they fail over
+            exec /home/exporter
+            watch_endpoints &
+        imagePullPolicy: IfNotPresent
+        tty: true
+        securityContext:
+          runAsNonRoot: true
+          runAsUser: 2016
+          runAsGroup: 2016
+          capabilities:
+            drop: ["ALL"]
+        env:
+          - name: ROOK_CEPH_USERNAME
+            valueFrom:
+              secretKeyRef:
+                name: rook-ceph-mon
+                key: ceph-username
+        volumeMounts:
+          - mountPath: /etc/ceph
+            name: ceph-config
+          - name: mon-endpoint-volume
+            mountPath: /etc/rook
+          - name: ceph-admin-secret
+            mountPath: /var/lib/rook-ceph-mon
+            readOnly: true
+      volumes:
+        - name: ceph-admin-secret
+          secret:
+            secretName: rook-ceph-mon
+            optional: false
+            items:
+              - key: ceph-secret
+                path: secret.keyring
+        - name: mon-endpoint-volume
+          configMap:
+            name: rook-ceph-mon-endpoints
+            items:
+              - key: data
+                path: mon-endpoints
+        - name: ceph-config
+          emptyDir: {}
 ---
 apiVersion: v1
 kind: Service

go_exporter/go.mod

@@ -2,15 +2,21 @@ module main.go
 
 go 1.25.4
 
+require (
+	github.com/prometheus/client_golang v1.23.2
+	go.uber.org/zap v1.27.1
+)
+
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/prometheus/client_golang v1.23.2 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
-	github.com/prometheus/common v0.66.1 // indirect
+	github.com/prometheus/common v0.67.4 // indirect
-	github.com/prometheus/procfs v0.16.1 // indirect
+	github.com/prometheus/procfs v0.19.2 // indirect
-	go.yaml.in/yaml/v2 v2.4.2 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/sys v0.35.0 // indirect
+	go.yaml.in/yaml/v2 v2.4.3 // indirect
-	google.golang.org/protobuf v1.36.8 // indirect
+	golang.org/x/sys v0.38.0 // indirect
+	google.golang.org/protobuf v1.36.10 // indirect
+	gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544 // indirect
 )

go_exporter/go.sum

@@ -10,12 +10,28 @@ github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNw
 github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
 github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
 github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
+github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+Lvsc=
+github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI=
 github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
 github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
+github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws=
+github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc=
+go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
 go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
 go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
+go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
+go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
 golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
 golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
+golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
 google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
+google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
+google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544 h1:WJH1qsOB4/zb/li+zLMn0vaAUJ5FqPv6HYLI3aQVg1k=
+gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544/go.mod h1:UhTeH/yXCK/KY7TX24mqPkaQ7gZeqmWd/8SSS8B3aHw=

go_exporter/main.go

@@ -1,14 +1,35 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
-	"log"
 	"net/http"
+	_ "os/exec"
+	"strings"
+	"time"
 
-	promhttp "github.com/prometheus/client_golang/prometheus/promhttp"
 	prometheus "github.com/prometheus/client_golang/prometheus"
+	promhttp "github.com/prometheus/client_golang/prometheus/promhttp"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+	pipe "gopkg.in/pipe.v2"
 )
 
+var logger *zap.SugaredLogger
+
+// Here I store all images info in scope of one pool
+type RBDUsage struct {
+	Images []RBDImageUsage
+}
+
+// Here I do collect info about images itself
+type RBDImageUsage struct {
+	Name          string `json:"name"`
+	id            int    `json: "id"`
+	RequestedSize uint64 `json:"provisioned_size"`
+	UsedSize      uint64 `json: "used_size"`
+}
+
 var fooMetric = prometheus.NewGauge(prometheus.GaugeOpts{
 	Name: "foo_metric",
 	Help: "Show whether a foo has occured in a cluster",
@@ -19,8 +40,24 @@ var barMetric = prometheus.NewGauge(prometheus.GaugeOpts{
 	Help: "Show whether a bar has happened in a cluster",
 })
 
+// Here I initialize logger and set some custom settings
+func loggerInit() *zap.SugaredLogger {
+	config := zap.NewDevelopmentConfig()
+	config.EncoderConfig.TimeKey = "timestamp"
+	config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
+	logger, err := config.Build()
+	if err != nil {
+		panic(fmt.Sprintf("Logger set up failed: %v", err))
+	}
+	defer logger.Sync()
+	return logger.Sugar()
+}
+
+// This func runs before main() to get all the things set up before main execution
 func init() {
-	fmt.Println("###Register my metrics in prometheus####")
+	logger = loggerInit()
+	logger.Info("Setting up logger is complete successfully")
+	logger.Info("Registering prom metrics")
 	prometheus.MustRegister(fooMetric)
 	prometheus.MustRegister(barMetric)
 
@@ -28,8 +65,156 @@ func init() {
 	barMetric.Set(1)
 }
 
+// List pools with application rbd enabled
+func listPools() ([]string, error) {
+	results := []string{}
+	command1 := "ceph"
+	args1 := []string{"osd", "pool", "ls", "detail"}
+	command2 := "grep"
+	args2 := []string{"application rbd"}
+	command3 := "awk"
+	args3 := []string{"{print $3}"}
+
+	logger.Infof("Listing pools")
+
+	// This is a pipe conjuction to execute "ceph | grep | awk" pipe
+	p := pipe.Line(
+		pipe.Exec(command1, args1...),
+		pipe.Exec(command2, args2...),
+		pipe.Exec(command3, args3...),
+	)
+
+	output, err := pipe.CombinedOutput(p)
+	if err != nil {
+		panic(err)
+	}
+
+	//Returns iterator
+	lines := strings.SplitSeq(string(output), "\n")
+
+	for v := range lines {
+		v = strings.TrimSpace(v)
+		// awk return result surrounded by single quotes so here I delete it
+		v = strings.Trim(v, "'")
+		// Sometimes here I have an empty string, that's why I check if it is not
+		if v != "" {
+			results = append(results, v)
+			continue
+		}
+	}
+	return results, nil
+}
+
+// List rbd of each pool
+func getRBD(poolList []string) map[string][]string {
+	RBDmap := make(map[string][]string)
+	args := []string{"ls", "-p"}
+
+	// Here I iterate over pool names
+	for _, v := range poolList {
+		var results []string
+		//...and start a command rbd ls -p <pool_name>
+		new_args := append(args, v)
+		p := pipe.Line(
+			pipe.Exec("rbd", new_args...),
+		)
+		output, err := pipe.CombinedOutput(p)
+		if err != nil {
+			panic(err)
+		}
+
+		//Returns iterator
+		lines := strings.SplitSeq(string(output), "\n")
+
+		for i := range lines {
+			//delete whitespaces around
+			i = strings.TrimSpace(i)
+			// check if there is no empty entries
+			if i != "" {
+				results = append(results, i)
+				continue
+			}
+		}
+		RBDmap[v] = results
+	}
+	return RBDmap
+}
+
+// Here I check total provisioned size of each RBD image in a pool
+func RbdChecker(rbdMap map[string][]string) {
+	total := make(map[string][]RBDUsage)
+
+	for pool, rbdlist := range rbdMap {
+		logger.Infof("Processing pool %s", pool)
+
+		for _, rbdName := range rbdlist {
+			total[pool] = append(total[pool],GetRBDStats(pool, rbdName))
+
+		}
+	}
+	fmt.Println(total)
+}
+
+// Grabbing info about specific image
+func GetRBDStats(pool string, rbdname string) RBDUsage {
+	logger.Infof("Calculating %s/%s", pool, rbdname)
+	rbdPath := fmt.Sprintf("%s/%s", pool, rbdname)
+	args := []string{"du", "--format", "json", rbdPath}
+
+	p := pipe.Line(
+		pipe.Exec("rbd", args...),
+	)
+	output, err := pipe.CombinedOutput(p)
+	if err != nil {
+		logger.Fatalf("Error in processing RBD %v", err)
+	}
+
+	var usage RBDUsage
+	if err := json.Unmarshal(output, &usage); err != nil {
+		logger.Fatalf("Error in unmarshaling %v", err)
+	}
+	return usage
+
+}
+
+// the main loop for monitoting
+func startCheking() {
+
+	//Ticks every 5 seconds
+	ticker := time.NewTicker(2 * time.Second)
+	defer ticker.Stop()
+
+	// Every tick I start listPools function
+	for range ticker.C {
+		poolList, err := listPools()
+		if err != nil {
+			logger.Fatalf("Error in listing pools %v", err)
+		}
+		// Get the map of all RBDs in all pools
+		rbdMap := getRBD(poolList)
+		RbdChecker(rbdMap)
+	}
+
+}
 
 func main() {
-	http.Handle("/metrics",promhttp.Handler())
+	defer logger.Sync()
-	log.Fatal(http.ListenAndServe(":9040",nil))
+	http.Handle("/metrics", promhttp.Handler())
+
+	// HTTP runs in separate thread cuz it blocks futher execution of main
+	go func() {
+		logger.Info("Starting http server")
+		// Here I check for errors if HTTP fails
+		if err := http.ListenAndServe(":9040", nil); err != nil {
+			logger.Fatalf("HTTP server failed to start %v", err)
+		}
+		logger.Info("HTTP server started")
+	}()
+
+	go func() {
+		logger.Info("Start checking")
+		startCheking()
+	}()
+
+	select {}
 }

go_exporter/start.sh

@@ -0,0 +1,65 @@
+- /bin/bash
+          - -c
+          - |
+            # Replicate the script from toolbox.sh inline so the ceph image
+            # can be run directly, instead of requiring the rook toolbox
+            CEPH_CONFIG="/etc/ceph/ceph.conf"
+            MON_CONFIG="/etc/rook/mon-endpoints"
+            KEYRING_FILE="/etc/ceph/keyring"
+
+            # create a ceph config file in its default location so ceph/rados tools can be used
+            # without specifying any arguments
+            write_endpoints() {
+              endpoints=$(cat ${MON_CONFIG})
+
+              # filter out the mon names
+              # external cluster can have numbers or hyphens in mon names, handling them in regex
+              # shellcheck disable=SC2001
+              mon_endpoints=$(echo "${endpoints}"| sed 's/[a-z0-9_-]\+=//g')
+
+              DATE=$(date)
+              echo "$DATE writing mon endpoints to ${CEPH_CONFIG}: ${endpoints}"
+                cat <<EOF > ${CEPH_CONFIG}
+            [global]
+            mon_host = ${mon_endpoints}
+
+            [client.admin]
+            keyring = ${KEYRING_FILE}
+            EOF
+            }
+
+            # watch the endpoints config file and update if the mon endpoints ever change
+            watch_endpoints() {
+              # get the timestamp for the target of the soft link
+              real_path=$(realpath ${MON_CONFIG})
+              initial_time=$(stat -c %Z "${real_path}")
+              while true; do
+                real_path=$(realpath ${MON_CONFIG})
+                latest_time=$(stat -c %Z "${real_path}")
+
+                if [[ "${latest_time}" != "${initial_time}" ]]; then
+                  write_endpoints
+                  initial_time=${latest_time}
+                fi
+
+                sleep 10
+              done
+            }
+
+            # read the secret from an env var (for backward compatibility), or from the secret file
+            ceph_secret=${ROOK_CEPH_SECRET}
+            if [[ "$ceph_secret" == "" ]]; then
+              ceph_secret=$(cat /var/lib/rook-ceph-mon/secret.keyring)
+            fi
+
+            # create the keyring file
+            cat <<EOF > ${KEYRING_FILE}
+            [${ROOK_CEPH_USERNAME}]
+            key = ${ceph_secret}
+            EOF
+
+            # write the initial config file
+            write_endpoints
+
+            # continuously update the mon endpoints if they fail over
+            watch_endpoints