diff --git a/ceph_socket/go.mod b/ceph_socket/go.mod new file mode 100644 index 0000000..a20fec6 --- /dev/null +++ b/ceph_socket/go.mod @@ -0,0 +1,5 @@ +module main.go + +go 1.25.4 + +require gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544 // indirect diff --git a/ceph_socket/go.sum b/ceph_socket/go.sum new file mode 100644 index 0000000..f8cc431 --- /dev/null +++ b/ceph_socket/go.sum @@ -0,0 +1,2 @@ +gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544 h1:WJH1qsOB4/zb/li+zLMn0vaAUJ5FqPv6HYLI3aQVg1k= +gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544/go.mod h1:UhTeH/yXCK/KY7TX24mqPkaQ7gZeqmWd/8SSS8B3aHw= diff --git a/ceph_socket/main.go b/ceph_socket/main.go new file mode 100644 index 0000000..daddc72 --- /dev/null +++ b/ceph_socket/main.go @@ -0,0 +1,22 @@ +package main + +import ( + "fmt" + "log" + + "gopkg.in/pipe.v2" +) + +func main() { + args := []string{"25"} + p := pipe.Line( + pipe.ReadFile("go.mod"), + pipe.Exec("grep",args...), + ) + + output, err := pipe.CombinedOutput(p) + if err != nil { + log.Fatal("Shit happened") + } + fmt.Println(string(output)) +} \ No newline at end of file diff --git a/go_exporter/Dockerfile b/go_exporter/Dockerfile index 20eb4bc..7207d66 100644 --- a/go_exporter/Dockerfile +++ b/go_exporter/Dockerfile @@ -7,9 +7,12 @@ RUN go mod download COPY . . RUN CGO_ENABLED=0 GOOS=linux go build -o exporter . -FROM alpine:latest -RUN apk --no-cache add ca-certificates -WORKDIR /root/ +FROM ubuntu:22.04 +RUN apt-get update && apt-get install -y \ + ceph-common \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /home/ COPY --from=builder /app/exporter . EXPOSE 9040 diff --git a/go_exporter/deploy.yaml b/go_exporter/deploy.yaml index 8b7d487..73c79a8 100644 --- a/go_exporter/deploy.yaml +++ b/go_exporter/deploy.yaml @@ -1,21 +1,131 @@ -apiVersion: v1 -kind: Pod +apiVersion: apps/v1 +kind: Deployment metadata: - name: export + name: export-deploy namespace: rook-ceph labels: - app: export - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9040" - prometheus.io/path: "/metrics" + app: export-deploy spec: - containers: - - name: export - image: serviceplant/goexp:0.0.5 - ports: - - containerPort: 9040 - name: metrics + replicas: 1 + selector: + matchLabels: + app: export-box + template: + metadata: + labels: + app: export-box + spec: + containers: + - name: export + image: serviceplant/goexp:0.0.23 + ports: + - containerPort: 9040 + name: metrics + command: + - /bin/bash + - -c + - | + # Replicate the script from toolbox.sh inline so the ceph image + # can be run directly, instead of requiring the rook toolbox + CEPH_CONFIG="/etc/ceph/ceph.conf" + MON_CONFIG="/etc/rook/mon-endpoints" + KEYRING_FILE="/etc/ceph/keyring" + + # create a ceph config file in its default location so ceph/rados tools can be used + # without specifying any arguments + write_endpoints() { + endpoints=$(cat ${MON_CONFIG}) + + # filter out the mon names + # external cluster can have numbers or hyphens in mon names, handling them in regex + # shellcheck disable=SC2001 + mon_endpoints=$(echo "${endpoints}"| sed 's/[a-z0-9_-]\+=//g') + + DATE=$(date) + echo "$DATE writing mon endpoints to ${CEPH_CONFIG}: ${endpoints}" + cat < ${CEPH_CONFIG} + [global] + mon_host = ${mon_endpoints} + + [client.admin] + keyring = ${KEYRING_FILE} + EOF + } + + # watch the endpoints config file and update if the mon endpoints ever change + watch_endpoints() { + # get the timestamp for the target of the soft link + real_path=$(realpath ${MON_CONFIG}) + initial_time=$(stat -c %Z "${real_path}") + while true; do + real_path=$(realpath ${MON_CONFIG}) + latest_time=$(stat -c %Z "${real_path}") + + if [[ "${latest_time}" != "${initial_time}" ]]; then + write_endpoints + initial_time=${latest_time} + fi + + sleep 10 + done + } + + # read the secret from an env var (for backward compatibility), or from the secret file + ceph_secret=${ROOK_CEPH_SECRET} + if [[ "$ceph_secret" == "" ]]; then + ceph_secret=$(cat /var/lib/rook-ceph-mon/secret.keyring) + fi + + # create the keyring file + cat < ${KEYRING_FILE} + [${ROOK_CEPH_USERNAME}] + key = ${ceph_secret} + EOF + + # write the initial config file + write_endpoints + + # continuously update the mon endpoints if they fail over + exec /home/exporter + watch_endpoints & + imagePullPolicy: IfNotPresent + tty: true + securityContext: + runAsNonRoot: true + runAsUser: 2016 + runAsGroup: 2016 + capabilities: + drop: ["ALL"] + env: + - name: ROOK_CEPH_USERNAME + valueFrom: + secretKeyRef: + name: rook-ceph-mon + key: ceph-username + volumeMounts: + - mountPath: /etc/ceph + name: ceph-config + - name: mon-endpoint-volume + mountPath: /etc/rook + - name: ceph-admin-secret + mountPath: /var/lib/rook-ceph-mon + readOnly: true + volumes: + - name: ceph-admin-secret + secret: + secretName: rook-ceph-mon + optional: false + items: + - key: ceph-secret + path: secret.keyring + - name: mon-endpoint-volume + configMap: + name: rook-ceph-mon-endpoints + items: + - key: data + path: mon-endpoints + - name: ceph-config + emptyDir: {} --- apiVersion: v1 kind: Service diff --git a/go_exporter/go.mod b/go_exporter/go.mod index 396282b..1ab399b 100644 --- a/go_exporter/go.mod +++ b/go_exporter/go.mod @@ -2,15 +2,21 @@ module main.go go 1.25.4 +require ( + github.com/prometheus/client_golang v1.23.2 + go.uber.org/zap v1.27.1 +) + require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.66.1 // indirect - github.com/prometheus/procfs v0.16.1 // indirect - go.yaml.in/yaml/v2 v2.4.2 // indirect - golang.org/x/sys v0.35.0 // indirect - google.golang.org/protobuf v1.36.8 // indirect + github.com/prometheus/common v0.67.4 // indirect + github.com/prometheus/procfs v0.19.2 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + golang.org/x/sys v0.38.0 // indirect + google.golang.org/protobuf v1.36.10 // indirect + gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544 // indirect ) diff --git a/go_exporter/go.sum b/go_exporter/go.sum index 6e2f63e..aafdc53 100644 --- a/go_exporter/go.sum +++ b/go_exporter/go.sum @@ -10,12 +10,28 @@ github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNw github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+Lvsc= +github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= +github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544 h1:WJH1qsOB4/zb/li+zLMn0vaAUJ5FqPv6HYLI3aQVg1k= +gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544/go.mod h1:UhTeH/yXCK/KY7TX24mqPkaQ7gZeqmWd/8SSS8B3aHw= diff --git a/go_exporter/goexp b/go_exporter/goexp new file mode 100755 index 0000000..b8b37cf Binary files /dev/null and b/go_exporter/goexp differ diff --git a/go_exporter/main.go b/go_exporter/main.go index 0444d5f..2400ef9 100644 --- a/go_exporter/main.go +++ b/go_exporter/main.go @@ -1,14 +1,35 @@ package main import ( + "encoding/json" "fmt" - "log" "net/http" + _ "os/exec" + "strings" + "time" - promhttp "github.com/prometheus/client_golang/prometheus/promhttp" prometheus "github.com/prometheus/client_golang/prometheus" + promhttp "github.com/prometheus/client_golang/prometheus/promhttp" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + pipe "gopkg.in/pipe.v2" ) +var logger *zap.SugaredLogger + +// Here I store all images info in scope of one pool +type RBDUsage struct { + Images []RBDImageUsage +} + +// Here I do collect info about images itself +type RBDImageUsage struct { + Name string `json:"name"` + id int `json: "id"` + RequestedSize uint64 `json:"provisioned_size"` + UsedSize uint64 `json: "used_size"` +} + var fooMetric = prometheus.NewGauge(prometheus.GaugeOpts{ Name: "foo_metric", Help: "Show whether a foo has occured in a cluster", @@ -19,8 +40,24 @@ var barMetric = prometheus.NewGauge(prometheus.GaugeOpts{ Help: "Show whether a bar has happened in a cluster", }) +// Here I initialize logger and set some custom settings +func loggerInit() *zap.SugaredLogger { + config := zap.NewDevelopmentConfig() + config.EncoderConfig.TimeKey = "timestamp" + config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + logger, err := config.Build() + if err != nil { + panic(fmt.Sprintf("Logger set up failed: %v", err)) + } + defer logger.Sync() + return logger.Sugar() +} + +// This func runs before main() to get all the things set up before main execution func init() { - fmt.Println("###Register my metrics in prometheus####") + logger = loggerInit() + logger.Info("Setting up logger is complete successfully") + logger.Info("Registering prom metrics") prometheus.MustRegister(fooMetric) prometheus.MustRegister(barMetric) @@ -28,8 +65,156 @@ func init() { barMetric.Set(1) } +// List pools with application rbd enabled +func listPools() ([]string, error) { + results := []string{} + command1 := "ceph" + args1 := []string{"osd", "pool", "ls", "detail"} + command2 := "grep" + args2 := []string{"application rbd"} + command3 := "awk" + args3 := []string{"{print $3}"} + + logger.Infof("Listing pools") + + // This is a pipe conjuction to execute "ceph | grep | awk" pipe + p := pipe.Line( + pipe.Exec(command1, args1...), + pipe.Exec(command2, args2...), + pipe.Exec(command3, args3...), + ) + + output, err := pipe.CombinedOutput(p) + if err != nil { + panic(err) + } + + //Returns iterator + lines := strings.SplitSeq(string(output), "\n") + + for v := range lines { + v = strings.TrimSpace(v) + // awk return result surrounded by single quotes so here I delete it + v = strings.Trim(v, "'") + // Sometimes here I have an empty string, that's why I check if it is not + if v != "" { + results = append(results, v) + continue + } + } + return results, nil +} + +// List rbd of each pool +func getRBD(poolList []string) map[string][]string { + RBDmap := make(map[string][]string) + args := []string{"ls", "-p"} + + // Here I iterate over pool names + for _, v := range poolList { + var results []string + //...and start a command rbd ls -p + new_args := append(args, v) + p := pipe.Line( + pipe.Exec("rbd", new_args...), + ) + output, err := pipe.CombinedOutput(p) + if err != nil { + panic(err) + } + + //Returns iterator + lines := strings.SplitSeq(string(output), "\n") + + for i := range lines { + //delete whitespaces around + i = strings.TrimSpace(i) + // check if there is no empty entries + if i != "" { + results = append(results, i) + continue + } + } + RBDmap[v] = results + } + return RBDmap +} + +// Here I check total provisioned size of each RBD image in a pool +func RbdChecker(rbdMap map[string][]string) { + total := make(map[string][]RBDUsage) + + for pool, rbdlist := range rbdMap { + logger.Infof("Processing pool %s", pool) + + for _, rbdName := range rbdlist { + total[pool] = append(total[pool],GetRBDStats(pool, rbdName)) + + } + } + fmt.Println(total) +} + +// Grabbing info about specific image +func GetRBDStats(pool string, rbdname string) RBDUsage { + logger.Infof("Calculating %s/%s", pool, rbdname) + rbdPath := fmt.Sprintf("%s/%s", pool, rbdname) + args := []string{"du", "--format", "json", rbdPath} + + p := pipe.Line( + pipe.Exec("rbd", args...), + ) + output, err := pipe.CombinedOutput(p) + if err != nil { + logger.Fatalf("Error in processing RBD %v", err) + } + + var usage RBDUsage + if err := json.Unmarshal(output, &usage); err != nil { + logger.Fatalf("Error in unmarshaling %v", err) + } + return usage + +} + +// the main loop for monitoting +func startCheking() { + + //Ticks every 5 seconds + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + + // Every tick I start listPools function + for range ticker.C { + poolList, err := listPools() + if err != nil { + logger.Fatalf("Error in listing pools %v", err) + } + // Get the map of all RBDs in all pools + rbdMap := getRBD(poolList) + RbdChecker(rbdMap) + } + +} func main() { - http.Handle("/metrics",promhttp.Handler()) - log.Fatal(http.ListenAndServe(":9040",nil)) + defer logger.Sync() + http.Handle("/metrics", promhttp.Handler()) + + // HTTP runs in separate thread cuz it blocks futher execution of main + go func() { + logger.Info("Starting http server") + // Here I check for errors if HTTP fails + if err := http.ListenAndServe(":9040", nil); err != nil { + logger.Fatalf("HTTP server failed to start %v", err) + } + logger.Info("HTTP server started") + }() + + go func() { + logger.Info("Start checking") + startCheking() + }() + + select {} } diff --git a/go_exporter/start.sh b/go_exporter/start.sh new file mode 100644 index 0000000..5627fa7 --- /dev/null +++ b/go_exporter/start.sh @@ -0,0 +1,65 @@ +- /bin/bash + - -c + - | + # Replicate the script from toolbox.sh inline so the ceph image + # can be run directly, instead of requiring the rook toolbox + CEPH_CONFIG="/etc/ceph/ceph.conf" + MON_CONFIG="/etc/rook/mon-endpoints" + KEYRING_FILE="/etc/ceph/keyring" + + # create a ceph config file in its default location so ceph/rados tools can be used + # without specifying any arguments + write_endpoints() { + endpoints=$(cat ${MON_CONFIG}) + + # filter out the mon names + # external cluster can have numbers or hyphens in mon names, handling them in regex + # shellcheck disable=SC2001 + mon_endpoints=$(echo "${endpoints}"| sed 's/[a-z0-9_-]\+=//g') + + DATE=$(date) + echo "$DATE writing mon endpoints to ${CEPH_CONFIG}: ${endpoints}" + cat < ${CEPH_CONFIG} + [global] + mon_host = ${mon_endpoints} + + [client.admin] + keyring = ${KEYRING_FILE} + EOF + } + + # watch the endpoints config file and update if the mon endpoints ever change + watch_endpoints() { + # get the timestamp for the target of the soft link + real_path=$(realpath ${MON_CONFIG}) + initial_time=$(stat -c %Z "${real_path}") + while true; do + real_path=$(realpath ${MON_CONFIG}) + latest_time=$(stat -c %Z "${real_path}") + + if [[ "${latest_time}" != "${initial_time}" ]]; then + write_endpoints + initial_time=${latest_time} + fi + + sleep 10 + done + } + + # read the secret from an env var (for backward compatibility), or from the secret file + ceph_secret=${ROOK_CEPH_SECRET} + if [[ "$ceph_secret" == "" ]]; then + ceph_secret=$(cat /var/lib/rook-ceph-mon/secret.keyring) + fi + + # create the keyring file + cat < ${KEYRING_FILE} + [${ROOK_CEPH_USERNAME}] + key = ${ceph_secret} + EOF + + # write the initial config file + write_endpoints + + # continuously update the mon endpoints if they fail over + watch_endpoints \ No newline at end of file