new version
This commit is contained in:
a.pivkin
Regular → Executable
@@ -1,20 +0,0 @@
|
||||
FROM golang:1.25-alpine AS builder
|
||||
|
||||
WORKDIR /app
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
COPY . .
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -o exporter .
|
||||
|
||||
FROM ubuntu:22.04
|
||||
RUN apt-get update && apt-get install -y \
|
||||
ceph-common \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /home/
|
||||
|
||||
COPY --from=builder /app/exporter .
|
||||
EXPOSE 9040
|
||||
|
||||
CMD [ "./exporter" ]
|
||||
@@ -0,0 +1,64 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
// "github.com/ceph/go-ceph/rados"
|
||||
"github.com/ceph/go-ceph/rbd"
|
||||
pipe "gopkg.in/pipe.v2"
|
||||
)
|
||||
|
||||
func PoolFactory(cephConn CephConnection, poolName string) (Pool, error) {
|
||||
var rbdlist []RBD = []RBD{}
|
||||
|
||||
ioctx, err := cephConn.conn.OpenIOContext(poolName)
|
||||
if err != nil {
|
||||
return Pool{}, fmt.Errorf("Couldn't set context for pool %s %w", poolName, err)
|
||||
}
|
||||
|
||||
defer ioctx.Destroy()
|
||||
|
||||
imageList, err := rbd.GetImageNames(ioctx)
|
||||
if err != nil {
|
||||
return Pool{}, fmt.Errorf("Couldn't get list of rbds %w", err)
|
||||
}
|
||||
for _, v := range imageList {
|
||||
stat, err := RBDFacroty(poolName, v)
|
||||
if err != nil {
|
||||
fmt.Errorf("Coundn't get stat from disk %s %w", v, err)
|
||||
panic(err)
|
||||
}
|
||||
// fmt.Println(stat)
|
||||
rbdlist = append(rbdlist, stat)
|
||||
}
|
||||
return Pool{
|
||||
Name: poolName,
|
||||
RBDlist: rbdlist,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func RBDFacroty(poolname string, rbdname string) (RBD, error) {
|
||||
|
||||
rbdPath := fmt.Sprintf("%s/%s", poolname, rbdname)
|
||||
args := []string{"du", "--format", "json", rbdPath}
|
||||
|
||||
p := pipe.Line(
|
||||
pipe.Exec("rbd", args...),
|
||||
)
|
||||
output, err := pipe.CombinedOutput(p)
|
||||
if err != nil {
|
||||
fmt.Errorf("Error in processing RBD %v", err)
|
||||
}
|
||||
|
||||
var rbdImage RBDUsage
|
||||
if err := json.Unmarshal(output, &rbdImage); err != nil {
|
||||
fmt.Errorf("Error in unmarshaling %w", err)
|
||||
}
|
||||
return RBD{
|
||||
Name: rbdImage.Images[0].Name,
|
||||
Id: rbdImage.Images[0].Id,
|
||||
RequestedSize: rbdImage.Images[0].RequestedSize,
|
||||
UsedSize: rbdImage.Images[0].UsedSize,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/ceph/go-ceph/rados"
|
||||
)
|
||||
|
||||
func connect() (_ CephConnection, err error) {
|
||||
var cephConn CephConnection = CephConnection{}
|
||||
|
||||
defer func() {
|
||||
if err != nil {
|
||||
err = fmt.Errorf("Error in func connect() %w", err)
|
||||
}
|
||||
}()
|
||||
|
||||
cephConn.conn, err = rados.NewConnWithClusterAndUser("ceph", "client.admin")
|
||||
if err != nil {
|
||||
return CephConnection{}, err
|
||||
}
|
||||
|
||||
err = cephConn.conn.ReadConfigFile(params.config)
|
||||
if err != nil {
|
||||
return CephConnection{}, err
|
||||
}
|
||||
|
||||
err = cephConn.conn.SetConfigOption("keyring", params.keyring)
|
||||
if err != nil {
|
||||
return CephConnection{}, err
|
||||
}
|
||||
|
||||
err = cephConn.conn.Connect()
|
||||
if err != nil {
|
||||
return CephConnection{}, err
|
||||
}
|
||||
|
||||
return cephConn, nil
|
||||
}
|
||||
@@ -1,151 +0,0 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: export-deploy
|
||||
namespace: rook-ceph
|
||||
labels:
|
||||
app: export-deploy
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "9040"
|
||||
prometheus.io/path: "/metrics"
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: export-box
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: export-box
|
||||
spec:
|
||||
containers:
|
||||
- name: export
|
||||
image: serviceplant/goexp:0.0.28
|
||||
ports:
|
||||
- containerPort: 9040
|
||||
name: metrics
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
# Replicate the script from toolbox.sh inline so the ceph image
|
||||
# can be run directly, instead of requiring the rook toolbox
|
||||
CEPH_CONFIG="/etc/ceph/ceph.conf"
|
||||
MON_CONFIG="/etc/rook/mon-endpoints"
|
||||
KEYRING_FILE="/etc/ceph/keyring"
|
||||
|
||||
# create a ceph config file in its default location so ceph/rados tools can be used
|
||||
# without specifying any arguments
|
||||
write_endpoints() {
|
||||
endpoints=$(cat ${MON_CONFIG})
|
||||
|
||||
# filter out the mon names
|
||||
# external cluster can have numbers or hyphens in mon names, handling them in regex
|
||||
# shellcheck disable=SC2001
|
||||
mon_endpoints=$(echo "${endpoints}"| sed 's/[a-z0-9_-]\+=//g')
|
||||
|
||||
DATE=$(date)
|
||||
echo "$DATE writing mon endpoints to ${CEPH_CONFIG}: ${endpoints}"
|
||||
cat <<EOF > ${CEPH_CONFIG}
|
||||
[global]
|
||||
mon_host = ${mon_endpoints}
|
||||
|
||||
[client.admin]
|
||||
keyring = ${KEYRING_FILE}
|
||||
EOF
|
||||
}
|
||||
|
||||
# watch the endpoints config file and update if the mon endpoints ever change
|
||||
watch_endpoints() {
|
||||
# get the timestamp for the target of the soft link
|
||||
real_path=$(realpath ${MON_CONFIG})
|
||||
initial_time=$(stat -c %Z "${real_path}")
|
||||
while true; do
|
||||
echo "I am watching!!!"
|
||||
real_path=$(realpath ${MON_CONFIG})
|
||||
latest_time=$(stat -c %Z "${real_path}")
|
||||
|
||||
if [[ "${latest_time}" != "${initial_time}" ]]; then
|
||||
write_endpoints
|
||||
initial_time=${latest_time}
|
||||
fi
|
||||
|
||||
sleep 10
|
||||
done
|
||||
}
|
||||
|
||||
# read the secret from an env var (for backward compatibility), or from the secret file
|
||||
ceph_secret=${ROOK_CEPH_SECRET}
|
||||
if [[ "$ceph_secret" == "" ]]; then
|
||||
ceph_secret=$(cat /var/lib/rook-ceph-mon/secret.keyring)
|
||||
fi
|
||||
|
||||
# create the keyring file
|
||||
cat <<EOF > ${KEYRING_FILE}
|
||||
[${ROOK_CEPH_USERNAME}]
|
||||
key = ${ceph_secret}
|
||||
EOF
|
||||
|
||||
# write the initial config file
|
||||
write_endpoints
|
||||
|
||||
# continuously update the mon endpoints if they fail over
|
||||
exec /home/exporter &
|
||||
watch_endpoints
|
||||
imagePullPolicy: IfNotPresent
|
||||
tty: true
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 2016
|
||||
runAsGroup: 2016
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
env:
|
||||
- name: ROOK_CEPH_USERNAME
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: rook-ceph-mon
|
||||
key: ceph-username
|
||||
volumeMounts:
|
||||
- mountPath: /etc/ceph
|
||||
name: ceph-config
|
||||
- name: mon-endpoint-volume
|
||||
mountPath: /etc/rook
|
||||
- name: ceph-admin-secret
|
||||
mountPath: /var/lib/rook-ceph-mon
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: ceph-admin-secret
|
||||
secret:
|
||||
secretName: rook-ceph-mon
|
||||
optional: false
|
||||
items:
|
||||
- key: ceph-secret
|
||||
path: secret.keyring
|
||||
- name: mon-endpoint-volume
|
||||
configMap:
|
||||
name: rook-ceph-mon-endpoints
|
||||
items:
|
||||
- key: data
|
||||
path: mon-endpoints
|
||||
- name: ceph-config
|
||||
emptyDir: {}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: export-service
|
||||
namespace: rook-ceph
|
||||
labels:
|
||||
svc_app: export
|
||||
rook_cluster: rook-ceph
|
||||
spec:
|
||||
selector:
|
||||
app: export-box
|
||||
ports:
|
||||
- name: metrics
|
||||
port: 9040
|
||||
targetPort: 9040
|
||||
protocol: TCP
|
||||
type: ClusterIP
|
||||
@@ -0,0 +1,16 @@
|
||||
package main
|
||||
|
||||
import "flag"
|
||||
|
||||
var params Params
|
||||
|
||||
// This func runs even before main()
|
||||
func init() {
|
||||
|
||||
config_file := flag.String("config", "/etc/ceph/ceph.conf", "placement of ceph config file")
|
||||
_keyring := flag.String("keyring", "/etc/ceph/ceph.client.admin.keyring", "placement of ceph keyring file")
|
||||
flag.Parse()
|
||||
|
||||
params.config = *config_file
|
||||
params.keyring = *_keyring
|
||||
}
|
||||
+5
-14
@@ -1,22 +1,13 @@
|
||||
module main.go
|
||||
|
||||
go 1.25.4
|
||||
go 1.25.5
|
||||
|
||||
require (
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
go.uber.org/zap v1.27.1
|
||||
github.com/ceph/go-ceph v0.36.0
|
||||
gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.67.4 // indirect
|
||||
github.com/prometheus/procfs v0.19.2 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
golang.org/x/sys v0.38.0 // indirect
|
||||
google.golang.org/protobuf v1.36.10 // indirect
|
||||
gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544 // indirect
|
||||
golang.org/x/sys v0.36.0 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
||||
)
|
||||
|
||||
+21
-35
@@ -1,37 +1,23 @@
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
|
||||
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
|
||||
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
||||
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
|
||||
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
|
||||
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
|
||||
github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+Lvsc=
|
||||
github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI=
|
||||
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
|
||||
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
|
||||
github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws=
|
||||
github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
|
||||
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc=
|
||||
go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
|
||||
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
|
||||
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
|
||||
go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
|
||||
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
|
||||
golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
|
||||
golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
|
||||
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
|
||||
google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
|
||||
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
|
||||
google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
github.com/ceph/go-ceph v0.36.0 h1:IDE4vEF+4fmjve+CPjD1WStgfQ+Lh6vD+9PMUI712KI=
|
||||
github.com/ceph/go-ceph v0.36.0/go.mod h1:fGCbndVDLuHW7q2954d6y+tgPFOBnRLqJRe2YXyngw4=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/gofrs/uuid/v5 v5.3.2 h1:2jfO8j3XgSwlz/wHqemAEugfnTlikAYHhnqQ8Xh4fE0=
|
||||
github.com/gofrs/uuid/v5 v5.3.2/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8=
|
||||
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
|
||||
golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544 h1:WJH1qsOB4/zb/li+zLMn0vaAUJ5FqPv6HYLI3aQVg1k=
|
||||
gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544/go.mod h1:UhTeH/yXCK/KY7TX24mqPkaQ7gZeqmWd/8SSS8B3aHw=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
Binary file not shown.
Executable
BIN
Binary file not shown.
+22
-241
@@ -1,252 +1,33 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
_ "os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
prometheus "github.com/prometheus/client_golang/prometheus"
|
||||
promhttp "github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zapcore"
|
||||
pipe "gopkg.in/pipe.v2"
|
||||
"os"
|
||||
)
|
||||
|
||||
var logger *zap.SugaredLogger
|
||||
|
||||
// Here I store all images info in scope of one pool
|
||||
type RBDUsage struct {
|
||||
Images []RBDImageUsage
|
||||
}
|
||||
|
||||
// Here I do collect info about images itself
|
||||
type RBDImageUsage struct {
|
||||
Name string `json:"name"`
|
||||
id int `json: "id"`
|
||||
RequestedSize uint64 `json:"provisioned_size"`
|
||||
UsedSize uint64 `json: "used_size"`
|
||||
}
|
||||
|
||||
var total_rbd_requested_size_per_pool = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Name: "total_rbd_requested_size_per_pool",
|
||||
Help: "total size of all requested RBDs in a specific pool",
|
||||
},
|
||||
[]string{"poolname"},
|
||||
)
|
||||
|
||||
var total_rbd_requested_size = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Name: "total_rbd_requested_size",
|
||||
Help: "total size of all RBDs in the cluster",
|
||||
})
|
||||
|
||||
// Here I initialize logger and set some custom settings
|
||||
func loggerInit() *zap.SugaredLogger {
|
||||
config := zap.NewDevelopmentConfig()
|
||||
config.EncoderConfig.TimeKey = "timestamp"
|
||||
config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
|
||||
logger, err := config.Build()
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Logger set up failed: %v", err))
|
||||
}
|
||||
defer logger.Sync()
|
||||
return logger.Sugar()
|
||||
}
|
||||
|
||||
// This func runs before main() to get all the things set up before main execution
|
||||
func init() {
|
||||
logger = loggerInit()
|
||||
logger.Info("Setting up logger is complete successfully")
|
||||
logger.Info("Registering prom metrics")
|
||||
|
||||
prometheus.MustRegister(
|
||||
total_rbd_requested_size_per_pool,
|
||||
total_rbd_requested_size,
|
||||
)
|
||||
|
||||
}
|
||||
|
||||
// List pools with application rbd enabled
|
||||
func listPools() ([]string, error) {
|
||||
results := []string{}
|
||||
command1 := "ceph"
|
||||
args1 := []string{"osd", "pool", "ls", "detail"}
|
||||
command2 := "grep"
|
||||
args2 := []string{"application rbd"}
|
||||
command3 := "awk"
|
||||
args3 := []string{"{print $3}"}
|
||||
|
||||
logger.Infof("Listing pools")
|
||||
|
||||
// This is a pipe conjuction to execute "ceph | grep | awk" pipe
|
||||
p := pipe.Line(
|
||||
pipe.Exec(command1, args1...),
|
||||
pipe.Exec(command2, args2...),
|
||||
pipe.Exec(command3, args3...),
|
||||
)
|
||||
|
||||
output, err := pipe.CombinedOutput(p)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
//Returns iterator
|
||||
lines := strings.SplitSeq(string(output), "\n")
|
||||
|
||||
for v := range lines {
|
||||
v = strings.TrimSpace(v)
|
||||
// awk return result surrounded by single quotes so here I delete it
|
||||
v = strings.Trim(v, "'")
|
||||
// Sometimes here I have an empty string, that's why I check if it is not
|
||||
if v != "" {
|
||||
results = append(results, v)
|
||||
continue
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// List rbd of each pool
|
||||
func getRBD(poolList []string) map[string][]string {
|
||||
RBDmap := make(map[string][]string)
|
||||
args := []string{"ls", "-p"}
|
||||
|
||||
// Here I iterate over pool names
|
||||
for _, v := range poolList {
|
||||
var results []string
|
||||
//...and start a command rbd ls -p <pool_name>
|
||||
new_args := append(args, v)
|
||||
p := pipe.Line(
|
||||
pipe.Exec("rbd", new_args...),
|
||||
)
|
||||
output, err := pipe.CombinedOutput(p)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
//Returns iterator
|
||||
lines := strings.SplitSeq(string(output), "\n")
|
||||
|
||||
for i := range lines {
|
||||
//delete whitespaces around
|
||||
i = strings.TrimSpace(i)
|
||||
// check if there is no empty entries
|
||||
if i != "" {
|
||||
results = append(results, i)
|
||||
continue
|
||||
}
|
||||
}
|
||||
RBDmap[v] = results
|
||||
}
|
||||
return RBDmap
|
||||
}
|
||||
|
||||
// Here I check total provisioned size of each RBD image in a pool
|
||||
func RbdChecker(rbdMap map[string][]string) map[string][]RBDUsage{
|
||||
total := make(map[string][]RBDUsage)
|
||||
|
||||
for pool, rbdlist := range rbdMap {
|
||||
logger.Infof("Processing pool %s", pool)
|
||||
|
||||
for _, rbdName := range rbdlist {
|
||||
total[pool] = append(total[pool],GetRBDStats(pool, rbdName))
|
||||
|
||||
}
|
||||
}
|
||||
logger.Debugf("Final map is %v",total)
|
||||
return total
|
||||
}
|
||||
|
||||
// Grabbing info about specific image
|
||||
func GetRBDStats(pool string, rbdname string) RBDUsage {
|
||||
rbdPath := fmt.Sprintf("%s/%s", pool, rbdname)
|
||||
args := []string{"du", "--format", "json", rbdPath}
|
||||
|
||||
p := pipe.Line(
|
||||
pipe.Exec("rbd", args...),
|
||||
)
|
||||
output, err := pipe.CombinedOutput(p)
|
||||
if err != nil {
|
||||
logger.Fatalf("Error in processing RBD %v", err)
|
||||
}
|
||||
|
||||
var usage RBDUsage
|
||||
if err := json.Unmarshal(output, &usage); err != nil {
|
||||
logger.Fatalf("Error in unmarshaling %v", err)
|
||||
}
|
||||
return usage
|
||||
|
||||
}
|
||||
|
||||
func FormMetrirs(rbdStats map[string][]RBDUsage) {
|
||||
// this is requested size overall cluster
|
||||
var totalSize uint64 = 0
|
||||
|
||||
// Iterate over pools
|
||||
for poolName := range rbdStats {
|
||||
var totalSizePerPool uint64 = 0
|
||||
logger.Debugf("Forming metrics for pool %s",poolName)
|
||||
|
||||
//Iterate over all RBDs in the pool
|
||||
for _,rbdName := range rbdStats[poolName] {
|
||||
logger.Debugf("Processings rbd %v",rbdName)
|
||||
totalSizePerPool = totalSizePerPool + rbdName.Images[0].RequestedSize
|
||||
// logger.Debugf("RBD name is %s and its size is %d",rbdName.Images[0].Name,rbdName.Images[0].RequestedSize)
|
||||
}
|
||||
logger.Debugf("Total size requested by RBDs of a pool %s is %d bytes",poolName,totalSizePerPool)
|
||||
|
||||
total_rbd_requested_size_per_pool.WithLabelValues(
|
||||
poolName,
|
||||
).Set(float64(totalSizePerPool))
|
||||
totalSize = totalSize + totalSizePerPool
|
||||
}
|
||||
logger.Debugf("Total size of all RBDs in a cluster is %d",totalSize)
|
||||
total_rbd_requested_size.Set(float64(totalSize))
|
||||
}
|
||||
|
||||
// the main loop for monitoting
|
||||
func startCheking() {
|
||||
|
||||
//Ticks every 5 seconds
|
||||
ticker := time.NewTicker(2 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
// Every tick I start listPools function
|
||||
for range ticker.C {
|
||||
poolList, err := listPools()
|
||||
if err != nil {
|
||||
logger.Fatalf("Error in listing pools %v", err)
|
||||
}
|
||||
// Get the map of all RBDs in all pools
|
||||
rbdMap := getRBD(poolList)
|
||||
//Get all RBDs info
|
||||
RBDStats := RbdChecker(rbdMap)
|
||||
//Fill out metrics
|
||||
FormMetrirs(RBDStats)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func main() {
|
||||
defer logger.Sync()
|
||||
http.Handle("/metrics", promhttp.Handler())
|
||||
|
||||
// HTTP runs in separate thread cuz it blocks futher execution of main
|
||||
go func() {
|
||||
logger.Info("Starting http server")
|
||||
// Here I check for errors if HTTP fails
|
||||
if err := http.ListenAndServe(":9040", nil); err != nil {
|
||||
logger.Fatalf("HTTP server failed to start %v", err)
|
||||
var result []Pool = []Pool{}
|
||||
cephConn, err := connect()
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
if wrapped := errors.Unwrap(err); wrapped != nil {
|
||||
fmt.Println(wrapped)
|
||||
}
|
||||
logger.Info("HTTP server started")
|
||||
}()
|
||||
os.Exit(1)
|
||||
}
|
||||
defer cephConn.conn.Shutdown()
|
||||
fmt.Println("Successfully connected")
|
||||
|
||||
go func() {
|
||||
logger.Info("Start checking")
|
||||
startCheking()
|
||||
}()
|
||||
poolList, err := cephConn.conn.ListPools()
|
||||
if err != nil {
|
||||
errors.New("Cannot get list of pools")
|
||||
}
|
||||
|
||||
for _, v := range poolList {
|
||||
x, _ := PoolFactory(cephConn, v)
|
||||
result = append(result, x)
|
||||
}
|
||||
fmt.Println(result)
|
||||
|
||||
select {}
|
||||
}
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: export-monitor
|
||||
namespace: rook-ceph
|
||||
labels:
|
||||
team: rook
|
||||
spec:
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- rook-ceph
|
||||
selector:
|
||||
matchLabels:
|
||||
svc_app: export
|
||||
rook_cluster: rook-ceph
|
||||
endpoints:
|
||||
- port: metrics
|
||||
interval: 30s
|
||||
path: /metrics
|
||||
honorLabels: true
|
||||
@@ -1,65 +0,0 @@
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
# Replicate the script from toolbox.sh inline so the ceph image
|
||||
# can be run directly, instead of requiring the rook toolbox
|
||||
CEPH_CONFIG="/etc/ceph/ceph.conf"
|
||||
MON_CONFIG="/etc/rook/mon-endpoints"
|
||||
KEYRING_FILE="/etc/ceph/keyring"
|
||||
|
||||
# create a ceph config file in its default location so ceph/rados tools can be used
|
||||
# without specifying any arguments
|
||||
write_endpoints() {
|
||||
endpoints=$(cat ${MON_CONFIG})
|
||||
|
||||
# filter out the mon names
|
||||
# external cluster can have numbers or hyphens in mon names, handling them in regex
|
||||
# shellcheck disable=SC2001
|
||||
mon_endpoints=$(echo "${endpoints}"| sed 's/[a-z0-9_-]\+=//g')
|
||||
|
||||
DATE=$(date)
|
||||
echo "$DATE writing mon endpoints to ${CEPH_CONFIG}: ${endpoints}"
|
||||
cat <<EOF > ${CEPH_CONFIG}
|
||||
[global]
|
||||
mon_host = ${mon_endpoints}
|
||||
|
||||
[client.admin]
|
||||
keyring = ${KEYRING_FILE}
|
||||
EOF
|
||||
}
|
||||
|
||||
# watch the endpoints config file and update if the mon endpoints ever change
|
||||
watch_endpoints() {
|
||||
# get the timestamp for the target of the soft link
|
||||
real_path=$(realpath ${MON_CONFIG})
|
||||
initial_time=$(stat -c %Z "${real_path}")
|
||||
while true; do
|
||||
real_path=$(realpath ${MON_CONFIG})
|
||||
latest_time=$(stat -c %Z "${real_path}")
|
||||
|
||||
if [[ "${latest_time}" != "${initial_time}" ]]; then
|
||||
write_endpoints
|
||||
initial_time=${latest_time}
|
||||
fi
|
||||
|
||||
sleep 10
|
||||
done
|
||||
}
|
||||
|
||||
# read the secret from an env var (for backward compatibility), or from the secret file
|
||||
ceph_secret=${ROOK_CEPH_SECRET}
|
||||
if [[ "$ceph_secret" == "" ]]; then
|
||||
ceph_secret=$(cat /var/lib/rook-ceph-mon/secret.keyring)
|
||||
fi
|
||||
|
||||
# create the keyring file
|
||||
cat <<EOF > ${KEYRING_FILE}
|
||||
[${ROOK_CEPH_USERNAME}]
|
||||
key = ${ceph_secret}
|
||||
EOF
|
||||
|
||||
# write the initial config file
|
||||
write_endpoints
|
||||
|
||||
# continuously update the mon endpoints if they fail over
|
||||
watch_endpoints
|
||||
@@ -0,0 +1,42 @@
|
||||
package main
|
||||
|
||||
import "github.com/ceph/go-ceph/rados"
|
||||
|
||||
type Params struct {
|
||||
config string
|
||||
keyring string
|
||||
}
|
||||
|
||||
type CephConnection struct {
|
||||
conn *rados.Conn
|
||||
}
|
||||
|
||||
type iRBD interface {
|
||||
getName() string
|
||||
getSize() int64
|
||||
}
|
||||
|
||||
type RBDUsage struct {
|
||||
Images []RBD `json:"images"`
|
||||
}
|
||||
|
||||
type RBD struct {
|
||||
Name string `json:"name"`
|
||||
Id string `json:"id"`
|
||||
RequestedSize uint64 `json:"provisioned_size"`
|
||||
UsedSize int64 `json:"used_size"`
|
||||
}
|
||||
|
||||
func (r RBD) getName() string {
|
||||
return r.Name
|
||||
}
|
||||
|
||||
func (r RBD) getSize() int64 {
|
||||
return r.UsedSize
|
||||
}
|
||||
|
||||
type Pool struct {
|
||||
Name string
|
||||
RBDlist []RBD
|
||||
// RBDlist []string
|
||||
}
|
||||
Reference in New Issue
Block a user