diff --git a/go_exporter/.fuse_hidden0000000d00000001 b/go_exporter/.fuse_hidden0000000d00000001 old mode 100644 new mode 100755 diff --git a/go_exporter/.fuse_hidden0000001000000002 b/go_exporter/.fuse_hidden0000001000000002 new file mode 100755 index 0000000..e69de29 diff --git a/go_exporter/.fuse_hidden0000001300000003 b/go_exporter/.fuse_hidden0000001300000003 new file mode 100755 index 0000000..e69de29 diff --git a/go_exporter/.fuse_hidden0000001600000004 b/go_exporter/.fuse_hidden0000001600000004 new file mode 100755 index 0000000..e69de29 diff --git a/go_exporter/.fuse_hidden0000001b00000005 b/go_exporter/.fuse_hidden0000001b00000005 new file mode 100755 index 0000000..e69de29 diff --git a/go_exporter/.fuse_hidden0000001e00000006 b/go_exporter/.fuse_hidden0000001e00000006 new file mode 100755 index 0000000..e69de29 diff --git a/go_exporter/.fuse_hidden0000002300000007 b/go_exporter/.fuse_hidden0000002300000007 new file mode 100755 index 0000000..e69de29 diff --git a/go_exporter/.fuse_hidden0000002800000008 b/go_exporter/.fuse_hidden0000002800000008 new file mode 100755 index 0000000..e69de29 diff --git a/go_exporter/.fuse_hidden0000002d00000009 b/go_exporter/.fuse_hidden0000002d00000009 new file mode 100755 index 0000000..e69de29 diff --git a/go_exporter/RBDFactory.go b/go_exporter/RBDFactory.go new file mode 100644 index 0000000..1808b01 --- /dev/null +++ b/go_exporter/RBDFactory.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "fmt" + + // "github.com/ceph/go-ceph/rados" + "github.com/ceph/go-ceph/rbd" + pipe "gopkg.in/pipe.v2" +) + +func PoolFactory(cephConn CephConnection, poolName string) (Pool, error) { + var rbdlist []RBD = []RBD{} + + ioctx, err := cephConn.conn.OpenIOContext(poolName) + if err != nil { + return Pool{}, fmt.Errorf("Couldn't set context for pool %s %w", poolName, err) + } + + defer ioctx.Destroy() + + imageList, err := rbd.GetImageNames(ioctx) + if err != nil { + return Pool{}, fmt.Errorf("Couldn't get list of rbds %w", err) + } + for _, v := range imageList { + stat, err := RBDFacroty(poolName, v) + if err != nil { + fmt.Errorf("Coundn't get stat from disk %s %w", v, err) + panic(err) + } + // fmt.Println(stat) + rbdlist = append(rbdlist, stat) + } + return Pool{ + Name: poolName, + RBDlist: rbdlist, + }, nil +} + +func RBDFacroty(poolname string, rbdname string) (RBD, error) { + + rbdPath := fmt.Sprintf("%s/%s", poolname, rbdname) + args := []string{"du", "--format", "json", rbdPath} + + p := pipe.Line( + pipe.Exec("rbd", args...), + ) + output, err := pipe.CombinedOutput(p) + if err != nil { + fmt.Errorf("Error in processing RBD %v", err) + } + + var rbdImage RBDUsage + if err := json.Unmarshal(output, &rbdImage); err != nil { + fmt.Errorf("Error in unmarshaling %w", err) + } + return RBD{ + Name: rbdImage.Images[0].Name, + Id: rbdImage.Images[0].Id, + RequestedSize: rbdImage.Images[0].RequestedSize, + UsedSize: rbdImage.Images[0].UsedSize, + }, nil +} \ No newline at end of file diff --git a/go_exporter/connection.go b/go_exporter/connection.go new file mode 100644 index 0000000..083ff67 --- /dev/null +++ b/go_exporter/connection.go @@ -0,0 +1,39 @@ +package main + +import ( + "fmt" + + "github.com/ceph/go-ceph/rados" +) + +func connect() (_ CephConnection, err error) { + var cephConn CephConnection = CephConnection{} + + defer func() { + if err != nil { + err = fmt.Errorf("Error in func connect() %w", err) + } + }() + + cephConn.conn, err = rados.NewConnWithClusterAndUser("ceph", "client.admin") + if err != nil { + return CephConnection{}, err + } + + err = cephConn.conn.ReadConfigFile(params.config) + if err != nil { + return CephConnection{}, err + } + + err = cephConn.conn.SetConfigOption("keyring", params.keyring) + if err != nil { + return CephConnection{}, err + } + + err = cephConn.conn.Connect() + if err != nil { + return CephConnection{}, err + } + + return cephConn, nil +} diff --git a/go_exporter/getOpts.go b/go_exporter/getOpts.go new file mode 100644 index 0000000..14df4fc --- /dev/null +++ b/go_exporter/getOpts.go @@ -0,0 +1,16 @@ +package main + +import "flag" + +var params Params + +// This func runs even before main() +func init() { + + config_file := flag.String("config", "/etc/ceph/ceph.conf", "placement of ceph config file") + _keyring := flag.String("keyring", "/etc/ceph/ceph.client.admin.keyring", "placement of ceph keyring file") + flag.Parse() + + params.config = *config_file + params.keyring = *_keyring +} diff --git a/go_exporter/go.mod b/go_exporter/go.mod index 1ab399b..749493c 100644 --- a/go_exporter/go.mod +++ b/go_exporter/go.mod @@ -1,22 +1,13 @@ module main.go -go 1.25.4 +go 1.25.5 require ( - github.com/prometheus/client_golang v1.23.2 - go.uber.org/zap v1.27.1 + github.com/ceph/go-ceph v0.36.0 + gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544 ) require ( - github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.67.4 // indirect - github.com/prometheus/procfs v0.19.2 // indirect - go.uber.org/multierr v1.11.0 // indirect - go.yaml.in/yaml/v2 v2.4.3 // indirect - golang.org/x/sys v0.38.0 // indirect - google.golang.org/protobuf v1.36.10 // indirect - gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544 // indirect + golang.org/x/sys v0.36.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go_exporter/go.sum b/go_exporter/go.sum index aafdc53..13bb673 100644 --- a/go_exporter/go.sum +++ b/go_exporter/go.sum @@ -1,37 +1,23 @@ -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= -github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= -github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= -github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= -github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= -github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+Lvsc= -github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI= -github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= -github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= -github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= -github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= -go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= -go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= -go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= -go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= -golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= -golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= -google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= -google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= -google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +github.com/ceph/go-ceph v0.36.0 h1:IDE4vEF+4fmjve+CPjD1WStgfQ+Lh6vD+9PMUI712KI= +github.com/ceph/go-ceph v0.36.0/go.mod h1:fGCbndVDLuHW7q2954d6y+tgPFOBnRLqJRe2YXyngw4= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gofrs/uuid/v5 v5.3.2 h1:2jfO8j3XgSwlz/wHqemAEugfnTlikAYHhnqQ8Xh4fE0= +github.com/gofrs/uuid/v5 v5.3.2/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544 h1:WJH1qsOB4/zb/li+zLMn0vaAUJ5FqPv6HYLI3aQVg1k= gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544/go.mod h1:UhTeH/yXCK/KY7TX24mqPkaQ7gZeqmWd/8SSS8B3aHw= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go_exporter/kek b/go_exporter/kek new file mode 100755 index 0000000..1105778 Binary files /dev/null and b/go_exporter/kek differ diff --git a/go_exporter/main.go b/go_exporter/main.go index 34b097c..2d603c5 100644 --- a/go_exporter/main.go +++ b/go_exporter/main.go @@ -1,252 +1,33 @@ package main import ( - "encoding/json" + "errors" "fmt" - "net/http" - _ "os/exec" - "strings" - "time" - - prometheus "github.com/prometheus/client_golang/prometheus" - promhttp "github.com/prometheus/client_golang/prometheus/promhttp" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" - pipe "gopkg.in/pipe.v2" + "os" ) -var logger *zap.SugaredLogger - -// Here I store all images info in scope of one pool -type RBDUsage struct { - Images []RBDImageUsage -} - -// Here I do collect info about images itself -type RBDImageUsage struct { - Name string `json:"name"` - id int `json: "id"` - RequestedSize uint64 `json:"provisioned_size"` - UsedSize uint64 `json: "used_size"` -} - -var total_rbd_requested_size_per_pool = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "total_rbd_requested_size_per_pool", - Help: "total size of all requested RBDs in a specific pool", - }, - []string{"poolname"}, -) - -var total_rbd_requested_size = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "total_rbd_requested_size", - Help: "total size of all RBDs in the cluster", -}) - -// Here I initialize logger and set some custom settings -func loggerInit() *zap.SugaredLogger { - config := zap.NewDevelopmentConfig() - config.EncoderConfig.TimeKey = "timestamp" - config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder - logger, err := config.Build() - if err != nil { - panic(fmt.Sprintf("Logger set up failed: %v", err)) - } - defer logger.Sync() - return logger.Sugar() -} - -// This func runs before main() to get all the things set up before main execution -func init() { - logger = loggerInit() - logger.Info("Setting up logger is complete successfully") - logger.Info("Registering prom metrics") - - prometheus.MustRegister( - total_rbd_requested_size_per_pool, - total_rbd_requested_size, - ) - -} - -// List pools with application rbd enabled -func listPools() ([]string, error) { - results := []string{} - command1 := "ceph" - args1 := []string{"osd", "pool", "ls", "detail"} - command2 := "grep" - args2 := []string{"application rbd"} - command3 := "awk" - args3 := []string{"{print $3}"} - - logger.Infof("Listing pools") - - // This is a pipe conjuction to execute "ceph | grep | awk" pipe - p := pipe.Line( - pipe.Exec(command1, args1...), - pipe.Exec(command2, args2...), - pipe.Exec(command3, args3...), - ) - - output, err := pipe.CombinedOutput(p) - if err != nil { - panic(err) - } - - //Returns iterator - lines := strings.SplitSeq(string(output), "\n") - - for v := range lines { - v = strings.TrimSpace(v) - // awk return result surrounded by single quotes so here I delete it - v = strings.Trim(v, "'") - // Sometimes here I have an empty string, that's why I check if it is not - if v != "" { - results = append(results, v) - continue - } - } - return results, nil -} - -// List rbd of each pool -func getRBD(poolList []string) map[string][]string { - RBDmap := make(map[string][]string) - args := []string{"ls", "-p"} - - // Here I iterate over pool names - for _, v := range poolList { - var results []string - //...and start a command rbd ls -p - new_args := append(args, v) - p := pipe.Line( - pipe.Exec("rbd", new_args...), - ) - output, err := pipe.CombinedOutput(p) - if err != nil { - panic(err) - } - - //Returns iterator - lines := strings.SplitSeq(string(output), "\n") - - for i := range lines { - //delete whitespaces around - i = strings.TrimSpace(i) - // check if there is no empty entries - if i != "" { - results = append(results, i) - continue - } - } - RBDmap[v] = results - } - return RBDmap -} - -// Here I check total provisioned size of each RBD image in a pool -func RbdChecker(rbdMap map[string][]string) map[string][]RBDUsage{ - total := make(map[string][]RBDUsage) - - for pool, rbdlist := range rbdMap { - logger.Infof("Processing pool %s", pool) - - for _, rbdName := range rbdlist { - total[pool] = append(total[pool],GetRBDStats(pool, rbdName)) - - } - } - logger.Debugf("Final map is %v",total) - return total -} - -// Grabbing info about specific image -func GetRBDStats(pool string, rbdname string) RBDUsage { - rbdPath := fmt.Sprintf("%s/%s", pool, rbdname) - args := []string{"du", "--format", "json", rbdPath} - - p := pipe.Line( - pipe.Exec("rbd", args...), - ) - output, err := pipe.CombinedOutput(p) - if err != nil { - logger.Fatalf("Error in processing RBD %v", err) - } - - var usage RBDUsage - if err := json.Unmarshal(output, &usage); err != nil { - logger.Fatalf("Error in unmarshaling %v", err) - } - return usage - -} - -func FormMetrirs(rbdStats map[string][]RBDUsage) { - // this is requested size overall cluster - var totalSize uint64 = 0 - - // Iterate over pools - for poolName := range rbdStats { - var totalSizePerPool uint64 = 0 - logger.Debugf("Forming metrics for pool %s",poolName) - - //Iterate over all RBDs in the pool - for _,rbdName := range rbdStats[poolName] { - logger.Debugf("Processings rbd %v",rbdName) - totalSizePerPool = totalSizePerPool + rbdName.Images[0].RequestedSize - // logger.Debugf("RBD name is %s and its size is %d",rbdName.Images[0].Name,rbdName.Images[0].RequestedSize) - } - logger.Debugf("Total size requested by RBDs of a pool %s is %d bytes",poolName,totalSizePerPool) - - total_rbd_requested_size_per_pool.WithLabelValues( - poolName, - ).Set(float64(totalSizePerPool)) - totalSize = totalSize + totalSizePerPool - } - logger.Debugf("Total size of all RBDs in a cluster is %d",totalSize) - total_rbd_requested_size.Set(float64(totalSize)) -} - -// the main loop for monitoting -func startCheking() { - - //Ticks every 5 seconds - ticker := time.NewTicker(2 * time.Second) - defer ticker.Stop() - - // Every tick I start listPools function - for range ticker.C { - poolList, err := listPools() - if err != nil { - logger.Fatalf("Error in listing pools %v", err) - } - // Get the map of all RBDs in all pools - rbdMap := getRBD(poolList) - //Get all RBDs info - RBDStats := RbdChecker(rbdMap) - //Fill out metrics - FormMetrirs(RBDStats) - } - -} - func main() { - defer logger.Sync() - http.Handle("/metrics", promhttp.Handler()) - - // HTTP runs in separate thread cuz it blocks futher execution of main - go func() { - logger.Info("Starting http server") - // Here I check for errors if HTTP fails - if err := http.ListenAndServe(":9040", nil); err != nil { - logger.Fatalf("HTTP server failed to start %v", err) + var result []Pool = []Pool{} + cephConn, err := connect() + if err != nil { + fmt.Println(err) + if wrapped := errors.Unwrap(err); wrapped != nil { + fmt.Println(wrapped) } - logger.Info("HTTP server started") - }() + os.Exit(1) + } + defer cephConn.conn.Shutdown() + fmt.Println("Successfully connected") - go func() { - logger.Info("Start checking") - startCheking() - }() + poolList, err := cephConn.conn.ListPools() + if err != nil { + errors.New("Cannot get list of pools") + } + + for _, v := range poolList { + x, _ := PoolFactory(cephConn, v) + result = append(result, x) + } + fmt.Println(result) - select {} } diff --git a/go_exporter/types.go b/go_exporter/types.go new file mode 100644 index 0000000..fe22890 --- /dev/null +++ b/go_exporter/types.go @@ -0,0 +1,42 @@ +package main + +import "github.com/ceph/go-ceph/rados" + +type Params struct { + config string + keyring string +} + +type CephConnection struct { + conn *rados.Conn +} + +type iRBD interface { + getName() string + getSize() int64 +} + +type RBDUsage struct { + Images []RBD `json:"images"` +} + +type RBD struct { + Name string `json:"name"` + Id string `json:"id"` + RequestedSize uint64 `json:"provisioned_size"` + UsedSize int64 `json:"used_size"` +} + +func (r RBD) getName() string { + return r.Name +} + +func (r RBD) getSize() int64 { + return r.UsedSize +} + +type Pool struct { + Name string + RBDlist []RBD + // RBDlist []string +} diff --git a/go_exporter_bkp/.fuse_hidden0000000d00000001 b/go_exporter_bkp/.fuse_hidden0000000d00000001 new file mode 100644 index 0000000..e69de29 diff --git a/go_exporter/Dockerfile b/go_exporter_bkp/Dockerfile similarity index 100% rename from go_exporter/Dockerfile rename to go_exporter_bkp/Dockerfile diff --git a/go_exporter/deploy.yaml b/go_exporter_bkp/deploy.yaml similarity index 100% rename from go_exporter/deploy.yaml rename to go_exporter_bkp/deploy.yaml diff --git a/go_exporter_bkp/go.mod b/go_exporter_bkp/go.mod new file mode 100644 index 0000000..1ab399b --- /dev/null +++ b/go_exporter_bkp/go.mod @@ -0,0 +1,22 @@ +module main.go + +go 1.25.4 + +require ( + github.com/prometheus/client_golang v1.23.2 + go.uber.org/zap v1.27.1 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.67.4 // indirect + github.com/prometheus/procfs v0.19.2 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + golang.org/x/sys v0.38.0 // indirect + google.golang.org/protobuf v1.36.10 // indirect + gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544 // indirect +) diff --git a/go_exporter_bkp/go.sum b/go_exporter_bkp/go.sum new file mode 100644 index 0000000..aafdc53 --- /dev/null +++ b/go_exporter_bkp/go.sum @@ -0,0 +1,37 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+Lvsc= +github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= +github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544 h1:WJH1qsOB4/zb/li+zLMn0vaAUJ5FqPv6HYLI3aQVg1k= +gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544/go.mod h1:UhTeH/yXCK/KY7TX24mqPkaQ7gZeqmWd/8SSS8B3aHw= diff --git a/go_exporter/goexp b/go_exporter_bkp/goexp similarity index 100% rename from go_exporter/goexp rename to go_exporter_bkp/goexp diff --git a/go_exporter_bkp/main.go b/go_exporter_bkp/main.go new file mode 100644 index 0000000..4820e22 --- /dev/null +++ b/go_exporter_bkp/main.go @@ -0,0 +1,267 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/http" + _ "os/exec" + "strings" + "time" + + prometheus "github.com/prometheus/client_golang/prometheus" + promhttp "github.com/prometheus/client_golang/prometheus/promhttp" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + pipe "gopkg.in/pipe.v2" +) + +var logger *zap.SugaredLogger + +// Here I store all images info in scope of one pool +type RBDUsage struct { + Images []RBDImageUsage +} + +// Here I do collect info about images itself +type RBDImageUsage struct { + Name string `json:"name"` + id int `json: "id"` + RequestedSize uint64 `json:"provisioned_size"` + UsedSize uint64 `json: "used_size"` +} + +var total_rbd_requested_size_per_pool = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "total_rbd_requested_size_per_pool", + Help: "total size of all requested RBDs in a specific pool", + }, + []string{"poolname"}, +) + +var total_rbd_requested_size = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "total_rbd_requested_size", + Help: "total size of all RBDs in the cluster", +}) + +// Here I initialize logger and set some custom settings +func loggerInit() *zap.SugaredLogger { + config := zap.NewDevelopmentConfig() + config.EncoderConfig.TimeKey = "timestamp" + config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + logger, err := config.Build() + if err != nil { + panic(fmt.Sprintf("Logger set up failed: %v", err)) + } + defer logger.Sync() + return logger.Sugar() +} + + + +// List pools with application rbd enabled +func listPools() ([]string, error) { + results := []string{} + command1 := "ceph" + args1 := []string{"osd", "pool", "ls", "detail"} + command2 := "grep" + args2 := []string{"application rbd"} + command3 := "awk" + args3 := []string{"{print $3}"} + + logger.Infof("Listing pools") + + // This is a pipe conjuction to execute "ceph | grep | awk" pipe + p := pipe.Line( + pipe.Exec(command1, args1...), + pipe.Exec(command2, args2...), + pipe.Exec(command3, args3...), + ) + + output, err := pipe.CombinedOutput(p) + if err != nil { + panic(err) + } + + //Returns iterator + lines := strings.SplitSeq(string(output), "\n") + + for v := range lines { + v = strings.TrimSpace(v) + // awk return result surrounded by single quotes so here I delete it + v = strings.Trim(v, "'") + // Sometimes here I have an empty string, that's why I check if it is not + if v != "" { + results = append(results, v) + continue + } + } + return results, nil +} + +// List rbd of each pool +func getRBD(poolList []string) map[string][]string { + RBDmap := make(map[string][]string) + args := []string{"ls", "-p"} + + // Here I iterate over pool names + for _, v := range poolList { + var results []string + //...and start a command rbd ls -p + new_args := append(args, v) + p := pipe.Line( + pipe.Exec("rbd", new_args...), + ) + output, err := pipe.CombinedOutput(p) + if err != nil { + panic(err) + } + + //Returns iterator + lines := strings.SplitSeq(string(output), "\n") + + for i := range lines { + //delete whitespaces around + i = strings.TrimSpace(i) + // check if there is no empty entries + if i != "" { + results = append(results, i) + continue + } + } + RBDmap[v] = results + } + return RBDmap +} + +// Here I check total provisioned size of each RBD image in a pool +func RbdChecker(rbdMap map[string][]string) map[string][]RBDUsage{ + total := make(map[string][]RBDUsage) + + for pool, rbdlist := range rbdMap { + logger.Infof("Processing pool %s", pool) + + for _, rbdName := range rbdlist { + total[pool] = append(total[pool],GetRBDStats(pool, rbdName)) + + } + } + logger.Debugf("Final map is %v",total) + return total +} + +// Grabbing info about specific image +func GetRBDStats(pool string, rbdname string) RBDUsage { + rbdPath := fmt.Sprintf("%s/%s", pool, rbdname) + args := []string{"du", "--format", "json", rbdPath} + + p := pipe.Line( + pipe.Exec("rbd", args...), + ) + output, err := pipe.CombinedOutput(p) + if err != nil { + logger.Fatalf("Error in processing RBD %v", err) + } + + var usage RBDUsage + if err := json.Unmarshal(output, &usage); err != nil { + logger.Fatalf("Error in unmarshaling %v", err) + } + return usage + +} + +func FormMetrirs(rbdStats map[string][]RBDUsage) { + // this is requested size overall cluster + var totalSize uint64 = 0 + + // Iterate over pools + for poolName := range rbdStats { + var totalSizePerPool uint64 = 0 + logger.Debugf("Forming metrics for pool %s",poolName) + + //Iterate over all RBDs in the pool + for _,rbdName := range rbdStats[poolName] { + logger.Debugf("Processings rbd %v",rbdName) + totalSizePerPool = totalSizePerPool + rbdName.Images[0].RequestedSize + // logger.Debugf("RBD name is %s and its size is %d",rbdName.Images[0].Name,rbdName.Images[0].RequestedSize) + } + logger.Debugf("Total size requested by RBDs of a pool %s is %d bytes",poolName,totalSizePerPool) + + total_rbd_requested_size_per_pool.WithLabelValues( + poolName, + ).Set(float64(totalSizePerPool)) + totalSize = totalSize + totalSizePerPool + } + logger.Debugf("Total size of all RBDs in a cluster is %d",totalSize) + total_rbd_requested_size.Set(float64(totalSize)) +} + +// the main loop for monitoting +func startCheking() { + + //Ticks every 5 seconds + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + + // Every tick I start listPools function + for range ticker.C { + poolList, err := listPools() + if err != nil { + logger.Fatalf("Error in listing pools %v", err) + } + // Get the map of all RBDs in all pools + rbdMap := getRBD(poolList) + //Get all RBDs info + RBDStats := RbdChecker(rbdMap) + //Fill out metrics + FormMetrirs(RBDStats) + } + +} + +// This func runs before main() to get all the things set up before main execution +func init() { + logger = loggerInit() + logger.Info("Setting up logger is complete successfully") + logger.Info("Registering prom metrics") + + prometheus.MustRegister( + total_rbd_requested_size_per_pool, + total_rbd_requested_size, + ) + +} + +type IPoolLister interface { + listPools() ([]string,error) +} + +type IRbdFinder interface { + getRBD(poollist []string) map[string][]string +} + +type IRbdChecker interface { + rbdChecker(rbdMap map[string][]string) map[string][]RBDUsage +} + + +func main() { + defer logger.Sync() + http.Handle("/metrics", promhttp.Handler()) + + // HTTP runs in separate thread cuz it blocks futher execution of main + go func() { + logger.Info("Starting http server") + // Here I check for errors if HTTP fails + if err := http.ListenAndServe(":9040", nil); err != nil { + logger.Fatalf("HTTP server failed to start %v", err) + } + logger.Info("HTTP server started") + }() + + go func() { + logger.Info("Start checking") + startCheking() + }() + + select {} +} diff --git a/go_exporter/service_monitor.yaml b/go_exporter_bkp/service_monitor.yaml similarity index 100% rename from go_exporter/service_monitor.yaml rename to go_exporter_bkp/service_monitor.yaml diff --git a/go_exporter/start.sh b/go_exporter_bkp/start.sh similarity index 100% rename from go_exporter/start.sh rename to go_exporter_bkp/start.sh diff --git a/oreilly/main.go b/oreilly/main.go index 399dadd..f279ce7 100644 --- a/oreilly/main.go +++ b/oreilly/main.go @@ -1,51 +1,20 @@ package main import ( - "fmt" - "os" - "strconv" - "sync" + "log" + "net/http" ) -func go1(ch chan<- string, wg1 *sync.WaitGroup) { - defer wg1.Done() - for i := 1; i < 11; i++ { - ch <- "go1_" + strconv.Itoa(i) - } -} - -func go2(ch chan<- string, wg1 *sync.WaitGroup) { - defer wg1.Done() - for i := 1; i < 11; i++ { - ch <- "go2_" + strconv.Itoa(i) - } -} - -func Reader(ch <-chan string, wg *sync.WaitGroup) { - defer wg.Done() - for { - if v, ok := <-ch; !ok { - fmt.Println("Channel closed") - os.Exit(1) - } else { - fmt.Println(v) - } - } +func lol(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("azazaza")) } func main() { - var wg1 sync.WaitGroup - var wg sync.WaitGroup - wg.Add(1) - wg1.Add(2) - ch := make(chan string) - go go1(ch, &wg1) - go go2(ch, &wg1) - go Reader(ch, &wg) + http.HandleFunc("/",lol) - go func() { - wg1.Wait() - close(ch) - }() - wg.Wait() + s := http.Server{ + Addr: ":9000", + } + log.Fatal(s.ListenAndServe()) } + diff --git a/patterns/simpleFactory/IGun.go b/patterns/simpleFactory/IGun.go new file mode 100644 index 0000000..3e7257a --- /dev/null +++ b/patterns/simpleFactory/IGun.go @@ -0,0 +1,8 @@ +package main + +type IGun interface { + setName(name string) + setPower(power int) + getName() string + getPower() int +} diff --git a/patterns/simpleFactory/ak47.go b/patterns/simpleFactory/ak47.go new file mode 100644 index 0000000..661f244 --- /dev/null +++ b/patterns/simpleFactory/ak47.go @@ -0,0 +1,14 @@ +package main + +type AK47 struct { + Gun +} + +func newAk47() IGun { + return AK47{ + Gun: Gun{ + name: "AK47", + power: 4, + }, + } +} diff --git a/patterns/simpleFactory/go.mod b/patterns/simpleFactory/go.mod new file mode 100644 index 0000000..f3d0661 --- /dev/null +++ b/patterns/simpleFactory/go.mod @@ -0,0 +1,3 @@ +module main.go + +go 1.25.5 diff --git a/patterns/simpleFactory/gun.go b/patterns/simpleFactory/gun.go new file mode 100644 index 0000000..deaeadc --- /dev/null +++ b/patterns/simpleFactory/gun.go @@ -0,0 +1,22 @@ +package main + +type Gun struct { + name string + power int +} + +func (g Gun) setName(name string) { + g.name = name +} + +func (g Gun) setPower(power int) { + g.power = power +} + +func (g Gun) getName() string { + return g.name +} + +func (g Gun) getPower() int { + return g.power +} diff --git a/patterns/simpleFactory/gunFactory.go b/patterns/simpleFactory/gunFactory.go new file mode 100644 index 0000000..9b32a1c --- /dev/null +++ b/patterns/simpleFactory/gunFactory.go @@ -0,0 +1,13 @@ +package main + +import "errors" + +func GetGun(gunType string) (IGun,error){ + if gunType == "ak47" { + return newAk47(),nil + } + if gunType == "musket" { + return newMusket(),nil + } + return nil,errors.New("No such a gun") +} \ No newline at end of file diff --git a/patterns/simpleFactory/main.go b/patterns/simpleFactory/main.go new file mode 100644 index 0000000..48f1181 --- /dev/null +++ b/patterns/simpleFactory/main.go @@ -0,0 +1,22 @@ +package main + +import "fmt" + +func PrintDetails(gunName IGun) { + fmt.Println("Gun model is ",gunName.getName()) + fmt.Println("Gun power is ",gunName.getPower()) +} + +func main() { + ak47,err := GetGun("ak47") + if err != nil { + fmt.Println(err) + } + musket,err := GetGun("musket") + if err != nil { + fmt.Println(err) + } + PrintDetails(ak47) + PrintDetails(musket) + +} \ No newline at end of file diff --git a/patterns/simpleFactory/musket.go b/patterns/simpleFactory/musket.go new file mode 100644 index 0000000..7cddef4 --- /dev/null +++ b/patterns/simpleFactory/musket.go @@ -0,0 +1,14 @@ +package main + +type musket struct { + Gun +} + +func newMusket() IGun { + return musket{ + Gun: Gun{ + name: "Musket gun", + power: 1, + }, + } +} \ No newline at end of file