gitea/learning_go
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

new version

Browse Source
This commit is contained in:
a.pivkin 2025-12-15 20:20:11 +03:00
parent 75066c8a08
commit 4e051565e2
34 changed files with 641 additions and 331 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
go_exporter/.fuse_hidden0000000d00000001 Normal file → Executable file
View File

0
go_exporter/.fuse_hidden0000001000000002 Executable file
View File

0
go_exporter/.fuse_hidden0000001300000003 Executable file
View File

0
go_exporter/.fuse_hidden0000001600000004 Executable file
View File

0
go_exporter/.fuse_hidden0000001b00000005 Executable file
View File

0
go_exporter/.fuse_hidden0000001e00000006 Executable file
View File

0
go_exporter/.fuse_hidden0000002300000007 Executable file
View File

0
go_exporter/.fuse_hidden0000002800000008 Executable file
View File

0
go_exporter/.fuse_hidden0000002d00000009 Executable file
View File

64
go_exporter/RBDFactory.go Normal file
View File

@ -0,0 +1,64 @@
package main
import (
"encoding/json"
"fmt"
// "github.com/ceph/go-ceph/rados"
"github.com/ceph/go-ceph/rbd"
pipe "gopkg.in/pipe.v2"
)
func PoolFactory(cephConn CephConnection, poolName string) (Pool, error) {
var rbdlist []RBD = []RBD{}
ioctx, err := cephConn.conn.OpenIOContext(poolName)
if err != nil {
return Pool{}, fmt.Errorf("Couldn't set context for pool %s %w", poolName, err)
}
defer ioctx.Destroy()
imageList, err := rbd.GetImageNames(ioctx)
if err != nil {
return Pool{}, fmt.Errorf("Couldn't get list of rbds %w", err)
}
for _, v := range imageList {
stat, err := RBDFacroty(poolName, v)
if err != nil {
fmt.Errorf("Coundn't get stat from disk %s %w", v, err)
panic(err)
}
// fmt.Println(stat)
rbdlist = append(rbdlist, stat)
}
return Pool{
Name: poolName,
RBDlist: rbdlist,
}, nil
}
func RBDFacroty(poolname string, rbdname string) (RBD, error) {
rbdPath := fmt.Sprintf("%s/%s", poolname, rbdname)
args := []string{"du", "--format", "json", rbdPath}
p := pipe.Line(
pipe.Exec("rbd", args...),
)
output, err := pipe.CombinedOutput(p)
if err != nil {
fmt.Errorf("Error in processing RBD %v", err)
}
var rbdImage RBDUsage
if err := json.Unmarshal(output, &rbdImage); err != nil {
fmt.Errorf("Error in unmarshaling %w", err)
}
return RBD{
Name: rbdImage.Images[0].Name,
Id: rbdImage.Images[0].Id,
RequestedSize: rbdImage.Images[0].RequestedSize,
UsedSize: rbdImage.Images[0].UsedSize,
}, nil
}

39
go_exporter/connection.go Normal file
View File

@ -0,0 +1,39 @@
package main
import (
"fmt"
"github.com/ceph/go-ceph/rados"
)
func connect() (_ CephConnection, err error) {
var cephConn CephConnection = CephConnection{}
defer func() {
if err != nil {
err = fmt.Errorf("Error in func connect() %w", err)
}
}()
cephConn.conn, err = rados.NewConnWithClusterAndUser("ceph", "client.admin")
if err != nil {
return CephConnection{}, err
}
err = cephConn.conn.ReadConfigFile(params.config)
if err != nil {
return CephConnection{}, err
}
err = cephConn.conn.SetConfigOption("keyring", params.keyring)
if err != nil {
return CephConnection{}, err
}
err = cephConn.conn.Connect()
if err != nil {
return CephConnection{}, err
}
return cephConn, nil
}

16
go_exporter/getOpts.go Normal file
View File

@ -0,0 +1,16 @@
package main
import "flag"
var params Params
// This func runs even before main()
func init() {
config_file := flag.String("config", "/etc/ceph/ceph.conf", "placement of ceph config file")
_keyring := flag.String("keyring", "/etc/ceph/ceph.client.admin.keyring", "placement of ceph keyring file")
flag.Parse()
params.config = *config_file
params.keyring = *_keyring
}

19
go_exporter/go.mod
View File

@ -1,22 +1,13 @@
module main.go
go 1.25.4
go 1.25.5
require (
github.com/prometheus/client_golang v1.23.2
go.uber.org/zap v1.27.1
github.com/ceph/go-ceph v0.36.0
gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544
)
require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.67.4 // indirect
github.com/prometheus/procfs v0.19.2 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
golang.org/x/sys v0.38.0 // indirect
google.golang.org/protobuf v1.36.10 // indirect
gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544 // indirect
golang.org/x/sys v0.36.0 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
)

56
go_exporter/go.sum
View File

@ -1,37 +1,23 @@
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+Lvsc=
github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI=
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws=
github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc=
go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
github.com/ceph/go-ceph v0.36.0 h1:IDE4vEF+4fmjve+CPjD1WStgfQ+Lh6vD+9PMUI712KI=
github.com/ceph/go-ceph v0.36.0/go.mod h1:fGCbndVDLuHW7q2954d6y+tgPFOBnRLqJRe2YXyngw4=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gofrs/uuid/v5 v5.3.2 h1:2jfO8j3XgSwlz/wHqemAEugfnTlikAYHhnqQ8Xh4fE0=
github.com/gofrs/uuid/v5 v5.3.2/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8=
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544 h1:WJH1qsOB4/zb/li+zLMn0vaAUJ5FqPv6HYLI3aQVg1k=
gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544/go.mod h1:UhTeH/yXCK/KY7TX24mqPkaQ7gZeqmWd/8SSS8B3aHw=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

BIN
go_exporter/kek Executable file
View File

Binary file not shown.

263
go_exporter/main.go
View File

@ -1,252 +1,33 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"net/http"
_ "os/exec"
"strings"
"time"
prometheus "github.com/prometheus/client_golang/prometheus"
promhttp "github.com/prometheus/client_golang/prometheus/promhttp"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
pipe "gopkg.in/pipe.v2"
"os"
)
var logger *zap.SugaredLogger
// Here I store all images info in scope of one pool
type RBDUsage struct {
Images []RBDImageUsage
}
// Here I do collect info about images itself
type RBDImageUsage struct {
Name string `json:"name"`
id int `json: "id"`
RequestedSize uint64 `json:"provisioned_size"`
UsedSize uint64 `json: "used_size"`
}
var total_rbd_requested_size_per_pool = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "total_rbd_requested_size_per_pool",
Help: "total size of all requested RBDs in a specific pool",
},
[]string{"poolname"},
)
var total_rbd_requested_size = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "total_rbd_requested_size",
Help: "total size of all RBDs in the cluster",
})
// Here I initialize logger and set some custom settings
func loggerInit() *zap.SugaredLogger {
config := zap.NewDevelopmentConfig()
config.EncoderConfig.TimeKey = "timestamp"
config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
logger, err := config.Build()
if err != nil {
panic(fmt.Sprintf("Logger set up failed: %v", err))
}
defer logger.Sync()
return logger.Sugar()
}
// This func runs before main() to get all the things set up before main execution
func init() {
logger = loggerInit()
logger.Info("Setting up logger is complete successfully")
logger.Info("Registering prom metrics")
prometheus.MustRegister(
total_rbd_requested_size_per_pool,
total_rbd_requested_size,
)
}
// List pools with application rbd enabled
func listPools() ([]string, error) {
results := []string{}
command1 := "ceph"
args1 := []string{"osd", "pool", "ls", "detail"}
command2 := "grep"
args2 := []string{"application rbd"}
command3 := "awk"
args3 := []string{"{print $3}"}
logger.Infof("Listing pools")
// This is a pipe conjuction to execute "ceph | grep | awk" pipe
p := pipe.Line(
pipe.Exec(command1, args1...),
pipe.Exec(command2, args2...),
pipe.Exec(command3, args3...),
)
output, err := pipe.CombinedOutput(p)
if err != nil {
panic(err)
}
//Returns iterator
lines := strings.SplitSeq(string(output), "\n")
for v := range lines {
v = strings.TrimSpace(v)
// awk return result surrounded by single quotes so here I delete it
v = strings.Trim(v, "'")
// Sometimes here I have an empty string, that's why I check if it is not
if v != "" {
results = append(results, v)
continue
}
}
return results, nil
}
// List rbd of each pool
func getRBD(poolList []string) map[string][]string {
RBDmap := make(map[string][]string)
args := []string{"ls", "-p"}
// Here I iterate over pool names
for _, v := range poolList {
var results []string
//...and start a command rbd ls -p <pool_name>
new_args := append(args, v)
p := pipe.Line(
pipe.Exec("rbd", new_args...),
)
output, err := pipe.CombinedOutput(p)
if err != nil {
panic(err)
}
//Returns iterator
lines := strings.SplitSeq(string(output), "\n")
for i := range lines {
//delete whitespaces around
i = strings.TrimSpace(i)
// check if there is no empty entries
if i != "" {
results = append(results, i)
continue
}
}
RBDmap[v] = results
}
return RBDmap
}
// Here I check total provisioned size of each RBD image in a pool
func RbdChecker(rbdMap map[string][]string) map[string][]RBDUsage{
total := make(map[string][]RBDUsage)
for pool, rbdlist := range rbdMap {
logger.Infof("Processing pool %s", pool)
for _, rbdName := range rbdlist {
total[pool] = append(total[pool],GetRBDStats(pool, rbdName))
}
}
logger.Debugf("Final map is %v",total)
return total
}
// Grabbing info about specific image
func GetRBDStats(pool string, rbdname string) RBDUsage {
rbdPath := fmt.Sprintf("%s/%s", pool, rbdname)
args := []string{"du", "--format", "json", rbdPath}
p := pipe.Line(
pipe.Exec("rbd", args...),
)
output, err := pipe.CombinedOutput(p)
if err != nil {
logger.Fatalf("Error in processing RBD %v", err)
}
var usage RBDUsage
if err := json.Unmarshal(output, &usage); err != nil {
logger.Fatalf("Error in unmarshaling %v", err)
}
return usage
}
func FormMetrirs(rbdStats map[string][]RBDUsage) {
// this is requested size overall cluster
var totalSize uint64 = 0
// Iterate over pools
for poolName := range rbdStats {
var totalSizePerPool uint64 = 0
logger.Debugf("Forming metrics for pool %s",poolName)
//Iterate over all RBDs in the pool
for _,rbdName := range rbdStats[poolName] {
logger.Debugf("Processings rbd %v",rbdName)
totalSizePerPool = totalSizePerPool + rbdName.Images[0].RequestedSize
// logger.Debugf("RBD name is %s and its size is %d",rbdName.Images[0].Name,rbdName.Images[0].RequestedSize)
}
logger.Debugf("Total size requested by RBDs of a pool %s is %d bytes",poolName,totalSizePerPool)
total_rbd_requested_size_per_pool.WithLabelValues(
poolName,
).Set(float64(totalSizePerPool))
totalSize = totalSize + totalSizePerPool
}
logger.Debugf("Total size of all RBDs in a cluster is %d",totalSize)
total_rbd_requested_size.Set(float64(totalSize))
}
// the main loop for monitoting
func startCheking() {
//Ticks every 5 seconds
ticker := time.NewTicker(2 * time.Second)
defer ticker.Stop()
// Every tick I start listPools function
for range ticker.C {
poolList, err := listPools()
if err != nil {
logger.Fatalf("Error in listing pools %v", err)
}
// Get the map of all RBDs in all pools
rbdMap := getRBD(poolList)
//Get all RBDs info
RBDStats := RbdChecker(rbdMap)
//Fill out metrics
FormMetrirs(RBDStats)
}
}
func main() {
defer logger.Sync()
http.Handle("/metrics", promhttp.Handler())
// HTTP runs in separate thread cuz it blocks futher execution of main
go func() {
logger.Info("Starting http server")
// Here I check for errors if HTTP fails
if err := http.ListenAndServe(":9040", nil); err != nil {
logger.Fatalf("HTTP server failed to start %v", err)
var result []Pool = []Pool{}
cephConn, err := connect()
if err != nil {
fmt.Println(err)
if wrapped := errors.Unwrap(err); wrapped != nil {
fmt.Println(wrapped)
}
logger.Info("HTTP server started")
}()
os.Exit(1)
}
defer cephConn.conn.Shutdown()
fmt.Println("Successfully connected")
go func() {
logger.Info("Start checking")
startCheking()
}()
poolList, err := cephConn.conn.ListPools()
if err != nil {
errors.New("Cannot get list of pools")
}
for _, v := range poolList {
x, _ := PoolFactory(cephConn, v)
result = append(result, x)
}
fmt.Println(result)
select {}
}

42
go_exporter/types.go Normal file
View File

@ -0,0 +1,42 @@
package main
import "github.com/ceph/go-ceph/rados"
type Params struct {
config string
keyring string
}
type CephConnection struct {
conn *rados.Conn
}
type iRBD interface {
getName() string
getSize() int64
}
type RBDUsage struct {
Images []RBD `json:"images"`
}
type RBD struct {
Name string `json:"name"`
Id string `json:"id"`
RequestedSize uint64 `json:"provisioned_size"`
UsedSize int64 `json:"used_size"`
}
func (r RBD) getName() string {
return r.Name
}
func (r RBD) getSize() int64 {
return r.UsedSize
}
type Pool struct {
Name string
RBDlist []RBD
// RBDlist []string
}

0
go_exporter_bkp/.fuse_hidden0000000d00000001 Normal file
View File

0
go_exporter/Dockerfile → go_exporter_bkp/Dockerfile
View File

0
go_exporter/deploy.yaml → go_exporter_bkp/deploy.yaml
View File

22
go_exporter_bkp/go.mod Normal file
View File

@ -0,0 +1,22 @@
module main.go
go 1.25.4
require (
github.com/prometheus/client_golang v1.23.2
go.uber.org/zap v1.27.1
)
require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.67.4 // indirect
github.com/prometheus/procfs v0.19.2 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
golang.org/x/sys v0.38.0 // indirect
google.golang.org/protobuf v1.36.10 // indirect
gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544 // indirect
)

37
go_exporter_bkp/go.sum Normal file
View File

@ -0,0 +1,37 @@
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+Lvsc=
github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI=
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws=
github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc=
go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544 h1:WJH1qsOB4/zb/li+zLMn0vaAUJ5FqPv6HYLI3aQVg1k=
gopkg.in/pipe.v2 v2.0.0-20140414041502-3c2ca4d52544/go.mod h1:UhTeH/yXCK/KY7TX24mqPkaQ7gZeqmWd/8SSS8B3aHw=

0
go_exporter/goexp → go_exporter_bkp/goexp
View File

267
go_exporter_bkp/main.go Normal file
View File

@ -0,0 +1,267 @@
package main
import (
"encoding/json"
"fmt"
"net/http"
_ "os/exec"
"strings"
"time"
prometheus "github.com/prometheus/client_golang/prometheus"
promhttp "github.com/prometheus/client_golang/prometheus/promhttp"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
pipe "gopkg.in/pipe.v2"
)
var logger *zap.SugaredLogger
// Here I store all images info in scope of one pool
type RBDUsage struct {
Images []RBDImageUsage
}
// Here I do collect info about images itself
type RBDImageUsage struct {
Name string `json:"name"`
id int `json: "id"`
RequestedSize uint64 `json:"provisioned_size"`
UsedSize uint64 `json: "used_size"`
}
var total_rbd_requested_size_per_pool = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "total_rbd_requested_size_per_pool",
Help: "total size of all requested RBDs in a specific pool",
},
[]string{"poolname"},
)
var total_rbd_requested_size = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "total_rbd_requested_size",
Help: "total size of all RBDs in the cluster",
})
// Here I initialize logger and set some custom settings
func loggerInit() *zap.SugaredLogger {
config := zap.NewDevelopmentConfig()
config.EncoderConfig.TimeKey = "timestamp"
config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
logger, err := config.Build()
if err != nil {
panic(fmt.Sprintf("Logger set up failed: %v", err))
}
defer logger.Sync()
return logger.Sugar()
}
// List pools with application rbd enabled
func listPools() ([]string, error) {
results := []string{}
command1 := "ceph"
args1 := []string{"osd", "pool", "ls", "detail"}
command2 := "grep"
args2 := []string{"application rbd"}
command3 := "awk"
args3 := []string{"{print $3}"}
logger.Infof("Listing pools")
// This is a pipe conjuction to execute "ceph | grep | awk" pipe
p := pipe.Line(
pipe.Exec(command1, args1...),
pipe.Exec(command2, args2...),
pipe.Exec(command3, args3...),
)
output, err := pipe.CombinedOutput(p)
if err != nil {
panic(err)
}
//Returns iterator
lines := strings.SplitSeq(string(output), "\n")
for v := range lines {
v = strings.TrimSpace(v)
// awk return result surrounded by single quotes so here I delete it
v = strings.Trim(v, "'")
// Sometimes here I have an empty string, that's why I check if it is not
if v != "" {
results = append(results, v)
continue
}
}
return results, nil
}
// List rbd of each pool
func getRBD(poolList []string) map[string][]string {
RBDmap := make(map[string][]string)
args := []string{"ls", "-p"}
// Here I iterate over pool names
for _, v := range poolList {
var results []string
//...and start a command rbd ls -p <pool_name>
new_args := append(args, v)
p := pipe.Line(
pipe.Exec("rbd", new_args...),
)
output, err := pipe.CombinedOutput(p)
if err != nil {
panic(err)
}
//Returns iterator
lines := strings.SplitSeq(string(output), "\n")
for i := range lines {
//delete whitespaces around
i = strings.TrimSpace(i)
// check if there is no empty entries
if i != "" {
results = append(results, i)
continue
}
}
RBDmap[v] = results
}
return RBDmap
}
// Here I check total provisioned size of each RBD image in a pool
func RbdChecker(rbdMap map[string][]string) map[string][]RBDUsage{
total := make(map[string][]RBDUsage)
for pool, rbdlist := range rbdMap {
logger.Infof("Processing pool %s", pool)
for _, rbdName := range rbdlist {
total[pool] = append(total[pool],GetRBDStats(pool, rbdName))
}
}
logger.Debugf("Final map is %v",total)
return total
}
// Grabbing info about specific image
func GetRBDStats(pool string, rbdname string) RBDUsage {
rbdPath := fmt.Sprintf("%s/%s", pool, rbdname)
args := []string{"du", "--format", "json", rbdPath}
p := pipe.Line(
pipe.Exec("rbd", args...),
)
output, err := pipe.CombinedOutput(p)
if err != nil {
logger.Fatalf("Error in processing RBD %v", err)
}
var usage RBDUsage
if err := json.Unmarshal(output, &usage); err != nil {
logger.Fatalf("Error in unmarshaling %v", err)
}
return usage
}
func FormMetrirs(rbdStats map[string][]RBDUsage) {
// this is requested size overall cluster
var totalSize uint64 = 0
// Iterate over pools
for poolName := range rbdStats {
var totalSizePerPool uint64 = 0
logger.Debugf("Forming metrics for pool %s",poolName)
//Iterate over all RBDs in the pool
for _,rbdName := range rbdStats[poolName] {
logger.Debugf("Processings rbd %v",rbdName)
totalSizePerPool = totalSizePerPool + rbdName.Images[0].RequestedSize
// logger.Debugf("RBD name is %s and its size is %d",rbdName.Images[0].Name,rbdName.Images[0].RequestedSize)
}
logger.Debugf("Total size requested by RBDs of a pool %s is %d bytes",poolName,totalSizePerPool)
total_rbd_requested_size_per_pool.WithLabelValues(
poolName,
).Set(float64(totalSizePerPool))
totalSize = totalSize + totalSizePerPool
}
logger.Debugf("Total size of all RBDs in a cluster is %d",totalSize)
total_rbd_requested_size.Set(float64(totalSize))
}
// the main loop for monitoting
func startCheking() {
//Ticks every 5 seconds
ticker := time.NewTicker(2 * time.Second)
defer ticker.Stop()
// Every tick I start listPools function
for range ticker.C {
poolList, err := listPools()
if err != nil {
logger.Fatalf("Error in listing pools %v", err)
}
// Get the map of all RBDs in all pools
rbdMap := getRBD(poolList)
//Get all RBDs info
RBDStats := RbdChecker(rbdMap)
//Fill out metrics
FormMetrirs(RBDStats)
}
}
// This func runs before main() to get all the things set up before main execution
func init() {
logger = loggerInit()
logger.Info("Setting up logger is complete successfully")
logger.Info("Registering prom metrics")
prometheus.MustRegister(
total_rbd_requested_size_per_pool,
total_rbd_requested_size,
)
}
type IPoolLister interface {
listPools() ([]string,error)
}
type IRbdFinder interface {
getRBD(poollist []string) map[string][]string
}
type IRbdChecker interface {
rbdChecker(rbdMap map[string][]string) map[string][]RBDUsage
}
func main() {
defer logger.Sync()
http.Handle("/metrics", promhttp.Handler())
// HTTP runs in separate thread cuz it blocks futher execution of main
go func() {
logger.Info("Starting http server")
// Here I check for errors if HTTP fails
if err := http.ListenAndServe(":9040", nil); err != nil {
logger.Fatalf("HTTP server failed to start %v", err)
}
logger.Info("HTTP server started")
}()
go func() {
logger.Info("Start checking")
startCheking()
}()
select {}
}

0
go_exporter/service_monitor.yaml → go_exporter_bkp/service_monitor.yaml
View File

0
go_exporter/start.sh → go_exporter_bkp/start.sh
View File

51
oreilly/main.go
View File

@ -1,51 +1,20 @@
package main
import (
"fmt"
"os"
"strconv"
"sync"
"log"
"net/http"
)
func go1(ch chan<- string, wg1 *sync.WaitGroup) {
defer wg1.Done()
for i := 1; i < 11; i++ {
ch <- "go1_" + strconv.Itoa(i)
}
}
func go2(ch chan<- string, wg1 *sync.WaitGroup) {
defer wg1.Done()
for i := 1; i < 11; i++ {
ch <- "go2_" + strconv.Itoa(i)
}
}
func Reader(ch <-chan string, wg *sync.WaitGroup) {
defer wg.Done()
for {
if v, ok := <-ch; !ok {
fmt.Println("Channel closed")
os.Exit(1)
} else {
fmt.Println(v)
}
}
func lol(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("azazaza"))
}
func main() {
var wg1 sync.WaitGroup
var wg sync.WaitGroup
wg.Add(1)
wg1.Add(2)
ch := make(chan string)
go go1(ch, &wg1)
go go2(ch, &wg1)
go Reader(ch, &wg)
http.HandleFunc("/",lol)
go func() {
wg1.Wait()
close(ch)
}()
wg.Wait()
s := http.Server{
Addr: ":9000",
}
log.Fatal(s.ListenAndServe())
}

8
patterns/simpleFactory/IGun.go Normal file
View File

@ -0,0 +1,8 @@
package main
type IGun interface {
setName(name string)
setPower(power int)
getName() string
getPower() int
}

14
patterns/simpleFactory/ak47.go Normal file
View File

@ -0,0 +1,14 @@
package main
type AK47 struct {
Gun
}
func newAk47() IGun {
return AK47{
Gun: Gun{
name: "AK47",
power: 4,
},
}
}

3
patterns/simpleFactory/go.mod Normal file
View File

@ -0,0 +1,3 @@
module main.go
go 1.25.5

22
patterns/simpleFactory/gun.go Normal file
View File

@ -0,0 +1,22 @@
package main
type Gun struct {
name string
power int
}
func (g Gun) setName(name string) {
g.name = name
}
func (g Gun) setPower(power int) {
g.power = power
}
func (g Gun) getName() string {
return g.name
}
func (g Gun) getPower() int {
return g.power
}

13
patterns/simpleFactory/gunFactory.go Normal file
View File

@ -0,0 +1,13 @@
package main
import "errors"
func GetGun(gunType string) (IGun,error){
if gunType == "ak47" {
return newAk47(),nil
}
if gunType == "musket" {
return newMusket(),nil
}
return nil,errors.New("No such a gun")
}

22
patterns/simpleFactory/main.go Normal file
View File

@ -0,0 +1,22 @@
package main
import "fmt"
func PrintDetails(gunName IGun) {
fmt.Println("Gun model is ",gunName.getName())
fmt.Println("Gun power is ",gunName.getPower())
}
func main() {
ak47,err := GetGun("ak47")
if err != nil {
fmt.Println(err)
}
musket,err := GetGun("musket")
if err != nil {
fmt.Println(err)
}
PrintDetails(ak47)
PrintDetails(musket)
}

14
patterns/simpleFactory/musket.go Normal file
View File

@ -0,0 +1,14 @@
package main
type musket struct {
Gun
}
func newMusket() IGun {
return musket{
Gun: Gun{
name: "Musket gun",
power: 1,
},
}
}