added logic for prometheus controllers

go.mod

@@ -7,7 +7,6 @@ toolchain go1.22.2
 require (
 	gitea.pena/PenaSide/common v0.0.0-20241128160655-fe730a08b5f1
 	gitea.pena/PenaSide/customer v0.0.0-20241118214409-34a88a3a7001 //dev
-	gitea.pena/PenaSide/trashlog v0.0.0-20241209132057-d978ecf39d21 //dev
 	github.com/caarlos0/env/v8 v8.0.0
 	github.com/go-playground/validator/v10 v10.14.1
 	github.com/gofiber/contrib/fiberzap v1.0.2
@@ -19,13 +18,22 @@ require (
 	go.uber.org/zap v1.27.0
 )
 
-require gitea.pena/PenaSide/linters-golang v0.0.0-20241207122018-933207374735
+require (
+	gitea.pena/PenaSide/linters-golang v0.0.0-20241207122018-933207374735
+	gitea.pena/PenaSide/trashlog v0.0.0-20241209132057-d978ecf39d21
+	github.com/prometheus/client_golang v1.20.5
+)
 
 require (
 	github.com/ClickHouse/clickhouse-go v1.5.4 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_golang v1.20.5 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.55.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/skeris/appInit v1.0.2 // indirect
 	go.etcd.io/bbolt v1.3.10 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf // indirect
@@ -61,7 +69,7 @@ require (
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/rs/xid v1.5.0 // indirect
 	github.com/sirupsen/logrus v1.9.2 // indirect
-	github.com/stretchr/testify v1.8.4
+	github.com/stretchr/testify v1.9.0
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasthttp v1.52.0
 	github.com/valyala/tcplisten v1.0.0 // indirect

go.sum

@@ -11,10 +11,14 @@ github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV
 github.com/ClickHouse/clickhouse-go v1.5.4/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=
 github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
 github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk=
 github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=
 github.com/caarlos0/env/v8 v8.0.0 h1:POhxHhSpuxrLMIdvTGARuZqR4Jjm8AYmoi/JKlcScs0=
 github.com/caarlos0/env/v8 v8.0.0/go.mod h1:7K4wMY9bH0esiXSSHlfHLX5xKGQMnkH5Fk4TDSSSzfo=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg=
 github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -71,6 +75,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
 github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
@@ -95,6 +101,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
 github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/onsi/ginkgo v1.16.0 h1:NBrNLB37exjJLxXtFOktx6CISBdS1aF8+7MwKlTV8U4=
@@ -112,6 +120,12 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
 github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
+github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
+github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
+github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
+github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -134,8 +148,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/themakers/bdd v0.0.0-20210316111417-6b1dfe326f33 h1:N9f/Q+2Ssa+yDcbfaoLTYvXmdeyUUxsJKdPUVsjSmiA=
 github.com/themakers/bdd v0.0.0-20210316111417-6b1dfe326f33/go.mod h1:rpcH99JknBh8seZmlOlUg51gasZH6QH34oXNsIwYT6E=
 github.com/themakers/hlog v0.0.0-20191205140925-235e0e4baddf h1:TJJm6KcBssmbWzplF5lzixXl1RBAi/ViPs1GaSOkhwo=

internal/app/app_verification.go

@@ -99,7 +99,7 @@ func Run(cfg *config.Config, build Build) {
 
 	adminSrv := server.NewHTTP(server.ServerConfig{
 		Logger:      logger,
-		Controllers: []server.Controller{cons.VerificationAdmin},
+		Controllers: []server.Controller{cons.VerificationAdmin, cons.PrometheusAdmin},
 		HLogger:     loggerHlog,
 	})
 

internal/controllers/admin/prometheus/prometheus.go

@@ -1,34 +1,34 @@
 package prometheus
 
 import (
-	"gitea.pena/PenaSide/verification/internal/controllers/admin/verification_admin"
+	"gitea.pena/PenaSide/verification/internal/repository"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"net/http"
 )
 
 type Prometheus struct {
-	verificationAdmin *verification_admin.VerifyAdminController
+	verificationRepo *repository.VerificationRepository
-	metrics           prometheusRegistry
+	metrics          prometheusRegistry
 }
 
 type prometheusRegistry struct {
-	totalBytes   prometheus.Counter // Общее количество загруженной памяти
+	totalBytes   prometheus.Gauge // Общее количество загруженной памяти
-	totalTime    prometheus.Counter // Общее затраченное время на 1 загрузку
+	totalTime    prometheus.Gauge // Общее затраченное время на 1 загрузку
-	totalUploads prometheus.Counter // Количество всего загрузок файлов
+	totalUploads prometheus.Gauge // Количество всего загрузок файлов
 }
 
-func NewPrometheus(verificationAdmin *verification_admin.VerifyAdminController) *Prometheus {
+func NewPrometheus(verificationRepo *repository.VerificationRepository) *Prometheus {
 	registry := prometheusRegistry{
-		totalBytes: prometheus.NewCounter(prometheus.CounterOpts{
+		totalBytes: prometheus.NewGauge(prometheus.GaugeOpts{
 			Name: "s3_total_uploaded_bytes",
 			Help: "Total memory loaded in bytes to S3",
 		}),
-		totalTime: prometheus.NewCounter(prometheus.CounterOpts{
+		totalTime: prometheus.NewGauge(prometheus.GaugeOpts{
 			Name: "s3_total_upload_time_ns",
 			Help: "Total time spent downloading files to S3",
 		}),
-		totalUploads: prometheus.NewCounter(prometheus.CounterOpts{
+		totalUploads: prometheus.NewGauge(prometheus.GaugeOpts{
 			Name: "s3_total_upload_count",
 			Help: "Number of total file downloads to S3",
 		}),
@@ -41,14 +41,20 @@ func NewPrometheus(verificationAdmin *verification_admin.VerifyAdminController)
 	)
 
 	return &Prometheus{
-		verificationAdmin: verificationAdmin,
+		verificationRepo: verificationRepo,
-		metrics:           registry,
+		metrics:          registry,
 	}
 }
 
 func (receiver *Prometheus) Metrics(w http.ResponseWriter, r *http.Request) {
-	//receiver.updateMetrics()
+	receiver.updateMetrics()
 	handler := promhttp.Handler()
 
 	handler.ServeHTTP(w, r)
 }
+
+func (receiver *Prometheus) updateMetrics() {
+	receiver.metrics.totalBytes.Set(float64(receiver.verificationRepo.Metrics.TotalBytes))
+	receiver.metrics.totalTime.Set(float64(receiver.verificationRepo.Metrics.TotalTime))
+	receiver.metrics.totalUploads.Set(float64(receiver.verificationRepo.Metrics.TotalUploads))
+}

internal/initialize/controllers.go

@@ -3,12 +3,14 @@ package initialize
 import (
 	"gitea.pena/PenaSide/customer/pkg/customer_clients"
 	"gitea.pena/PenaSide/verification/internal/client"
+	"gitea.pena/PenaSide/verification/internal/controllers/admin/prometheus"
 	"gitea.pena/PenaSide/verification/internal/controllers/admin/verification_admin"
 	"gitea.pena/PenaSide/verification/internal/controllers/user"
 )
 
 type Controllers struct {
 	VerificationAdmin *verification_admin.VerifyAdminController
+	PrometheusAdmin   *prometheus.Prometheus
 	VerificationUser  *user.VerifyUserController
 }
 
@@ -18,6 +20,9 @@ func NewControllers(reps *Repositories, telegram *client.Telegram, customer *cus
 			Repository: reps.Verification,
 			Customer:   customer,
 		}),
+		PrometheusAdmin: prometheus.NewPrometheus(
+			reps.Verification,
+		),
 		VerificationUser: user.NewVerificationUserController(user.VerifyUserControllerDeps{
 			Repository: reps.Verification,
 			Telegram:   telegram,

internal/repository/verification.go

@@ -24,13 +24,13 @@ type VerificationRepository struct {
 	s3          *minio.Client
 	folder, url string
 
-	metrics Metrics
+	Metrics Metrics
 }
 
 type Metrics struct {
-	totalBytes   int64
+	TotalBytes   int64
-	totalTime    int64
+	TotalTime    int64
-	totalUploads int64
+	TotalUploads int64
 }
 
 const (
@@ -63,10 +63,12 @@ func (r *VerificationRepository) Insert(
 		return nil, err
 	}
 
+	startTime := time.Now()
 	_, err = r.s3.PutObject(ctx, VerificationBucket, fmt.Sprintf("%s/%s/%s", r.folder, userID, innFH.Filename), inn, innFH.Size, minio.PutObjectOptions{})
 	if r.err(err) {
 		return nil, err
 	}
+	r.recordMetrics(innFH.Size, time.Since(startTime).Nanoseconds())
 
 	rule, err := ruleFH.Open()
 	if r.err(err) {
@@ -74,10 +76,12 @@ func (r *VerificationRepository) Insert(
 	}
 
 	// Put rule file
+	startTime = time.Now()
 	_, err = r.s3.PutObject(ctx, VerificationBucket, fmt.Sprintf("%s/%s/%s", r.folder, userID, ruleFH.Filename), rule, ruleFH.Size, minio.PutObjectOptions{})
 	if r.err(err) {
 		return nil, err
 	}
+	r.recordMetrics(ruleFH.Size, time.Since(startTime).Nanoseconds())
 
 	// Put certificate file
 	if certFH != nil {
@@ -86,10 +90,12 @@ func (r *VerificationRepository) Insert(
 			return nil, err
 		}
 
+		startTime = time.Now()
 		_, err = r.s3.PutObject(ctx, VerificationBucket, fmt.Sprintf("%s/%s/%s", r.folder, userID, certFH.Filename), cert, certFH.Size, minio.PutObjectOptions{})
 		if r.err(err) {
 			return nil, err
 		}
+		r.recordMetrics(certFH.Size, time.Since(startTime).Nanoseconds())
 
 		record.Files = []models.VerificationFile{
 			{
@@ -253,8 +259,8 @@ func (r *VerificationRepository) err(err error) bool {
 	return false
 }
 
-func (r *VerificationRepository) recordMetrics(bytes int64, duration time.Duration) {
+func (r *VerificationRepository) recordMetrics(bytes int64, duration int64) {
-	atomic.AddInt64(&r.metrics.totalBytes, bytes)
+	atomic.AddInt64(&r.Metrics.TotalBytes, bytes)
-	atomic.AddInt64(&r.metrics.totalTime, int64(duration))
+	atomic.AddInt64(&r.Metrics.TotalTime, duration)
-	atomic.AddInt64(&r.metrics.totalUploads, 1)
+	atomic.AddInt64(&r.Metrics.TotalUploads, 1)
 }

internal/server/http.go

@@ -3,12 +3,12 @@ package server
 import (
 	"context"
 	"fmt"
+	"gitea.pena/PenaSide/common/log_mw"
 	"github.com/gofiber/contrib/fiberzap"
 	"github.com/gofiber/fiber/v2"
 	"github.com/gofiber/fiber/v2/middleware/recover"
 	"github.com/themakers/hlog"
 	"go.uber.org/zap"
-	"gitea.pena/PenaSide/common/log_mw"
 )
 
 type ServerConfig struct {

test.env

@@ -1,20 +1,17 @@
-TELEGRAM_TOKEN=6712573453:AAFbioUuXf0Te73MUCqa0_h09qEQ1iQREas
+TELEGRAM_TOKEN=6712573453:AAFqTOsgwe_j48ZQ1GzWKQDT5Nwr-SAWjz8
-TELEGRAM_CHANNEL_ID=542073142
+TELEGRAM_CHANNEL_ID=-1002217604546
 HTTP_ADDRESS_USER=0.0.0.0:8080
 HTTP_ADDRESS_ADMIN=0.0.0.0:8081
 MONGO_HOST=localhost
-MONGO_PORT=27020
+MONGO_URL=mongodb://test:test@localhost:27020/
-MONGO_USER=test
+MONGO_DB_NAME=admin
-MONGO_PASSWORD=test
+S3_ENDPOINT=localhost:9000
-MONGO_AUTH=admin
+S3_ACCESS_KEY_ID=minioadmin
-MONGO_DATABASE_NAME=admin
+S3_SECRET_KEY=minioadmin
-S3_ENDPOINT=localhost:9005
-S3_ACCESS_KEY_ID=admin
-S3_SECRET_KEY=admin123
 CUSTOMER_SVC_ADDRESS=localhost:9003
 STAGING_URL=https://sadmin.pena
 S3_FOLDER=videodata
-S3_FILE_URL=http://localhost:9005/videodata
+S3_FILE_URL=http://localhost:9000/videodata
 CUSTOMER_RPC_HOST=localhost:9003
 TRASH_LOG_HOST=localhost:7113
 MODULE_LOGGER=verify-local

tests/e2e/create_verify_user_test.go

@@ -3,12 +3,12 @@ package e2e
 import (
 	"bytes"
 	"fmt"
+	"gitea.pena/PenaSide/verification/tests/helpers"
 	"github.com/gofiber/fiber/v2"
 	"github.com/stretchr/testify/assert"
 	"io"
 	"mime/multipart"
 	"os"
-	"gitea.pena/PenaSide/verification/tests/helpers"
 	"testing"
 )
 

tests/e2e/prometheus_test.go

@@ -0,0 +1,30 @@
+package e2e
+
+import (
+	"fmt"
+	"gitea.pena/PenaSide/verification/tests/helpers"
+	"github.com/gofiber/fiber/v2"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func Test_Metrics(t *testing.T) {
+	url := "http://localhost:8081/metrics"
+	client := fiber.AcquireClient()
+
+	token, err := helpers.CreateJwt("64e53ed187392e122e5d3d50")
+	assert.NoError(t, err)
+
+	assert.NoError(t, err)
+
+	agent := client.Get(url)
+	agent.Set("Authorization", fmt.Sprintf("Bearer %s", token))
+	statusCode, respBody, errs := agent.Bytes()
+	if len(errs) > 0 {
+		assert.NoError(t, errs[0])
+	}
+	assert.Equal(t, 200, statusCode)
+
+	fmt.Println(string(respBody))
+
+}