Merge remote-tracking branch 'origin/dev' into testSSE

2024-11-21 16:49:37 +03:00 · 2024-11-21 16:49:37 +03:00 · 4533cd318b
commit 4533cd318b
parent bf66e90a91 dea05acf3f
5 changed files with 234 additions and 15 deletions
--- a/go.mod
+++ b/go.mod
@ -4,6 +4,7 @@ go 1.22.0

 require (
 	gitea.pena/PenaSide/linters-golang v0.0.0-20241114215743-9a8e7d58cf96
+	gitea.pena/PenaSide/trashlog v0.0.0-20241107132923-0f7d4d57eb4b
 	github.com/go-resty/resty/v2 v2.11.0
 	github.com/gofiber/fiber/v2 v2.52.1
 	github.com/golang-jwt/jwt/v5 v5.2.0
@ -24,7 +25,6 @@ require (
 	gopkg.in/tucnak/telebot.v2 v2.5.0
 	penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240607202348-efe5f2bf3e8c
 	penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240520145524-451212248881
-	penahub.gitlab.yandexcloud.net/external/trashlog v0.1.5
 )

 require (
@ -70,5 +70,4 @@ require (
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	penahub.gitlab.yandexcloud.net/devops/linters/golang.git v0.0.0-20240829220549-d35409b619a3 // indirect
 )
--- a/go.sum
+++ b/go.sum
@ -3,6 +3,8 @@ gitea.pena/PenaSide/common v0.0.0-20241120141501-1695a0981562 h1:LobhWlICMcbCCI+
 gitea.pena/PenaSide/common v0.0.0-20241120141501-1695a0981562/go.mod h1:l71j3W1yROhOSfjWZ6wcMuzjBR37gu2ZTcXsorEJoiw=
 gitea.pena/PenaSide/linters-golang v0.0.0-20241114215743-9a8e7d58cf96 h1:m4EMXEhsA/glI6eJeZnRGUhYPSQdcWj3hzT2IDNlWS0=
 gitea.pena/PenaSide/linters-golang v0.0.0-20241114215743-9a8e7d58cf96/go.mod h1:gdd+vOT6up9STkEbxa2qESLIMZFjCmRbkcheFQCVgZU=
+gitea.pena/PenaSide/trashlog v0.0.0-20241107132923-0f7d4d57eb4b h1:wVswXBfVDI6xi16o2A0wt8i6nswHkgPdd6U7ak0KUjo=
+gitea.pena/PenaSide/trashlog v0.0.0-20241107132923-0f7d4d57eb4b/go.mod h1:LwHJCrPVumS6Rdorrr8NLEi4kpCed6ZVcIYnmEqbIVM=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0=
 github.com/ClickHouse/clickhouse-go v1.5.4/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=
@ -320,7 +322,3 @@ penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240607202348-efe5
 penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240607202348-efe5f2bf3e8c/go.mod h1:+bPxq2wfW5S1gd+83vZYmHm33AE7nEBfznWS8AM1TKE=
 penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240520145524-451212248881 h1:U1/WGQdwZsmrV/ta7Uqm13Dg07IPN/5omS8gzBJYZv4=
 penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240520145524-451212248881/go.mod h1:oRyhT55ctjqp/7ZxIzkR7OsQ7T/NLibsfrbb7Ytns64=
-penahub.gitlab.yandexcloud.net/devops/linters/golang.git v0.0.0-20240829220549-d35409b619a3 h1:sf6e2mp582L3i/FMDd2q6QuWm1njRXzYpIX0SipsvM4=
-penahub.gitlab.yandexcloud.net/devops/linters/golang.git v0.0.0-20240829220549-d35409b619a3/go.mod h1:i7M72RIpkSjcQtHID6KKj9RT/EYZ1rxS6tIPKWa/BSY=
-penahub.gitlab.yandexcloud.net/external/trashlog v0.1.5 h1:amsK0bkSJxBisk334aFo5ZmVPvN1dBT0Sv5j3V5IsT8=
-penahub.gitlab.yandexcloud.net/external/trashlog v0.1.5/go.mod h1:J8kQNEP4bL7ZNKHxuT4tfe6a3FHyovpAPkyytN4qllc=
--- a/internal/app/app.go
+++ b/internal/app/app.go
@ -4,25 +4,25 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"gitea.pena/PenaSide/customer/internal/interface/controller/http"
+	"gitea.pena/PenaSide/trashlog/app"
+	"gitea.pena/PenaSide/trashlog/wrappers/zaptrashlog"
 	"github.com/themakers/hlog"
 	"go.uber.org/zap/zapcore"
 	tb "gopkg.in/tucnak/telebot.v2"
 	"os/signal"
 	"penahub.gitlab.yandexcloud.net/backend/penahub_common/mongo"
-	"penahub.gitlab.yandexcloud.net/external/trashlog/app"
-	"penahub.gitlab.yandexcloud.net/external/trashlog/wrappers/zaptrashlog"
-	"gitea.pena/PenaSide/customer/internal/interface/controller/http"
 	"syscall"
 	"time"

-	"github.com/twmb/franz-go/pkg/kgo"
-	"go.uber.org/zap"
-	qutils "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/utils"
 	"gitea.pena/PenaSide/customer/internal/initialize"
 	"gitea.pena/PenaSide/customer/internal/models"
 	"gitea.pena/PenaSide/customer/internal/server"
 	"gitea.pena/PenaSide/customer/pkg/closer"
 	"gitea.pena/PenaSide/customer/pkg/kafka"
+	"github.com/twmb/franz-go/pkg/kgo"
+	"go.uber.org/zap"
+	qutils "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/utils"
 )

 const (
--- a/scripts/data_to_clickhouse/main.go
+++ b/scripts/data_to_clickhouse/main.go
@ -0,0 +1,222 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"gitea.pena/PenaSide/customer/internal/interface/client"
+	"gitea.pena/PenaSide/customer/internal/models"
+	"gitea.pena/PenaSide/customer/internal/utils"
+	"gitea.pena/PenaSide/trashlog/wrappers/zaptrashlog"
+	"github.com/themakers/hlog"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+	"log"
+	"time"
+)
+
+var TrashLogHost = "localhost:7113"
+var Version = "replenishment"
+var Commit = "last"
+var BuildTime = time.Now().Unix()
+var ModuleLogger = "customer-replenishment"
+var AuthServiceUrl = ""
+var MongoURL = "mongodb://test:test@localhost:27020/?authSource=admin"
+var MongoDatabase = "admin"
+var AccountCollection = "accounts"
+var HistoryCollection = "history"
+
+func main() {
+	ctx := context.Background()
+	logger, err := zap.NewProduction(zap.AddStacktrace(zap.DPanicLevel))
+	if err != nil {
+		log.Fatalf("failed to init zap logger: %v", err)
+	}
+
+	clickHouseLogger, err := zaptrashlog.NewCore(ctx, zap.InfoLevel, TrashLogHost, Version, Commit, BuildTime)
+	if err != nil {
+		panic(err)
+	}
+
+	loggerForHlog := logger.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core {
+		return zapcore.NewTee(core, clickHouseLogger)
+	}), zap.AddCallerSkip(2))
+
+	loggerHlog := hlog.New(loggerForHlog).Module(ModuleLogger)
+	loggerHlog.With(models.AllFields{})
+
+	authClient := client.NewAuthClient(client.AuthClientDeps{
+		Logger: logger,
+		URLs: &models.AuthMicroserviceURL{
+			User: AuthServiceUrl,
+		},
+	})
+
+	mongoClient, err := connectToMongoDB(ctx, MongoURL)
+	if err != nil {
+		logger.Fatal("failed to connect to MongoDB", zap.Error(err))
+	}
+	defer func() {
+		if err := mongoClient.Disconnect(ctx); err != nil {
+			logger.Error("failed to disconnect MongoDB client", zap.Error(err))
+		}
+	}()
+
+	err = SetAccountData(ctx, loggerHlog, authClient, mongoClient)
+	if err != nil {
+		logger.Error("failed set account data to clickHouse", zap.Error(err))
+		return
+	}
+
+	err = SetPayCartData(ctx, loggerHlog, mongoClient)
+	if err != nil {
+		logger.Error("failed set pay cart data to clickHouse", zap.Error(err))
+		return
+	}
+
+	time.Sleep(15 * time.Second)
+}
+
+func connectToMongoDB(ctx context.Context, mongoURL string) (*mongo.Client, error) {
+	clientOpts := options.Client().ApplyURI(mongoURL)
+	client, err := mongo.Connect(ctx, clientOpts)
+	if err != nil {
+		return nil, err
+	}
+
+	err = client.Ping(ctx, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	return client, nil
+}
+
+func SetAccountData(ctx context.Context, logger hlog.Logger, authClient *client.AuthClient, mongoClient *mongo.Client) error {
+	cl := mongoClient.Database(MongoDatabase).Collection(AccountCollection)
+	filter := bson.M{"isDeleted": false}
+	opts := options.Find()
+	cursor, err := cl.Find(ctx, filter, opts)
+	if err != nil {
+		return err
+	}
+	defer cursor.Close(ctx)
+
+	var accounts []models.Account
+	for cursor.Next(ctx) {
+		var account models.Account
+		if err := cursor.Decode(&account); err != nil {
+			return err
+		}
+		accounts = append(accounts, account)
+	}
+
+	if err := cursor.Err(); err != nil {
+		return err
+	}
+
+	for _, account := range accounts {
+		logger.Emit(models.InfoGetAccount{
+			CtxUserID:    account.UserID,
+			CtxAccountID: account.ID,
+		})
+
+		for _, item := range account.Cart {
+			logger.Emit(models.InfoAddToCart{
+				CtxUserID:    account.UserID,
+				CtxAccountID: account.ID,
+				CtxTariffID:  item,
+			})
+		}
+
+		userData, err := authClient.GetUser(ctx, account.UserID)
+		if err != nil {
+			fmt.Printf("error getting user from auth service,scip him, userid:%s, error: %s", account.UserID, err.Error())
+			continue
+		}
+		quiz := ""
+		if account.From != "" {
+			quiz = "quiz"
+		}
+
+		logger.Emit(models.InfoCreateAccount{
+			CtxUserID:      account.UserID,
+			CtxAccountID:   account.ID,
+			KeyFromSource:  quiz,
+			KeyFromID:      account.From,
+			KeyFromPartner: account.Partner,
+			CtxLogin:       userData.Login,
+			CtxEmail:       userData.Email,
+			CtxPhone:       userData.PhoneNumber,
+			KeyCurrency:    account.Wallet.Currency,
+		})
+	}
+
+	return nil
+}
+
+func SetPayCartData(ctx context.Context, logger hlog.Logger, mongoClient *mongo.Client) error {
+	clHistory := mongoClient.Database(MongoDatabase).Collection(HistoryCollection)
+	clAccount := mongoClient.Database(MongoDatabase).Collection(AccountCollection)
+	filter := bson.M{"isDeleted": false}
+	opts := options.Find()
+	cursor, err := clHistory.Find(ctx, filter, opts)
+	if err != nil {
+		return err
+	}
+	defer cursor.Close(ctx)
+
+	var histories []models.History
+	for cursor.Next(ctx) {
+		var history models.History
+		if err := cursor.Decode(&history); err != nil {
+			return err
+		}
+		histories = append(histories, history)
+	}
+
+	if err := cursor.Err(); err != nil {
+		return err
+	}
+
+	for _, history := range histories {
+		currentAccount, err := findByUserID(ctx, clAccount, history.UserID)
+		if err != nil {
+			fmt.Printf("Error getting account by userID:%s", err.Error())
+			continue
+		}
+
+		logger.Emit(models.InfoPayCart{
+			CtxUserID:    history.UserID,
+			CtxAccountID: currentAccount.ID,
+			KeySuccess:   uint8(1),
+			CtxPrice:     history.RawDetails.Price,
+			//CtxTariff:    strings.Join(account.Cart, ","),
+			//CtxDiscount:  strings.Join(utils.GetAppliedDiscountsIDs(discountResponse.AppliedDiscounts), ","),
+			//CtxRowPrice:  int64(tariffsAmount),
+			CtxRowData: utils.MarshalRawDetails(history.RawDetails),
+		})
+	}
+
+	return nil
+}
+
+func findByUserID(ctx context.Context, clAccount *mongo.Collection, userID string) (*models.Account, error) {
+	filter := bson.M{
+		"userId":    userID,
+		"isDeleted": false,
+	}
+	var account models.Account
+	err := clAccount.FindOne(ctx, filter).Decode(&account)
+	if err != nil {
+		if err == mongo.ErrNoDocuments {
+			fmt.Printf("No account found for userID: %s", userID)
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	return &account, nil
+}
--- a/tests/integration/trashLogger_test.go
+++ b/tests/integration/trashLogger_test.go
@ -3,15 +3,15 @@ package integration
 import (
 	"context"
 	"database/sql"
+	"gitea.pena/PenaSide/customer/internal/models"
+	"gitea.pena/PenaSide/trashlog/app"
+	"gitea.pena/PenaSide/trashlog/wrappers/zaptrashlog"
 	"github.com/stretchr/testify/assert"
 	"github.com/themakers/hlog"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 	"log"
 	"os"
-	"penahub.gitlab.yandexcloud.net/external/trashlog/app"
-	"penahub.gitlab.yandexcloud.net/external/trashlog/wrappers/zaptrashlog"
-	"gitea.pena/PenaSide/customer/internal/models"
 	"testing"
 	"time"
 )