From 05f044c17b2002781909360e717ec30d5ffa4832 Mon Sep 17 00:00:00 2001 From: Pavel Date: Sun, 2 Jun 2024 11:19:22 +0300 Subject: [PATCH 01/13] init hlog and add hlog events --- app/app.go | 38 ++++++++++++++------ go.mod | 6 +++- go.sum | 29 ++++++++++++++-- models/hlog_events.go | 81 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 140 insertions(+), 14 deletions(-) create mode 100644 models/hlog_events.go diff --git a/app/app.go b/app/app.go index fe7f058..2cf8c32 100644 --- a/app/app.go +++ b/app/app.go @@ -8,6 +8,7 @@ import ( "github.com/skeris/appInit" "github.com/themakers/hlog" "go.uber.org/zap" + "go.uber.org/zap/zapcore" "penahub.gitlab.yandexcloud.net/backend/penahub_common/privilege" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/dal" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/healthchecks" @@ -17,9 +18,11 @@ import ( "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/brokers" "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/clients/auth" "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/initialize" + "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/models" "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/server" "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/service" "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/tools" + "penahub.gitlab.yandexcloud.net/external/trashlog.git/wrappers/zaptrashlog" "time" ) @@ -66,6 +69,8 @@ type Options struct { KafkaBrokers string `env:"KAFKA_BROKERS" default:"localhost:9092"` KafkaTopic string `env:"KAFKA_TOPIC" default:"test-topic"` KafkaGroup string `env:"KAFKA_GROUP" default:"mailnotifier"` + TrashLogHost string `env:"TRASH_LOG_HOST" default:"localhost:7113"` + ModuleLogger string `env:"MODULE_LOGGER" default:"core-local"` } func New(ctx context.Context, opts interface{}, ver appInit.Version) (appInit.CommonApp, error) { @@ -93,14 +98,24 @@ func New(ctx context.Context, opts interface{}, ver appInit.Version) (appInit.Co } } - zapLogger = zapLogger.With( - zap.String("SvcCommit", ver.Commit), - zap.String("SvcVersion", ver.Release), - zap.String("SvcBuildTime", ver.BuildTime), - ) + //zapLogger = zapLogger.With( + // zap.String("SvcCommit", ver.Commit), + // zap.String("SvcVersion", ver.Release), + // zap.String("SvcBuildTime", ver.BuildTime), + //) - logger := hlog.New(zapLogger) - logger.Emit(InfoSvcStarted{}) + clickHouseLogger, err := zaptrashlog.NewCore(ctx, zap.InfoLevel, options.TrashLogHost, ver.Release, ver.Commit, time.Now().Unix()) + if err != nil { + panic(err) + } + + loggerForHlog := zapLogger.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core { + return zapcore.NewTee(core, clickHouseLogger) + })) + + loggerHlog := hlog.New(loggerForHlog).Module(options.ModuleLogger) + loggerHlog.With(models.AllFields{}) + loggerHlog.Emit(InfoSvcStarted{}) authClient := auth.NewAuthClient(options.AuthServiceURL) @@ -150,6 +165,7 @@ func New(ctx context.Context, opts interface{}, ver appInit.Version) (appInit.Co encrypt := utils.NewEncrypt(options.PubKey, options.PrivKey) app := fiber.New() app.Use(middleware.JWTAuth()) + app.Use(middleware.ContextLogger(loggerHlog)) app.Get("/liveness", healthchecks.Liveness) app.Get("/readiness", healthchecks.Readiness(&workerErr)) //todo parametrized readiness. should discuss ready reason @@ -164,7 +180,7 @@ func New(ctx context.Context, opts interface{}, ver appInit.Version) (appInit.Co svc.Register(app) - logger.Emit(InfoSvcReady{}) + loggerHlog.Emit(InfoSvcReady{}) go func() { defer func() { @@ -173,19 +189,19 @@ func New(ctx context.Context, opts interface{}, ver appInit.Version) (appInit.Co } err := grpc.Stop(ctx) err = app.Shutdown() - logger.Emit(InfoSvcShutdown{Signal: err.Error()}) + loggerHlog.Emit(InfoSvcShutdown{Signal: err.Error()}) }() if options.IsProd { if err := app.ListenTLS(fmt.Sprintf(":%s", options.NumberPort), options.CrtFile, options.KeyFile); err != nil { - logger.Emit(ErrorCanNotServe{ + loggerHlog.Emit(ErrorCanNotServe{ Err: err, }) errChan <- err } } else { if err := app.Listen(fmt.Sprintf(":%s", options.NumberPort)); err != nil { - logger.Emit(ErrorCanNotServe{ + loggerHlog.Emit(ErrorCanNotServe{ Err: err, }) errChan <- err diff --git a/go.mod b/go.mod index 0adce39..5bb6584 100644 --- a/go.mod +++ b/go.mod @@ -18,13 +18,16 @@ require ( google.golang.org/grpc v1.64.0 google.golang.org/protobuf v1.34.1 penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240223054633-6cb3d5ce45b6 - penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240520145524-451212248881 + penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240601180932-af93405efbcf penahub.gitlab.yandexcloud.net/backend/quiz/worker.git v0.0.0-20240421230341-0e086fcbb990 + penahub.gitlab.yandexcloud.net/external/trashlog.git v0.1.2-0.20240523172059-9bbe8a9faa31 ) require ( + github.com/ClickHouse/clickhouse-go v1.5.4 // indirect github.com/andybalholm/brotli v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -54,6 +57,7 @@ require ( github.com/valyala/tcplisten v1.0.0 // indirect github.com/xuri/efp v0.0.0-20240408161823-9ad904a10d6d // indirect github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 // indirect + go.etcd.io/bbolt v1.3.6 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.23.0 // indirect golang.org/x/net v0.25.0 // indirect diff --git a/go.sum b/go.sum index 47df28f..5534e0a 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,18 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0= +github.com/ClickHouse/clickhouse-go v1.5.4/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk= +github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg= +github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -20,12 +26,15 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= @@ -50,6 +59,7 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -68,6 +78,7 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -77,6 +88,7 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.70 h1:1u9NtMgfK1U42kUxcsl5v0yj6TEOPR497OAQxpJnn2g= @@ -90,12 +102,16 @@ github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042 github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pioz/faker v1.7.3 h1:Tez8Emuq0UN+/d6mo3a9m/9ZZ/zdfJk0c5RtRatrceM= github.com/pioz/faker v1.7.3/go.mod h1:xSpay5w/oz1a6+ww0M3vfpe40pSIykeUPeWEc3TvVlc= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -126,6 +142,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tealeg/xlsx v1.0.5 h1:+f8oFmvY8Gw1iUXzPk+kz+4GpbDZPK1FhPiQRd+ypgE= github.com/tealeg/xlsx v1.0.5/go.mod h1:btRS8dz54TDnvKNosuAqxrM1QgN1udgk9O34bDCnORM= +github.com/themakers/bdd v0.0.0-20210316111417-6b1dfe326f33 h1:N9f/Q+2Ssa+yDcbfaoLTYvXmdeyUUxsJKdPUVsjSmiA= +github.com/themakers/bdd v0.0.0-20210316111417-6b1dfe326f33/go.mod h1:rpcH99JknBh8seZmlOlUg51gasZH6QH34oXNsIwYT6E= github.com/themakers/hlog v0.0.0-20191205140925-235e0e4baddf h1:TJJm6KcBssmbWzplF5lzixXl1RBAi/ViPs1GaSOkhwo= github.com/themakers/hlog v0.0.0-20191205140925-235e0e4baddf/go.mod h1:1FsorU3vnXO9xS9SrhUp8fRb/6H/Zfll0rPt1i4GWaA= github.com/twmb/franz-go v1.16.1 h1:rpWc7fB9jd7TgmCyfxzenBI+QbgS8ZfJOUQE+tzPtbE= @@ -146,6 +164,8 @@ github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 h1:hPVCafDV85blFTabnqKgNh github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= @@ -196,6 +216,7 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -249,6 +270,8 @@ gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/tucnak/telebot.v2 v2.5.0 h1:i+NynLo443Vp+Zn3Gv9JBjh3Z/PaiKAQwcnhNI7y6Po= +gopkg.in/tucnak/telebot.v2 v2.5.0/go.mod h1:BgaIIx50PSRS9pG59JH+geT82cfvoJU/IaI5TJdN3v8= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= @@ -262,7 +285,9 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240223054633-6cb3d5ce45b6 h1:oV+/HNX+JPoQ3/GUx08hio7d45WpY0AMGrFs7j70QlA= penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240223054633-6cb3d5ce45b6/go.mod h1:lTmpjry+8evVkXWbEC+WMOELcFkRD1lFMc7J09mOndM= -penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240520145524-451212248881 h1:U1/WGQdwZsmrV/ta7Uqm13Dg07IPN/5omS8gzBJYZv4= -penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240520145524-451212248881/go.mod h1:oRyhT55ctjqp/7ZxIzkR7OsQ7T/NLibsfrbb7Ytns64= +penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240601180932-af93405efbcf h1:BnzAkKXo6CksQTqpVzsCTSMdWNwMpkBKb8eS/DDDHBI= +penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240601180932-af93405efbcf/go.mod h1:n66zm88Dh12+idyfqh0vU5nd9BZYxM6Pv0XYnmy0398= penahub.gitlab.yandexcloud.net/backend/quiz/worker.git v0.0.0-20240421230341-0e086fcbb990 h1:jiO8GWO+3sCnDAV8/NAV8tQIUwae/I6/xiDilW7zf0o= penahub.gitlab.yandexcloud.net/backend/quiz/worker.git v0.0.0-20240421230341-0e086fcbb990/go.mod h1:zswBuTwmEsFHBVRu1nkG3/Fwylk5Vcm8OUm9iWxccSE= +penahub.gitlab.yandexcloud.net/external/trashlog.git v0.1.2-0.20240523172059-9bbe8a9faa31 h1:WlRVJnzU0sti+qBq/JTCgFPU0RoxIqGHu7hzDirxE2k= +penahub.gitlab.yandexcloud.net/external/trashlog.git v0.1.2-0.20240523172059-9bbe8a9faa31/go.mod h1:3ml0dAGT8U8RhpevKBfRgG6yKZum8EI2uJxAb2WCIy4= diff --git a/models/hlog_events.go b/models/hlog_events.go new file mode 100644 index 0000000..fe85164 --- /dev/null +++ b/models/hlog_events.go @@ -0,0 +1,81 @@ +package models + +type AllFields struct { + CtxUserIP string + CtxUserPort string + KeyDomain string + KeyPath string + CtxUserID string + CtxAccountID string + CtxID string + CtxQuizID string + CtxQuestionID string +} + +type InfoAccountCreated struct { + CtxUserIP string + CtxUserPort string + KeyDomain string + KeyPath string + CtxUserID string //айдишник юзера из токена + CtxAccountID string // айдишник свежесозданного аккаунта +} +type InfoQuizCreated struct { + CtxUserIP string + CtxUserPort string + KeyDomain string + KeyPath string + CtxUserID string //айдишник юзера из токена + CtxID string //айдишник создаваемого квиза +} +type InfoQuizPublish struct { // если статус меняется на start + CtxUserIP string + CtxUserPort string + KeyDomain string + KeyPath string + CtxUserID string //айдишник юзера из токена + CtxID string //айдишник публикуемого квиза +} +type InfoQuizDelete struct { // если квиз удаляется + CtxUserIP string + CtxUserPort string + KeyDomain string + KeyPath string + CtxUserID string //айдишник юзера из токена + CtxID string //айдишник удаляемого квиза +} +type InfoQuizStop struct { // если статус меняется на stop + CtxUserIP string + CtxUserPort string + KeyDomain string + KeyPath string + CtxUserID string //айдишник юзера из токена + CtxID string //айдишник останавливаемого квиза +} +type InfoQuizTemplateCopy struct { + CtxUserIP string + CtxUserPort string + KeyDomain string + KeyPath string + CtxUserID string //айдишник юзера из токена + CtxID string //айдишник копируемого квиза + CtxQuizID string // айдишник опроса после копирования +} +type InfoQuestionCreate struct { + CtxUserIP string + CtxUserPort string + KeyDomain string + KeyPath string + CtxUserID string //айдишник юзера из токена + CtxID string //айдишник останавливаемого квиза + CtxQuestionID string // айдишник вопроса +} +type InfoQuestionDelete struct { + CtxUserIP string + CtxUserPort string + KeyDomain string + KeyPath string + CtxUserID string //айдишник юзера из токена + CtxID string //айдишник останавливаемого квиза + CtxQuestionID string // айдишник вопроса +} From 8a0071e81b129d24398146c74996c4770bf9b177 Mon Sep 17 00:00:00 2001 From: Pavel Date: Sun, 2 Jun 2024 12:36:22 +0300 Subject: [PATCH 02/13] init hlog events in core methods --- go.mod | 2 +- go.sum | 4 +-- models/hlog_events.go | 26 ++++++++--------- service/account_svc.go | 16 +++++++++-- service/question_svc.go | 38 ++++++++++++++++++++++++- service/quiz_svc.go | 62 ++++++++++++++++++++++++++++++++++++++++- 6 files changed, 128 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index 5bb6584..8c919fb 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( google.golang.org/grpc v1.64.0 google.golang.org/protobuf v1.34.1 penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240223054633-6cb3d5ce45b6 - penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240601180932-af93405efbcf + penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240602092938-b0f7572884dc penahub.gitlab.yandexcloud.net/backend/quiz/worker.git v0.0.0-20240421230341-0e086fcbb990 penahub.gitlab.yandexcloud.net/external/trashlog.git v0.1.2-0.20240523172059-9bbe8a9faa31 ) diff --git a/go.sum b/go.sum index 5534e0a..768b096 100644 --- a/go.sum +++ b/go.sum @@ -285,8 +285,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240223054633-6cb3d5ce45b6 h1:oV+/HNX+JPoQ3/GUx08hio7d45WpY0AMGrFs7j70QlA= penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240223054633-6cb3d5ce45b6/go.mod h1:lTmpjry+8evVkXWbEC+WMOELcFkRD1lFMc7J09mOndM= -penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240601180932-af93405efbcf h1:BnzAkKXo6CksQTqpVzsCTSMdWNwMpkBKb8eS/DDDHBI= -penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240601180932-af93405efbcf/go.mod h1:n66zm88Dh12+idyfqh0vU5nd9BZYxM6Pv0XYnmy0398= +penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240602092938-b0f7572884dc h1:EpgjB6mhOey0aBdkkNPSdMPVYKGrYl2JUPNKcq6/SZM= +penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240602092938-b0f7572884dc/go.mod h1:n66zm88Dh12+idyfqh0vU5nd9BZYxM6Pv0XYnmy0398= penahub.gitlab.yandexcloud.net/backend/quiz/worker.git v0.0.0-20240421230341-0e086fcbb990 h1:jiO8GWO+3sCnDAV8/NAV8tQIUwae/I6/xiDilW7zf0o= penahub.gitlab.yandexcloud.net/backend/quiz/worker.git v0.0.0-20240421230341-0e086fcbb990/go.mod h1:zswBuTwmEsFHBVRu1nkG3/Fwylk5Vcm8OUm9iWxccSE= penahub.gitlab.yandexcloud.net/external/trashlog.git v0.1.2-0.20240523172059-9bbe8a9faa31 h1:WlRVJnzU0sti+qBq/JTCgFPU0RoxIqGHu7hzDirxE2k= diff --git a/models/hlog_events.go b/models/hlog_events.go index fe85164..9cb6bbd 100644 --- a/models/hlog_events.go +++ b/models/hlog_events.go @@ -7,9 +7,9 @@ type AllFields struct { KeyPath string CtxUserID string CtxAccountID string - CtxID string - CtxQuizID string - CtxQuestionID string + CtxID uint64 + CtxQuizID uint64 + CtxQuestionID uint64 } type InfoAccountCreated struct { @@ -26,7 +26,7 @@ type InfoQuizCreated struct { KeyDomain string KeyPath string CtxUserID string //айдишник юзера из токена - CtxID string //айдишник создаваемого квиза + CtxID uint64 //айдишник создаваемого квиза } type InfoQuizPublish struct { // если статус меняется на start CtxUserIP string @@ -34,7 +34,7 @@ type InfoQuizPublish struct { // если статус меняется на sta KeyDomain string KeyPath string CtxUserID string //айдишник юзера из токена - CtxID string //айдишник публикуемого квиза + CtxID uint64 //айдишник публикуемого квиза } type InfoQuizDelete struct { // если квиз удаляется CtxUserIP string @@ -42,7 +42,7 @@ type InfoQuizDelete struct { // если квиз удаляется KeyDomain string KeyPath string CtxUserID string //айдишник юзера из токена - CtxID string //айдишник удаляемого квиза + CtxID uint64 //айдишник удаляемого квиза } type InfoQuizStop struct { // если статус меняется на stop CtxUserIP string @@ -50,7 +50,7 @@ type InfoQuizStop struct { // если статус меняется на stop KeyDomain string KeyPath string CtxUserID string //айдишник юзера из токена - CtxID string //айдишник останавливаемого квиза + CtxID uint64 //айдишник останавливаемого квиза } type InfoQuizTemplateCopy struct { CtxUserIP string @@ -58,8 +58,8 @@ type InfoQuizTemplateCopy struct { KeyDomain string KeyPath string CtxUserID string //айдишник юзера из токена - CtxID string //айдишник копируемого квиза - CtxQuizID string // айдишник опроса после копирования + CtxID uint64 //айдишник копируемого квиза + CtxQuizID uint64 // айдишник опроса после копирования } type InfoQuestionCreate struct { CtxUserIP string @@ -67,8 +67,8 @@ type InfoQuestionCreate struct { KeyDomain string KeyPath string CtxUserID string //айдишник юзера из токена - CtxID string //айдишник останавливаемого квиза - CtxQuestionID string // айдишник вопроса + CtxID uint64 //айдишник останавливаемого квиза + CtxQuestionID uint64 // айдишник вопроса } type InfoQuestionDelete struct { CtxUserIP string @@ -76,6 +76,6 @@ type InfoQuestionDelete struct { KeyDomain string KeyPath string CtxUserID string //айдишник юзера из токена - CtxID string //айдишник останавливаемого квиза - CtxQuestionID string // айдишник вопроса + CtxID uint64 //айдишник останавливаемого квиза + CtxQuestionID uint64 // айдишник вопроса } diff --git a/service/account_svc.go b/service/account_svc.go index d9b6832..5ad2fca 100644 --- a/service/account_svc.go +++ b/service/account_svc.go @@ -6,6 +6,8 @@ import ( "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/middleware" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/model" "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/brokers" + "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/models" + "strings" "time" ) @@ -69,6 +71,7 @@ func (s *Service) createAccount(ctx *fiber.Ctx) error { if !ok { return ctx.Status(fiber.StatusUnauthorized).SendString("account id is required") } + hlogger := middleware.GetLogger(ctx) existingAccount, err := s.dal.AccountRepo.GetAccountByID(ctx.Context(), accountID) if err != nil && err != sql.ErrNoRows { @@ -98,17 +101,26 @@ func (s *Service) createAccount(ctx *fiber.Ctx) error { }, } - if err := s.dal.AccountRepo.CreateAccount(ctx.Context(), &newAccount); err != nil { + createdAcc, err := s.dal.AccountRepo.CreateAccount(ctx.Context(), &newAccount) + if err != nil { return ctx.Status(fiber.StatusInternalServerError).SendString(err.Error()) } + hlogger.Emit(models.InfoAccountCreated{ + CtxUserIP: ctx.IP(), + CtxUserPort: ctx.Port(), + KeyDomain: strings.Join(ctx.Subdomains(), "/"), + KeyPath: ctx.Path(), + CtxUserID: accountID, + CtxAccountID: createdAcc.ID, + }) + err = s.producer.ToMailNotify(ctx.Context(), brokers.Message{ AccountID: accountID, Email: email, ServiceKey: s.serviceName, SendAt: time.Now(), }) - if err != nil { return ctx.Status(fiber.StatusInternalServerError).SendString(err.Error()) } diff --git a/service/question_svc.go b/service/question_svc.go index d1ba465..da9d56c 100644 --- a/service/question_svc.go +++ b/service/question_svc.go @@ -3,7 +3,10 @@ package service import ( "github.com/gofiber/fiber/v2" "github.com/lib/pq" + "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/middleware" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/model" + "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/models" + "strings" "unicode/utf8" ) @@ -21,6 +24,12 @@ type QuestionCreateReq struct { // CreateQuestion service handler for creating question for quiz func (s *Service) CreateQuestion(ctx *fiber.Ctx) error { + accountID, ok := middleware.GetAccountId(ctx) + if !ok { + return ctx.Status(fiber.StatusUnauthorized).SendString("account id is required") + } + hlogger := middleware.GetLogger(ctx) + var req QuestionCreateReq if err := ctx.BodyParser(&req); err != nil { return ctx.Status(fiber.StatusBadRequest).SendString("Invalid request data") @@ -54,7 +63,8 @@ func (s *Service) CreateQuestion(ctx *fiber.Ctx) error { Page: req.Page, Content: req.Content, } - if err := s.dal.QuestionRepo.CreateQuestion(ctx.Context(), &result); err != nil { + questionID, err := s.dal.QuestionRepo.CreateQuestion(ctx.Context(), &result) + if err != nil { if e, ok := err.(*pq.Error); ok { if e.Constraint == "quiz_relation" { return ctx.Status(fiber.StatusFailedDependency).SendString(e.Error()) @@ -63,6 +73,16 @@ func (s *Service) CreateQuestion(ctx *fiber.Ctx) error { return ctx.Status(fiber.StatusInternalServerError).SendString(err.Error()) } + hlogger.Emit(models.InfoQuestionCreate{ + CtxUserIP: ctx.IP(), + CtxUserPort: ctx.Port(), + KeyDomain: strings.Join(ctx.Subdomains(), "/"), + KeyPath: ctx.Path(), + CtxUserID: accountID, + CtxID: req.QuizId, + CtxQuestionID: questionID, + }) + return ctx.Status(fiber.StatusOK).JSON(result) } @@ -274,6 +294,12 @@ func (s *Service) GetQuestionHistory(ctx *fiber.Ctx) error { // DeleteQuestion handler for fake delete question func (s *Service) DeleteQuestion(ctx *fiber.Ctx) error { + accountID, ok := middleware.GetAccountId(ctx) + if !ok { + return ctx.Status(fiber.StatusUnauthorized).SendString("account id is required") + } + hlogger := middleware.GetLogger(ctx) + var req DeactivateReq if err := ctx.BodyParser(&req); err != nil { return ctx.Status(fiber.StatusBadRequest).SendString("Invalid request data") @@ -288,6 +314,16 @@ func (s *Service) DeleteQuestion(ctx *fiber.Ctx) error { return ctx.Status(fiber.StatusInternalServerError).SendString(err.Error()) } + hlogger.Emit(models.InfoQuestionDelete{ + CtxUserIP: ctx.IP(), + CtxUserPort: ctx.Port(), + KeyDomain: strings.Join(ctx.Subdomains(), "/"), + KeyPath: ctx.Path(), + CtxUserID: accountID, + CtxID: deleted.QuizId, + CtxQuestionID: deleted.Id, + }) + return ctx.JSON(DeactivateResp{ Deactivated: deleted.Id, }) diff --git a/service/quiz_svc.go b/service/quiz_svc.go index 56a74b1..34b8359 100644 --- a/service/quiz_svc.go +++ b/service/quiz_svc.go @@ -7,6 +7,8 @@ import ( "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/middleware" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/model" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/repository/quiz" + "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/models" + "strings" "time" "unicode/utf8" ) @@ -46,6 +48,8 @@ func (s *Service) CreateQuiz(ctx *fiber.Ctx) error { return ctx.Status(fiber.StatusUnauthorized).SendString("account id is required") } + hlogger := middleware.GetLogger(ctx) + // check that we can store name if utf8.RuneCountInString(req.Name) > 700 { return ctx.Status(fiber.StatusUnprocessableEntity).SendString("name field should have less then 700 chars") @@ -92,10 +96,20 @@ func (s *Service) CreateQuiz(ctx *fiber.Ctx) error { GroupId: req.GroupId, } - if err := s.dal.QuizRepo.CreateQuiz(ctx.Context(), &record); err != nil { + quizID, err := s.dal.QuizRepo.CreateQuiz(ctx.Context(), &record) + if err != nil { return ctx.Status(fiber.StatusInternalServerError).SendString(err.Error()) } + hlogger.Emit(models.InfoQuizCreated{ + CtxUserIP: ctx.IP(), + CtxUserPort: ctx.Port(), + KeyDomain: strings.Join(ctx.Subdomains(), "/"), + KeyPath: ctx.Path(), + CtxUserID: accountId, + CtxID: quizID, + }) + return ctx.Status(fiber.StatusCreated).JSON(record) } @@ -197,6 +211,8 @@ func (s *Service) UpdateQuiz(ctx *fiber.Ctx) error { return ctx.Status(fiber.StatusUnauthorized).SendString("account id is required") } + hlogger := middleware.GetLogger(ctx) + if req.Id == 0 { return ctx.Status(fiber.StatusFailedDependency).SendString("need id of question for update") } @@ -301,6 +317,27 @@ func (s *Service) UpdateQuiz(ctx *fiber.Ctx) error { return ctx.Status(fiber.StatusInternalServerError).SendString(err.Error()) } + if req.Status == model.StatusStart { + hlogger.Emit(models.InfoQuizPublish{ + CtxUserIP: ctx.IP(), + CtxUserPort: ctx.Port(), + KeyDomain: strings.Join(ctx.Subdomains(), "/"), + KeyPath: ctx.Path(), + CtxUserID: accountId, + CtxID: quiz.Id, + }) + } + if req.Status == model.StatusStop { + hlogger.Emit(models.InfoQuizStop{ + CtxUserIP: ctx.IP(), + CtxUserPort: ctx.Port(), + KeyDomain: strings.Join(ctx.Subdomains(), "/"), + KeyPath: ctx.Path(), + CtxUserID: accountId, + CtxID: quiz.Id, + }) + } + return ctx.JSON(UpdateResp{ Updated: quiz.Id, }) @@ -392,6 +429,7 @@ func (s *Service) DeleteQuiz(ctx *fiber.Ctx) error { if !ok { return ctx.Status(fiber.StatusUnauthorized).SendString("account id is required") } + hlogger := middleware.GetLogger(ctx) if req.Id == 0 { return ctx.Status(fiber.StatusFailedDependency).SendString("id for deleting is required") @@ -402,6 +440,15 @@ func (s *Service) DeleteQuiz(ctx *fiber.Ctx) error { return ctx.Status(fiber.StatusInternalServerError).SendString(err.Error()) } + hlogger.Emit(models.InfoQuizDelete{ + CtxUserIP: ctx.IP(), + CtxUserPort: ctx.Port(), + KeyDomain: strings.Join(ctx.Subdomains(), "/"), + KeyPath: ctx.Path(), + CtxUserID: accountId, + CtxID: req.Id, + }) + return ctx.JSON(DeactivateResp{ Deactivated: deleted.Id, }) @@ -490,6 +537,7 @@ func (s *Service) TemplateCopy(ctx *fiber.Ctx) error { if !ok { return ctx.Status(fiber.StatusUnauthorized).SendString("account id is required") } + hlogger := middleware.GetLogger(ctx) var req struct { Qid string `json:"qid"` @@ -508,5 +556,17 @@ func (s *Service) TemplateCopy(ctx *fiber.Ctx) error { return ctx.Status(fiber.StatusInternalServerError).SendString(err.Error()) } + hlogger.Emit(models.InfoQuizTemplateCopy{ + CtxUserIP: ctx.IP(), + CtxUserPort: ctx.Port(), + KeyDomain: strings.Join(ctx.Subdomains(), "/"), + KeyPath: ctx.Path(), + CtxUserID: accountID, + // todo либо возвращать id копируемого квиза либо поле с qid + // для него потому что id получаем уже в запросе sql + //CtxID: req.Qid, + CtxQuizID: uint64(qizID), + }) + return ctx.Status(fiber.StatusOK).JSON(fiber.Map{"id": qizID}) } From 860a1209ac44a74736307e15c00a908eb569d7e4 Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 7 Jun 2024 17:53:45 +0300 Subject: [PATCH 03/13] delete base user log data and movw mw to common pj --- app/app.go | 3 ++- go.mod | 2 +- go.sum | 2 ++ models/hlog_events.go | 54 +++++++++-------------------------------- service/account_svc.go | 8 ++---- service/question_svc.go | 14 +++-------- service/quiz_svc.go | 48 +++++++++++------------------------- 7 files changed, 35 insertions(+), 96 deletions(-) diff --git a/app/app.go b/app/app.go index 2cf8c32..b337b6b 100644 --- a/app/app.go +++ b/app/app.go @@ -9,6 +9,7 @@ import ( "github.com/themakers/hlog" "go.uber.org/zap" "go.uber.org/zap/zapcore" + "penahub.gitlab.yandexcloud.net/backend/penahub_common/log_mw" "penahub.gitlab.yandexcloud.net/backend/penahub_common/privilege" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/dal" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/healthchecks" @@ -165,7 +166,7 @@ func New(ctx context.Context, opts interface{}, ver appInit.Version) (appInit.Co encrypt := utils.NewEncrypt(options.PubKey, options.PrivKey) app := fiber.New() app.Use(middleware.JWTAuth()) - app.Use(middleware.ContextLogger(loggerHlog)) + app.Use(log_mw.ContextLogger(loggerHlog)) app.Get("/liveness", healthchecks.Liveness) app.Get("/readiness", healthchecks.Readiness(&workerErr)) //todo parametrized readiness. should discuss ready reason diff --git a/go.mod b/go.mod index 8c919fb..8b7b782 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( go.uber.org/zap v1.27.0 google.golang.org/grpc v1.64.0 google.golang.org/protobuf v1.34.1 - penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240223054633-6cb3d5ce45b6 + penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240607142502-8257e6c4aa5a penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240602092938-b0f7572884dc penahub.gitlab.yandexcloud.net/backend/quiz/worker.git v0.0.0-20240421230341-0e086fcbb990 penahub.gitlab.yandexcloud.net/external/trashlog.git v0.1.2-0.20240523172059-9bbe8a9faa31 diff --git a/go.sum b/go.sum index 768b096..9d61cdd 100644 --- a/go.sum +++ b/go.sum @@ -285,6 +285,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240223054633-6cb3d5ce45b6 h1:oV+/HNX+JPoQ3/GUx08hio7d45WpY0AMGrFs7j70QlA= penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240223054633-6cb3d5ce45b6/go.mod h1:lTmpjry+8evVkXWbEC+WMOELcFkRD1lFMc7J09mOndM= +penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240607142502-8257e6c4aa5a h1:dGW8ErUVdwGJBq6uc5AHYn6Yt10CDNkMIpV+yrDDTLs= +penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240607142502-8257e6c4aa5a/go.mod h1:+bPxq2wfW5S1gd+83vZYmHm33AE7nEBfznWS8AM1TKE= penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240602092938-b0f7572884dc h1:EpgjB6mhOey0aBdkkNPSdMPVYKGrYl2JUPNKcq6/SZM= penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240602092938-b0f7572884dc/go.mod h1:n66zm88Dh12+idyfqh0vU5nd9BZYxM6Pv0XYnmy0398= penahub.gitlab.yandexcloud.net/backend/quiz/worker.git v0.0.0-20240421230341-0e086fcbb990 h1:jiO8GWO+3sCnDAV8/NAV8tQIUwae/I6/xiDilW7zf0o= diff --git a/models/hlog_events.go b/models/hlog_events.go index 9cb6bbd..cb60bc2 100644 --- a/models/hlog_events.go +++ b/models/hlog_events.go @@ -13,68 +13,36 @@ type AllFields struct { } type InfoAccountCreated struct { - CtxUserIP string - CtxUserPort string - KeyDomain string - KeyPath string CtxUserID string //айдишник юзера из токена CtxAccountID string // айдишник свежесозданного аккаунта } type InfoQuizCreated struct { - CtxUserIP string - CtxUserPort string - KeyDomain string - KeyPath string - CtxUserID string //айдишник юзера из токена - CtxID uint64 //айдишник создаваемого квиза + CtxUserID string //айдишник юзера из токена + CtxID uint64 //айдишник создаваемого квиза } type InfoQuizPublish struct { // если статус меняется на start - CtxUserIP string - CtxUserPort string - KeyDomain string - KeyPath string - CtxUserID string //айдишник юзера из токена - CtxID uint64 //айдишник публикуемого квиза + CtxUserID string //айдишник юзера из токена + CtxID uint64 //айдишник публикуемого квиза } type InfoQuizDelete struct { // если квиз удаляется - CtxUserIP string - CtxUserPort string - KeyDomain string - KeyPath string - CtxUserID string //айдишник юзера из токена - CtxID uint64 //айдишник удаляемого квиза + CtxUserID string //айдишник юзера из токена + CtxID uint64 //айдишник удаляемого квиза } type InfoQuizStop struct { // если статус меняется на stop - CtxUserIP string - CtxUserPort string - KeyDomain string - KeyPath string - CtxUserID string //айдишник юзера из токена - CtxID uint64 //айдишник останавливаемого квиза + CtxUserID string //айдишник юзера из токена + CtxID uint64 //айдишник останавливаемого квиза } type InfoQuizTemplateCopy struct { - CtxUserIP string - CtxUserPort string - KeyDomain string - KeyPath string - CtxUserID string //айдишник юзера из токена - CtxID uint64 //айдишник копируемого квиза - CtxQuizID uint64 // айдишник опроса после копирования + CtxUserID string //айдишник юзера из токена + CtxID uint64 //айдишник копируемого квиза + CtxQuizID uint64 // айдишник опроса после копирования } type InfoQuestionCreate struct { - CtxUserIP string - CtxUserPort string - KeyDomain string - KeyPath string CtxUserID string //айдишник юзера из токена CtxID uint64 //айдишник останавливаемого квиза CtxQuestionID uint64 // айдишник вопроса } type InfoQuestionDelete struct { - CtxUserIP string - CtxUserPort string - KeyDomain string - KeyPath string CtxUserID string //айдишник юзера из токена CtxID uint64 //айдишник останавливаемого квиза CtxQuestionID uint64 // айдишник вопроса diff --git a/service/account_svc.go b/service/account_svc.go index 5ad2fca..2883d5c 100644 --- a/service/account_svc.go +++ b/service/account_svc.go @@ -3,11 +3,11 @@ package service import ( "database/sql" "github.com/gofiber/fiber/v2" + "penahub.gitlab.yandexcloud.net/backend/penahub_common/log_mw" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/middleware" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/model" "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/brokers" "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/models" - "strings" "time" ) @@ -71,7 +71,7 @@ func (s *Service) createAccount(ctx *fiber.Ctx) error { if !ok { return ctx.Status(fiber.StatusUnauthorized).SendString("account id is required") } - hlogger := middleware.GetLogger(ctx) + hlogger := log_mw.ExtractLogger(ctx) existingAccount, err := s.dal.AccountRepo.GetAccountByID(ctx.Context(), accountID) if err != nil && err != sql.ErrNoRows { @@ -107,10 +107,6 @@ func (s *Service) createAccount(ctx *fiber.Ctx) error { } hlogger.Emit(models.InfoAccountCreated{ - CtxUserIP: ctx.IP(), - CtxUserPort: ctx.Port(), - KeyDomain: strings.Join(ctx.Subdomains(), "/"), - KeyPath: ctx.Path(), CtxUserID: accountID, CtxAccountID: createdAcc.ID, }) diff --git a/service/question_svc.go b/service/question_svc.go index da9d56c..d8e4a90 100644 --- a/service/question_svc.go +++ b/service/question_svc.go @@ -3,10 +3,10 @@ package service import ( "github.com/gofiber/fiber/v2" "github.com/lib/pq" + "penahub.gitlab.yandexcloud.net/backend/penahub_common/log_mw" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/middleware" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/model" "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/models" - "strings" "unicode/utf8" ) @@ -28,7 +28,7 @@ func (s *Service) CreateQuestion(ctx *fiber.Ctx) error { if !ok { return ctx.Status(fiber.StatusUnauthorized).SendString("account id is required") } - hlogger := middleware.GetLogger(ctx) + hlogger := log_mw.ExtractLogger(ctx) var req QuestionCreateReq if err := ctx.BodyParser(&req); err != nil { @@ -74,10 +74,6 @@ func (s *Service) CreateQuestion(ctx *fiber.Ctx) error { } hlogger.Emit(models.InfoQuestionCreate{ - CtxUserIP: ctx.IP(), - CtxUserPort: ctx.Port(), - KeyDomain: strings.Join(ctx.Subdomains(), "/"), - KeyPath: ctx.Path(), CtxUserID: accountID, CtxID: req.QuizId, CtxQuestionID: questionID, @@ -298,7 +294,7 @@ func (s *Service) DeleteQuestion(ctx *fiber.Ctx) error { if !ok { return ctx.Status(fiber.StatusUnauthorized).SendString("account id is required") } - hlogger := middleware.GetLogger(ctx) + hlogger := log_mw.ExtractLogger(ctx) var req DeactivateReq if err := ctx.BodyParser(&req); err != nil { @@ -315,10 +311,6 @@ func (s *Service) DeleteQuestion(ctx *fiber.Ctx) error { } hlogger.Emit(models.InfoQuestionDelete{ - CtxUserIP: ctx.IP(), - CtxUserPort: ctx.Port(), - KeyDomain: strings.Join(ctx.Subdomains(), "/"), - KeyPath: ctx.Path(), CtxUserID: accountID, CtxID: deleted.QuizId, CtxQuestionID: deleted.Id, diff --git a/service/quiz_svc.go b/service/quiz_svc.go index 34b8359..afef131 100644 --- a/service/quiz_svc.go +++ b/service/quiz_svc.go @@ -4,11 +4,11 @@ import ( "fmt" "github.com/gofiber/fiber/v2" "net/url" + "penahub.gitlab.yandexcloud.net/backend/penahub_common/log_mw" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/middleware" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/model" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/repository/quiz" "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/models" - "strings" "time" "unicode/utf8" ) @@ -48,7 +48,7 @@ func (s *Service) CreateQuiz(ctx *fiber.Ctx) error { return ctx.Status(fiber.StatusUnauthorized).SendString("account id is required") } - hlogger := middleware.GetLogger(ctx) + hlogger := log_mw.ExtractLogger(ctx) // check that we can store name if utf8.RuneCountInString(req.Name) > 700 { @@ -102,12 +102,8 @@ func (s *Service) CreateQuiz(ctx *fiber.Ctx) error { } hlogger.Emit(models.InfoQuizCreated{ - CtxUserIP: ctx.IP(), - CtxUserPort: ctx.Port(), - KeyDomain: strings.Join(ctx.Subdomains(), "/"), - KeyPath: ctx.Path(), - CtxUserID: accountId, - CtxID: quizID, + CtxUserID: accountId, + CtxID: quizID, }) return ctx.Status(fiber.StatusCreated).JSON(record) @@ -211,7 +207,7 @@ func (s *Service) UpdateQuiz(ctx *fiber.Ctx) error { return ctx.Status(fiber.StatusUnauthorized).SendString("account id is required") } - hlogger := middleware.GetLogger(ctx) + hlogger := log_mw.ExtractLogger(ctx) if req.Id == 0 { return ctx.Status(fiber.StatusFailedDependency).SendString("need id of question for update") @@ -319,22 +315,14 @@ func (s *Service) UpdateQuiz(ctx *fiber.Ctx) error { if req.Status == model.StatusStart { hlogger.Emit(models.InfoQuizPublish{ - CtxUserIP: ctx.IP(), - CtxUserPort: ctx.Port(), - KeyDomain: strings.Join(ctx.Subdomains(), "/"), - KeyPath: ctx.Path(), - CtxUserID: accountId, - CtxID: quiz.Id, + CtxUserID: accountId, + CtxID: quiz.Id, }) } if req.Status == model.StatusStop { hlogger.Emit(models.InfoQuizStop{ - CtxUserIP: ctx.IP(), - CtxUserPort: ctx.Port(), - KeyDomain: strings.Join(ctx.Subdomains(), "/"), - KeyPath: ctx.Path(), - CtxUserID: accountId, - CtxID: quiz.Id, + CtxUserID: accountId, + CtxID: quiz.Id, }) } @@ -429,7 +417,7 @@ func (s *Service) DeleteQuiz(ctx *fiber.Ctx) error { if !ok { return ctx.Status(fiber.StatusUnauthorized).SendString("account id is required") } - hlogger := middleware.GetLogger(ctx) + hlogger := log_mw.ExtractLogger(ctx) if req.Id == 0 { return ctx.Status(fiber.StatusFailedDependency).SendString("id for deleting is required") @@ -441,12 +429,8 @@ func (s *Service) DeleteQuiz(ctx *fiber.Ctx) error { } hlogger.Emit(models.InfoQuizDelete{ - CtxUserIP: ctx.IP(), - CtxUserPort: ctx.Port(), - KeyDomain: strings.Join(ctx.Subdomains(), "/"), - KeyPath: ctx.Path(), - CtxUserID: accountId, - CtxID: req.Id, + CtxUserID: accountId, + CtxID: req.Id, }) return ctx.JSON(DeactivateResp{ @@ -537,7 +521,7 @@ func (s *Service) TemplateCopy(ctx *fiber.Ctx) error { if !ok { return ctx.Status(fiber.StatusUnauthorized).SendString("account id is required") } - hlogger := middleware.GetLogger(ctx) + hlogger := log_mw.ExtractLogger(ctx) var req struct { Qid string `json:"qid"` @@ -557,11 +541,7 @@ func (s *Service) TemplateCopy(ctx *fiber.Ctx) error { } hlogger.Emit(models.InfoQuizTemplateCopy{ - CtxUserIP: ctx.IP(), - CtxUserPort: ctx.Port(), - KeyDomain: strings.Join(ctx.Subdomains(), "/"), - KeyPath: ctx.Path(), - CtxUserID: accountID, + CtxUserID: accountID, // todo либо возвращать id копируемого квиза либо поле с qid // для него потому что id получаем уже в запросе sql //CtxID: req.Qid, From 685420a3351fce78b344840092bd122c95f15591 Mon Sep 17 00:00:00 2001 From: skeris Date: Wed, 12 Jun 2024 16:27:42 +0300 Subject: [PATCH 04/13] ci: hlog envs --- .gitignore | 3 ++- app/app.go | 14 +++++++------- deployments/staging/docker-compose.yaml | 2 ++ go.mod | 24 +++++++++++++----------- go.sum | 21 +++++++++++++++++++++ initialize/rpc_controllers.go | 2 +- main.go | 2 +- rpc_service/mail_notify.go | 2 +- server/grpc.go | 4 ++-- service/account_svc.go | 4 ++-- service/question_svc.go | 2 +- service/quiz_svc.go | 3 +-- service/result_svc.go | 2 +- service/service.go | 4 ++-- 14 files changed, 57 insertions(+), 32 deletions(-) diff --git a/.gitignore b/.gitignore index 1ffb511..ca20f47 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,5 @@ squiz gen worker/worker storer/storer -answerer/answerer \ No newline at end of file +answerer/answerer +core diff --git a/app/app.go b/app/app.go index d4cab6a..156dfbc 100644 --- a/app/app.go +++ b/app/app.go @@ -15,13 +15,13 @@ import ( "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/healthchecks" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/middleware" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/model" - "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/brokers" - "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/clients/auth" - "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/initialize" - "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/models" - "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/server" - "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/service" - "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/tools" + "penahub.gitlab.yandexcloud.net/backend/quiz/core/brokers" + "penahub.gitlab.yandexcloud.net/backend/quiz/core/clients/auth" + "penahub.gitlab.yandexcloud.net/backend/quiz/core/initialize" + "penahub.gitlab.yandexcloud.net/backend/quiz/core/models" + "penahub.gitlab.yandexcloud.net/backend/quiz/core/server" + "penahub.gitlab.yandexcloud.net/backend/quiz/core/service" + "penahub.gitlab.yandexcloud.net/backend/quiz/core/tools" "penahub.gitlab.yandexcloud.net/external/trashlog.git/wrappers/zaptrashlog" "time" ) diff --git a/deployments/staging/docker-compose.yaml b/deployments/staging/docker-compose.yaml index 376c537..44ec80d 100644 --- a/deployments/staging/docker-compose.yaml +++ b/deployments/staging/docker-compose.yaml @@ -18,6 +18,8 @@ services: KAFKA_BROKERS: 10.8.0.6:9092 KAFKA_TOPIC: "mailnotifier" GRPC_HOST: "0.0.0.0" + TRASH_LOG_HOST: "10.8.0.15:7113" + MODULE_LOGGER: "quiz-core-staging" ports: - 10.8.0.5:1488:1488 - 10.8.0.5:9000:9000 diff --git a/go.mod b/go.mod index 55bd478..811dc89 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module penahub.gitlab.yandexcloud.net/backend/quiz/core.git +module penahub.gitlab.yandexcloud.net/backend/quiz/core go 1.21.4 @@ -16,11 +16,11 @@ require ( github.com/xuri/excelize/v2 v2.8.1 go.uber.org/zap v1.27.0 google.golang.org/grpc v1.64.0 - google.golang.org/protobuf v1.34.1 - penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240607142502-8257e6c4aa5a - penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240602092938-b0f7572884dc + google.golang.org/protobuf v1.34.2 + penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240607202348-efe5f2bf3e8c + penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240608210553-4b0f100655ae penahub.gitlab.yandexcloud.net/backend/quiz/worker.git v0.0.0-20240421230341-0e086fcbb990 - penahub.gitlab.yandexcloud.net/external/trashlog.git v0.1.2-0.20240523172059-9bbe8a9faa31 + penahub.gitlab.yandexcloud.net/external/trashlog.git v0.1.2-0.20240605170345-09de7d822830 ) require ( @@ -45,6 +45,7 @@ require ( github.com/minio/minio-go/v7 v7.0.70 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/richardlehane/mscfb v1.0.4 // indirect github.com/richardlehane/msoleps v1.0.3 // indirect @@ -56,14 +57,15 @@ require ( github.com/valyala/tcplisten v1.0.0 // indirect github.com/xuri/efp v0.0.0-20240408161823-9ad904a10d6d // indirect github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 // indirect - go.etcd.io/bbolt v1.3.6 // indirect + go.etcd.io/bbolt v1.3.10 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.23.0 // indirect - golang.org/x/net v0.25.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/tucnak/telebot.v2 v2.5.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index f321776..68d7c09 100644 --- a/go.sum +++ b/go.sum @@ -136,6 +136,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= @@ -166,6 +167,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= +go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= @@ -186,6 +189,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4= golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= @@ -206,6 +211,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -224,10 +231,14 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -251,6 +262,8 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 h1:AgADTJarZTBqgjiUzRgfaBchgYB3/WFTC80GPwsMcRI= google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3 h1:9Xyg6I9IWQZhRVfCWjKK+l6kI0jHcPesVlMnT//aHNo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= @@ -260,6 +273,8 @@ google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= @@ -286,9 +301,15 @@ penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240223054633-6cb3 penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240223054633-6cb3d5ce45b6/go.mod h1:lTmpjry+8evVkXWbEC+WMOELcFkRD1lFMc7J09mOndM= penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240607142502-8257e6c4aa5a h1:dGW8ErUVdwGJBq6uc5AHYn6Yt10CDNkMIpV+yrDDTLs= penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240607142502-8257e6c4aa5a/go.mod h1:+bPxq2wfW5S1gd+83vZYmHm33AE7nEBfznWS8AM1TKE= +penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240607202348-efe5f2bf3e8c h1:CWb4UcuNXhd1KTNOmy2U0TJO4+Qxgxrj5cwkyFqbgrk= +penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240607202348-efe5f2bf3e8c/go.mod h1:+bPxq2wfW5S1gd+83vZYmHm33AE7nEBfznWS8AM1TKE= penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240602092938-b0f7572884dc h1:EpgjB6mhOey0aBdkkNPSdMPVYKGrYl2JUPNKcq6/SZM= penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240602092938-b0f7572884dc/go.mod h1:n66zm88Dh12+idyfqh0vU5nd9BZYxM6Pv0XYnmy0398= +penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240608210553-4b0f100655ae h1:rwSxoF9nrR+1X+wSVVuCgirbyzLPDDefRPxVwshC1ss= +penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240608210553-4b0f100655ae/go.mod h1:n66zm88Dh12+idyfqh0vU5nd9BZYxM6Pv0XYnmy0398= penahub.gitlab.yandexcloud.net/backend/quiz/worker.git v0.0.0-20240421230341-0e086fcbb990 h1:jiO8GWO+3sCnDAV8/NAV8tQIUwae/I6/xiDilW7zf0o= penahub.gitlab.yandexcloud.net/backend/quiz/worker.git v0.0.0-20240421230341-0e086fcbb990/go.mod h1:zswBuTwmEsFHBVRu1nkG3/Fwylk5Vcm8OUm9iWxccSE= penahub.gitlab.yandexcloud.net/external/trashlog.git v0.1.2-0.20240523172059-9bbe8a9faa31 h1:WlRVJnzU0sti+qBq/JTCgFPU0RoxIqGHu7hzDirxE2k= penahub.gitlab.yandexcloud.net/external/trashlog.git v0.1.2-0.20240523172059-9bbe8a9faa31/go.mod h1:3ml0dAGT8U8RhpevKBfRgG6yKZum8EI2uJxAb2WCIy4= +penahub.gitlab.yandexcloud.net/external/trashlog.git v0.1.2-0.20240605170345-09de7d822830 h1:OElRKl1ig9/X4b+TmKhiZNjzIv4QMhH4/V0IfwWoCCY= +penahub.gitlab.yandexcloud.net/external/trashlog.git v0.1.2-0.20240605170345-09de7d822830/go.mod h1:3ml0dAGT8U8RhpevKBfRgG6yKZum8EI2uJxAb2WCIy4= diff --git a/initialize/rpc_controllers.go b/initialize/rpc_controllers.go index 3cb4f7b..f65b8a0 100644 --- a/initialize/rpc_controllers.go +++ b/initialize/rpc_controllers.go @@ -2,7 +2,7 @@ package initialize import ( "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/dal" - "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/rpc_service" + "penahub.gitlab.yandexcloud.net/backend/quiz/core/rpc_service" ) type RpcRegister struct { diff --git a/main.go b/main.go index 7ac320d..73c322b 100644 --- a/main.go +++ b/main.go @@ -2,7 +2,7 @@ package main import ( "github.com/skeris/appInit" - "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/app" + "penahub.gitlab.yandexcloud.net/backend/quiz/core/app" ) func main() { diff --git a/rpc_service/mail_notify.go b/rpc_service/mail_notify.go index ebdbec6..0ec5bf8 100644 --- a/rpc_service/mail_notify.go +++ b/rpc_service/mail_notify.go @@ -3,7 +3,7 @@ package rpc_service import ( "context" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/dal" - "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/proto/notifyer" + "penahub.gitlab.yandexcloud.net/backend/quiz/core/proto/notifyer" ) type MailNotify struct { diff --git a/server/grpc.go b/server/grpc.go index a77e527..c416a0c 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -9,8 +9,8 @@ import ( "go.uber.org/zap" "google.golang.org/grpc" "net" - "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/initialize" - "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/proto/notifyer" + "penahub.gitlab.yandexcloud.net/backend/quiz/core/initialize" + "penahub.gitlab.yandexcloud.net/backend/quiz/core/proto/notifyer" "time" ) diff --git a/service/account_svc.go b/service/account_svc.go index 9eb0d7e..e19faea 100644 --- a/service/account_svc.go +++ b/service/account_svc.go @@ -8,8 +8,8 @@ import ( "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/middleware" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/model" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/pj_errors" - "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/brokers" - "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/models" + "penahub.gitlab.yandexcloud.net/backend/quiz/core/brokers" + "penahub.gitlab.yandexcloud.net/backend/quiz/core/models" "time" ) diff --git a/service/question_svc.go b/service/question_svc.go index d8e4a90..ce503f1 100644 --- a/service/question_svc.go +++ b/service/question_svc.go @@ -6,7 +6,7 @@ import ( "penahub.gitlab.yandexcloud.net/backend/penahub_common/log_mw" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/middleware" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/model" - "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/models" + "penahub.gitlab.yandexcloud.net/backend/quiz/core/models" "unicode/utf8" ) diff --git a/service/quiz_svc.go b/service/quiz_svc.go index 7cb3d15..8f7366e 100644 --- a/service/quiz_svc.go +++ b/service/quiz_svc.go @@ -2,12 +2,11 @@ package service import ( "github.com/gofiber/fiber/v2" - "net/url" "penahub.gitlab.yandexcloud.net/backend/penahub_common/log_mw" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/middleware" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/model" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/repository/quiz" - "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/models" + "penahub.gitlab.yandexcloud.net/backend/quiz/core/models" "time" "unicode/utf8" ) diff --git a/service/result_svc.go b/service/result_svc.go index 16bfaa1..59f5ab8 100644 --- a/service/result_svc.go +++ b/service/result_svc.go @@ -6,7 +6,7 @@ import ( "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/middleware" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/model" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/repository/result" - "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/tools" + "penahub.gitlab.yandexcloud.net/backend/quiz/core/tools" "strconv" "time" ) diff --git a/service/service.go b/service/service.go index f5dcd27..d6644b4 100644 --- a/service/service.go +++ b/service/service.go @@ -3,8 +3,8 @@ package service import ( "github.com/gofiber/fiber/v2" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/dal" - "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/brokers" - "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/clients/auth" + "penahub.gitlab.yandexcloud.net/backend/quiz/core/brokers" + "penahub.gitlab.yandexcloud.net/backend/quiz/core/clients/auth" ) // Service is an entity for http requests handling From 014664ff9447220c0e7a2ab559f3963d23db777b Mon Sep 17 00:00:00 2001 From: skeris Date: Wed, 12 Jun 2024 16:49:52 +0300 Subject: [PATCH 05/13] fix hlog structs --- models/hlog_events.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/models/hlog_events.go b/models/hlog_events.go index cb60bc2..7a3f152 100644 --- a/models/hlog_events.go +++ b/models/hlog_events.go @@ -7,9 +7,9 @@ type AllFields struct { KeyPath string CtxUserID string CtxAccountID string - CtxID uint64 - CtxQuizID uint64 - CtxQuestionID uint64 + CtxIDInt int64 + CtxQuizID int64 + CtxQuestionID int64 } type InfoAccountCreated struct { @@ -18,32 +18,32 @@ type InfoAccountCreated struct { } type InfoQuizCreated struct { CtxUserID string //айдишник юзера из токена - CtxID uint64 //айдишник создаваемого квиза + CtxIDInt int64 //айдишник создаваемого квиза } type InfoQuizPublish struct { // если статус меняется на start CtxUserID string //айдишник юзера из токена - CtxID uint64 //айдишник публикуемого квиза + CtxIDInt int64 //айдишник публикуемого квиза } type InfoQuizDelete struct { // если квиз удаляется CtxUserID string //айдишник юзера из токена - CtxID uint64 //айдишник удаляемого квиза + CtxIDInt int64 //айдишник удаляемого квиза } type InfoQuizStop struct { // если статус меняется на stop CtxUserID string //айдишник юзера из токена - CtxID uint64 //айдишник останавливаемого квиза + CtxIDInt int64 //айдишник останавливаемого квиза } type InfoQuizTemplateCopy struct { CtxUserID string //айдишник юзера из токена - CtxID uint64 //айдишник копируемого квиза - CtxQuizID uint64 // айдишник опроса после копирования + CtxIDInt int64 //айдишник копируемого квиза + CtxQuizID int64 // айдишник опроса после копирования } type InfoQuestionCreate struct { CtxUserID string //айдишник юзера из токена - CtxID uint64 //айдишник останавливаемого квиза - CtxQuestionID uint64 // айдишник вопроса + CtxIDInt int64 //айдишник останавливаемого квиза + CtxQuestionID int64 // айдишник вопроса } type InfoQuestionDelete struct { CtxUserID string //айдишник юзера из токена - CtxID uint64 //айдишник останавливаемого квиза - CtxQuestionID uint64 // айдишник вопроса + CtxIDInt int64 //айдишник останавливаемого квиза + CtxQuestionID int64 // айдишник вопроса } From afb47bf87d5b9479ed5d00d7a42d6b355e8999ed Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 13 Jun 2024 14:09:51 +0300 Subject: [PATCH 06/13] add prepare for integrate wirh clickHouse --- app/app.go | 1 + service/service.go | 1 + service/statistic_svc.go | 15 +++++++++++++++ 3 files changed, 17 insertions(+) diff --git a/app/app.go b/app/app.go index 156dfbc..fb77db0 100644 --- a/app/app.go +++ b/app/app.go @@ -68,6 +68,7 @@ type Options struct { KafkaGroup string `env:"KAFKA_GROUP" default:"mailnotifier"` TrashLogHost string `env:"TRASH_LOG_HOST" default:"localhost:7113"` ModuleLogger string `env:"MODULE_LOGGER" default:"core-local"` + ClickHouseCred string `env:"CLICK_HOUSE_CRED"` } func New(ctx context.Context, opts interface{}, ver appInit.Version) (appInit.CommonApp, error) { diff --git a/service/service.go b/service/service.go index d6644b4..27cee03 100644 --- a/service/service.go +++ b/service/service.go @@ -73,4 +73,5 @@ func (s *Service) Register(app *fiber.App) { app.Post("/statistic/:quizID/general", s.GetGeneralStatistics) app.Post("/statistic/:quizID/questions", s.GetQuestionsStatistics) app.Post("/statistic", s.AllServiceStatistics) + app.Get("/statistics/:quizId/pipelines", s.GetPipelinesStatistics) } diff --git a/service/statistic_svc.go b/service/statistic_svc.go index 12d38e6..d944ed1 100644 --- a/service/statistic_svc.go +++ b/service/statistic_svc.go @@ -105,3 +105,18 @@ func (s *Service) AllServiceStatistics(ctx *fiber.Ctx) error { return ctx.Status(fiber.StatusOK).JSON(allSvcStats) } + +func (s *Service) GetPipelinesStatistics(ctx *fiber.Ctx) error { + var req StatisticReq + if err := ctx.BodyParser(&req); err != nil { + ctx.Status(fiber.StatusBadRequest).SendString("Invalid request data") + } + + quizIDStr := ctx.Params("quizID") + quizID, err := strconv.ParseInt(quizIDStr, 10, 64) + if err != nil { + return ctx.Status(fiber.StatusBadRequest).SendString("Invalid quiz ID format") + } + + return ctx.Status(fiber.StatusOK).JSON("allSvcStats") +} From fea41fe95ec447821aab8a7128aa75027f53f187 Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 13 Jun 2024 19:14:16 +0300 Subject: [PATCH 07/13] added rest method for getting pipeline stats, connect to clichousee repo success --- app/app.go | 12 +++++++++++- go.mod | 4 +--- go.sum | 32 ++++---------------------------- service/service.go | 3 +++ service/statistic_svc.go | 6 +++++- tests/mailNotify_test.go | 4 ++-- 6 files changed, 26 insertions(+), 35 deletions(-) diff --git a/app/app.go b/app/app.go index fb77db0..44ed3e2 100644 --- a/app/app.go +++ b/app/app.go @@ -68,7 +68,7 @@ type Options struct { KafkaGroup string `env:"KAFKA_GROUP" default:"mailnotifier"` TrashLogHost string `env:"TRASH_LOG_HOST" default:"localhost:7113"` ModuleLogger string `env:"MODULE_LOGGER" default:"core-local"` - ClickHouseCred string `env:"CLICK_HOUSE_CRED"` + ClickHouseCred string `env:"CLICK_HOUSE_CRED" default:"tcp://10.8.0.15:9000/default?sslmode=disable"` } func New(ctx context.Context, opts interface{}, ver appInit.Version) (appInit.CommonApp, error) { @@ -123,6 +123,12 @@ func New(ctx context.Context, opts interface{}, ver appInit.Version) (appInit.Co return nil, err } + chDal, err := dal.NewClickHouseDAL(ctx, options.ClickHouseCred) + if err != nil { + fmt.Println("failed init clickhouse", err) + return nil, err + } + kafkaClient, err := initialize.KafkaInit(ctx, initialize.KafkaDeps{ KafkaGroup: options.KafkaGroup, KafkaBrokers: options.KafkaBrokers, @@ -171,6 +177,7 @@ func New(ctx context.Context, opts interface{}, ver appInit.Version) (appInit.Co AuthClient: authClient, Producer: producer, ServiceName: options.ServiceName, + ChDAL: chDal, }) svc.Register(app) @@ -182,6 +189,9 @@ func New(ctx context.Context, opts interface{}, ver appInit.Version) (appInit.Co if pgdal != nil { pgdal.Close() } + if chDal != nil { + chDal.Close(ctx) + } err := grpc.Stop(ctx) err = app.Shutdown() loggerHlog.Emit(InfoSvcShutdown{Signal: err.Error()}) diff --git a/go.mod b/go.mod index 811dc89..369fad4 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( google.golang.org/grpc v1.64.0 google.golang.org/protobuf v1.34.2 penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240607202348-efe5f2bf3e8c - penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240608210553-4b0f100655ae + penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240613160144-46fac5b68652 penahub.gitlab.yandexcloud.net/backend/quiz/worker.git v0.0.0-20240421230341-0e086fcbb990 penahub.gitlab.yandexcloud.net/external/trashlog.git v0.1.2-0.20240605170345-09de7d822830 ) @@ -45,7 +45,6 @@ require ( github.com/minio/minio-go/v7 v7.0.70 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/richardlehane/mscfb v1.0.4 // indirect github.com/richardlehane/msoleps v1.0.3 // indirect @@ -66,6 +65,5 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/tucnak/telebot.v2 v2.5.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 68d7c09..421a1de 100644 --- a/go.sum +++ b/go.sum @@ -136,13 +136,10 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tealeg/xlsx v1.0.5 h1:+f8oFmvY8Gw1iUXzPk+kz+4GpbDZPK1FhPiQRd+ypgE= -github.com/tealeg/xlsx v1.0.5/go.mod h1:btRS8dz54TDnvKNosuAqxrM1QgN1udgk9O34bDCnORM= github.com/themakers/bdd v0.0.0-20210316111417-6b1dfe326f33 h1:N9f/Q+2Ssa+yDcbfaoLTYvXmdeyUUxsJKdPUVsjSmiA= github.com/themakers/bdd v0.0.0-20210316111417-6b1dfe326f33/go.mod h1:rpcH99JknBh8seZmlOlUg51gasZH6QH34oXNsIwYT6E= github.com/themakers/hlog v0.0.0-20191205140925-235e0e4baddf h1:TJJm6KcBssmbWzplF5lzixXl1RBAi/ViPs1GaSOkhwo= @@ -165,8 +162,6 @@ github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 h1:hPVCafDV85blFTabnqKgNh github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -187,8 +182,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -209,8 +202,6 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -219,24 +210,21 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -260,8 +248,6 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 h1:AgADTJarZTBqgjiUzRgfaBchgYB3/WFTC80GPwsMcRI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3 h1:9Xyg6I9IWQZhRVfCWjKK+l6kI0jHcPesVlMnT//aHNo= google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -271,8 +257,6 @@ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -297,19 +281,11 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240223054633-6cb3d5ce45b6 h1:oV+/HNX+JPoQ3/GUx08hio7d45WpY0AMGrFs7j70QlA= -penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240223054633-6cb3d5ce45b6/go.mod h1:lTmpjry+8evVkXWbEC+WMOELcFkRD1lFMc7J09mOndM= -penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240607142502-8257e6c4aa5a h1:dGW8ErUVdwGJBq6uc5AHYn6Yt10CDNkMIpV+yrDDTLs= -penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240607142502-8257e6c4aa5a/go.mod h1:+bPxq2wfW5S1gd+83vZYmHm33AE7nEBfznWS8AM1TKE= penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240607202348-efe5f2bf3e8c h1:CWb4UcuNXhd1KTNOmy2U0TJO4+Qxgxrj5cwkyFqbgrk= penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240607202348-efe5f2bf3e8c/go.mod h1:+bPxq2wfW5S1gd+83vZYmHm33AE7nEBfznWS8AM1TKE= -penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240602092938-b0f7572884dc h1:EpgjB6mhOey0aBdkkNPSdMPVYKGrYl2JUPNKcq6/SZM= -penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240602092938-b0f7572884dc/go.mod h1:n66zm88Dh12+idyfqh0vU5nd9BZYxM6Pv0XYnmy0398= -penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240608210553-4b0f100655ae h1:rwSxoF9nrR+1X+wSVVuCgirbyzLPDDefRPxVwshC1ss= -penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240608210553-4b0f100655ae/go.mod h1:n66zm88Dh12+idyfqh0vU5nd9BZYxM6Pv0XYnmy0398= +penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240613160144-46fac5b68652 h1:QM7aM0/PCK2XfrgqQIqfUCLHXlXCINrPe6AJrDMc7b0= +penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240613160144-46fac5b68652/go.mod h1:n66zm88Dh12+idyfqh0vU5nd9BZYxM6Pv0XYnmy0398= penahub.gitlab.yandexcloud.net/backend/quiz/worker.git v0.0.0-20240421230341-0e086fcbb990 h1:jiO8GWO+3sCnDAV8/NAV8tQIUwae/I6/xiDilW7zf0o= penahub.gitlab.yandexcloud.net/backend/quiz/worker.git v0.0.0-20240421230341-0e086fcbb990/go.mod h1:zswBuTwmEsFHBVRu1nkG3/Fwylk5Vcm8OUm9iWxccSE= -penahub.gitlab.yandexcloud.net/external/trashlog.git v0.1.2-0.20240523172059-9bbe8a9faa31 h1:WlRVJnzU0sti+qBq/JTCgFPU0RoxIqGHu7hzDirxE2k= -penahub.gitlab.yandexcloud.net/external/trashlog.git v0.1.2-0.20240523172059-9bbe8a9faa31/go.mod h1:3ml0dAGT8U8RhpevKBfRgG6yKZum8EI2uJxAb2WCIy4= penahub.gitlab.yandexcloud.net/external/trashlog.git v0.1.2-0.20240605170345-09de7d822830 h1:OElRKl1ig9/X4b+TmKhiZNjzIv4QMhH4/V0IfwWoCCY= penahub.gitlab.yandexcloud.net/external/trashlog.git v0.1.2-0.20240605170345-09de7d822830/go.mod h1:3ml0dAGT8U8RhpevKBfRgG6yKZum8EI2uJxAb2WCIy4= diff --git a/service/service.go b/service/service.go index 27cee03..d6ba5d7 100644 --- a/service/service.go +++ b/service/service.go @@ -13,6 +13,7 @@ type Service struct { authClient *auth.AuthClient producer *brokers.Producer serviceName string + chDAL *dal.ClickHouseDAL } type Deps struct { @@ -20,6 +21,7 @@ type Deps struct { AuthClient *auth.AuthClient Producer *brokers.Producer ServiceName string + ChDAL *dal.ClickHouseDAL } func New(deps Deps) *Service { @@ -28,6 +30,7 @@ func New(deps Deps) *Service { authClient: deps.AuthClient, producer: deps.Producer, serviceName: deps.ServiceName, + chDAL: deps.ChDAL, } } diff --git a/service/statistic_svc.go b/service/statistic_svc.go index d944ed1..f02fb3b 100644 --- a/service/statistic_svc.go +++ b/service/statistic_svc.go @@ -118,5 +118,9 @@ func (s *Service) GetPipelinesStatistics(ctx *fiber.Ctx) error { return ctx.Status(fiber.StatusBadRequest).SendString("Invalid quiz ID format") } - return ctx.Status(fiber.StatusOK).JSON("allSvcStats") + result, err := s.chDAL.StatisticClickRepo.GetPipelinesStatistics(ctx.Context(), quizID, req.From, req.To) + if err != nil { + return ctx.Status(fiber.StatusInternalServerError).SendString(err.Error()) + } + return ctx.Status(fiber.StatusOK).JSON(result) } diff --git a/tests/mailNotify_test.go b/tests/mailNotify_test.go index 23a7e6d..13483a5 100644 --- a/tests/mailNotify_test.go +++ b/tests/mailNotify_test.go @@ -5,8 +5,8 @@ import ( "github.com/pioz/faker" "go.uber.org/zap" "log" - "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/brokers" - "penahub.gitlab.yandexcloud.net/backend/quiz/core.git/initialize" + "penahub.gitlab.yandexcloud.net/backend/quiz/core/brokers" + "penahub.gitlab.yandexcloud.net/backend/quiz/core/initialize" "testing" "time" ) From e66c653289089cc4abb033cc3f4824eb0a352983 Mon Sep 17 00:00:00 2001 From: skeris Date: Sat, 15 Jun 2024 03:18:22 +0300 Subject: [PATCH 08/13] debug --- app/app.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/app/app.go b/app/app.go index 156dfbc..774ea87 100644 --- a/app/app.go +++ b/app/app.go @@ -8,7 +8,7 @@ import ( "github.com/skeris/appInit" "github.com/themakers/hlog" "go.uber.org/zap" - "go.uber.org/zap/zapcore" + //"go.uber.org/zap/zapcore" "penahub.gitlab.yandexcloud.net/backend/penahub_common/log_mw" "penahub.gitlab.yandexcloud.net/backend/penahub_common/privilege" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/dal" @@ -22,7 +22,7 @@ import ( "penahub.gitlab.yandexcloud.net/backend/quiz/core/server" "penahub.gitlab.yandexcloud.net/backend/quiz/core/service" "penahub.gitlab.yandexcloud.net/backend/quiz/core/tools" - "penahub.gitlab.yandexcloud.net/external/trashlog.git/wrappers/zaptrashlog" + //"penahub.gitlab.yandexcloud.net/external/trashlog.git/wrappers/zaptrashlog" "time" ) @@ -101,14 +101,14 @@ func New(ctx context.Context, opts interface{}, ver appInit.Version) (appInit.Co // zap.String("SvcBuildTime", ver.BuildTime), //) - clickHouseLogger, err := zaptrashlog.NewCore(ctx, zap.InfoLevel, options.TrashLogHost, ver.Release, ver.Commit, time.Now().Unix()) - if err != nil { - panic(err) - } + //clickHouseLogger, err := zaptrashlog.NewCore(ctx, zap.InfoLevel, options.TrashLogHost, ver.Release, ver.Commit, time.Now().Unix()) + //if err != nil { + // panic(err) + //} - loggerForHlog := zapLogger.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core { - return zapcore.NewTee(core, clickHouseLogger) - })) + loggerForHlog := zapLogger//.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core { + // return zapcore.NewTee(core, clickHouseLogger) + //})) loggerHlog := hlog.New(loggerForHlog).Module(options.ModuleLogger) loggerHlog.With(models.AllFields{}) From eadd404c0c08ed866b3d3aa10d95f5e35bc6a253 Mon Sep 17 00:00:00 2001 From: skeris Date: Sat, 15 Jun 2024 22:36:34 +0300 Subject: [PATCH 09/13] -- --- app/app.go | 28 ++++++++++++++-------------- go.mod | 2 +- go.sum | 2 ++ 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/app/app.go b/app/app.go index 774ea87..1924941 100644 --- a/app/app.go +++ b/app/app.go @@ -8,7 +8,7 @@ import ( "github.com/skeris/appInit" "github.com/themakers/hlog" "go.uber.org/zap" - //"go.uber.org/zap/zapcore" + "go.uber.org/zap/zapcore" "penahub.gitlab.yandexcloud.net/backend/penahub_common/log_mw" "penahub.gitlab.yandexcloud.net/backend/penahub_common/privilege" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/dal" @@ -22,7 +22,7 @@ import ( "penahub.gitlab.yandexcloud.net/backend/quiz/core/server" "penahub.gitlab.yandexcloud.net/backend/quiz/core/service" "penahub.gitlab.yandexcloud.net/backend/quiz/core/tools" - //"penahub.gitlab.yandexcloud.net/external/trashlog.git/wrappers/zaptrashlog" + "penahub.gitlab.yandexcloud.net/external/trashlog.git/wrappers/zaptrashlog" "time" ) @@ -95,20 +95,20 @@ func New(ctx context.Context, opts interface{}, ver appInit.Version) (appInit.Co } } - //zapLogger = zapLogger.With( - // zap.String("SvcCommit", ver.Commit), - // zap.String("SvcVersion", ver.Release), - // zap.String("SvcBuildTime", ver.BuildTime), - //) + zapLogger = zapLogger.With( + zap.String("SvcCommit", ver.Commit), + zap.String("SvcVersion", ver.Release), + zap.String("SvcBuildTime", ver.BuildTime), + ) - //clickHouseLogger, err := zaptrashlog.NewCore(ctx, zap.InfoLevel, options.TrashLogHost, ver.Release, ver.Commit, time.Now().Unix()) - //if err != nil { - // panic(err) - //} + clickHouseLogger, err := zaptrashlog.NewCore(ctx, zap.InfoLevel, options.TrashLogHost, ver.Release, ver.Commit, time.Now().Unix()) + if err != nil { + panic(err) + } - loggerForHlog := zapLogger//.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core { - // return zapcore.NewTee(core, clickHouseLogger) - //})) + loggerForHlog := zapLogger.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core { + return zapcore.NewTee(core, clickHouseLogger) + })) loggerHlog := hlog.New(loggerForHlog).Module(options.ModuleLogger) loggerHlog.With(models.AllFields{}) diff --git a/go.mod b/go.mod index 811dc89..a81267a 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240607202348-efe5f2bf3e8c penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240608210553-4b0f100655ae penahub.gitlab.yandexcloud.net/backend/quiz/worker.git v0.0.0-20240421230341-0e086fcbb990 - penahub.gitlab.yandexcloud.net/external/trashlog.git v0.1.2-0.20240605170345-09de7d822830 + penahub.gitlab.yandexcloud.net/external/trashlog.git v0.1.2-0.20240615192328-b2f5dffe92ae ) require ( diff --git a/go.sum b/go.sum index 68d7c09..c13f77d 100644 --- a/go.sum +++ b/go.sum @@ -313,3 +313,5 @@ penahub.gitlab.yandexcloud.net/external/trashlog.git v0.1.2-0.20240523172059-9bb penahub.gitlab.yandexcloud.net/external/trashlog.git v0.1.2-0.20240523172059-9bbe8a9faa31/go.mod h1:3ml0dAGT8U8RhpevKBfRgG6yKZum8EI2uJxAb2WCIy4= penahub.gitlab.yandexcloud.net/external/trashlog.git v0.1.2-0.20240605170345-09de7d822830 h1:OElRKl1ig9/X4b+TmKhiZNjzIv4QMhH4/V0IfwWoCCY= penahub.gitlab.yandexcloud.net/external/trashlog.git v0.1.2-0.20240605170345-09de7d822830/go.mod h1:3ml0dAGT8U8RhpevKBfRgG6yKZum8EI2uJxAb2WCIy4= +penahub.gitlab.yandexcloud.net/external/trashlog.git v0.1.2-0.20240615192328-b2f5dffe92ae h1:vlGInCsQSDA464q7OMv3EWGUviQWQdEcpLc8HIRo+rE= +penahub.gitlab.yandexcloud.net/external/trashlog.git v0.1.2-0.20240615192328-b2f5dffe92ae/go.mod h1:3ml0dAGT8U8RhpevKBfRgG6yKZum8EI2uJxAb2WCIy4= From aeed388fdd842b6e35ce7b80a0cb5e4f5783fe26 Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 20 Jun 2024 15:29:29 +0300 Subject: [PATCH 10/13] update mod --- go.mod | 3 ++- go.sum | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 369fad4..9fad14f 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( google.golang.org/grpc v1.64.0 google.golang.org/protobuf v1.34.2 penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240607202348-efe5f2bf3e8c - penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240613160144-46fac5b68652 + penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240620122425-3755aa4ac985 penahub.gitlab.yandexcloud.net/backend/quiz/worker.git v0.0.0-20240421230341-0e086fcbb990 penahub.gitlab.yandexcloud.net/external/trashlog.git v0.1.2-0.20240605170345-09de7d822830 ) @@ -50,6 +50,7 @@ require ( github.com/richardlehane/msoleps v1.0.3 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rs/xid v1.5.0 // indirect + github.com/tealeg/xlsx v1.0.5 // indirect github.com/twmb/franz-go/pkg/kmsg v1.8.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.53.0 // indirect diff --git a/go.sum b/go.sum index 421a1de..46aaee8 100644 --- a/go.sum +++ b/go.sum @@ -140,6 +140,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tealeg/xlsx v1.0.5 h1:+f8oFmvY8Gw1iUXzPk+kz+4GpbDZPK1FhPiQRd+ypgE= +github.com/tealeg/xlsx v1.0.5/go.mod h1:btRS8dz54TDnvKNosuAqxrM1QgN1udgk9O34bDCnORM= github.com/themakers/bdd v0.0.0-20210316111417-6b1dfe326f33 h1:N9f/Q+2Ssa+yDcbfaoLTYvXmdeyUUxsJKdPUVsjSmiA= github.com/themakers/bdd v0.0.0-20210316111417-6b1dfe326f33/go.mod h1:rpcH99JknBh8seZmlOlUg51gasZH6QH34oXNsIwYT6E= github.com/themakers/hlog v0.0.0-20191205140925-235e0e4baddf h1:TJJm6KcBssmbWzplF5lzixXl1RBAi/ViPs1GaSOkhwo= @@ -261,6 +263,7 @@ google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6h google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= @@ -283,8 +286,10 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240607202348-efe5f2bf3e8c h1:CWb4UcuNXhd1KTNOmy2U0TJO4+Qxgxrj5cwkyFqbgrk= penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240607202348-efe5f2bf3e8c/go.mod h1:+bPxq2wfW5S1gd+83vZYmHm33AE7nEBfznWS8AM1TKE= -penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240613160144-46fac5b68652 h1:QM7aM0/PCK2XfrgqQIqfUCLHXlXCINrPe6AJrDMc7b0= -penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240613160144-46fac5b68652/go.mod h1:n66zm88Dh12+idyfqh0vU5nd9BZYxM6Pv0XYnmy0398= +penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240620093753-d2537b8ba40e h1:o9ZXlQGrKgEEavBcw+YdO7kun8xt5xKVlvls7z49D5k= +penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240620093753-d2537b8ba40e/go.mod h1:nfZkoj8MCYaoP+xiPeUn5D0lIzinUr1qDkNfX0ng9rk= +penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240620122425-3755aa4ac985 h1:i05GzLIL+iLirwoba2QihIxqKkylSiw6OQFrSAyL4GE= +penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240620122425-3755aa4ac985/go.mod h1:nfZkoj8MCYaoP+xiPeUn5D0lIzinUr1qDkNfX0ng9rk= penahub.gitlab.yandexcloud.net/backend/quiz/worker.git v0.0.0-20240421230341-0e086fcbb990 h1:jiO8GWO+3sCnDAV8/NAV8tQIUwae/I6/xiDilW7zf0o= penahub.gitlab.yandexcloud.net/backend/quiz/worker.git v0.0.0-20240421230341-0e086fcbb990/go.mod h1:zswBuTwmEsFHBVRu1nkG3/Fwylk5Vcm8OUm9iWxccSE= penahub.gitlab.yandexcloud.net/external/trashlog.git v0.1.2-0.20240605170345-09de7d822830 h1:OElRKl1ig9/X4b+TmKhiZNjzIv4QMhH4/V0IfwWoCCY= From ba06247eb1275af1b7beb49a5d3de58afcba70d5 Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 20 Jun 2024 15:46:14 +0300 Subject: [PATCH 11/13] update openapi --- openapi.yaml | 54 +++++++++++++++++++++++++++++++++++++++++++++- service/service.go | 2 +- 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/openapi.yaml b/openapi.yaml index 4f2c543..71a6833 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -352,6 +352,31 @@ components: description: таймштамп времени, после которого выбирать статистику. если 0 или не передано - этого ограничения нет. нижняя граница времени type: integer responses: + Statistic: + description: + content: + 'application/json': + schema: + type: object + properties: + Count: + type: integer + format: int64 + description: Количество сессий прошедших через этот вопрос + QuestionID: + type: integer + format: int64 + description: id вопроса + PipeLineStatsResp: + description: + content: + 'application/json': + schema: + type: object + additionalProperties: + type: array + items: + $ref: '#/components/responses/Statistic' GetQuizResultsResp: description: список результатов, уместившийся на запрошенной странице content: @@ -1527,4 +1552,31 @@ paths: type: object properties: message: - type: string \ No newline at end of file + type: string + /statistics/:quizID/pipelines: + get: + description: получение статистики по векторам прохождения респондентами опроса с ветвлением и без, на выход отдается мапа с ключем последний вопрос и массивом точек "точек прохождения пользователем вопросов" грубо говоря массив с векторами как двигался респондент по возможным путям, в этом массиве question id и count прошедших сессий через него + parameters: + - name: quizID + in: path + required: true + description: id опроса + schema: + type: integer + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/requestBodies/GetStatisticsReq' + responses: + '200': + description: Успешное получение статистики + content: + application/json: + schema: + $ref: '#/components/responses/PipeLineStatsResp' + '400': + description: Bad Request + '500': + description: Internal Server Error diff --git a/service/service.go b/service/service.go index d6ba5d7..a99ffad 100644 --- a/service/service.go +++ b/service/service.go @@ -76,5 +76,5 @@ func (s *Service) Register(app *fiber.App) { app.Post("/statistic/:quizID/general", s.GetGeneralStatistics) app.Post("/statistic/:quizID/questions", s.GetQuestionsStatistics) app.Post("/statistic", s.AllServiceStatistics) - app.Get("/statistics/:quizId/pipelines", s.GetPipelinesStatistics) + app.Get("/statistics/:quizID/pipelines", s.GetPipelinesStatistics) } From 3c53f7365e023314647815297326b002571133cf Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 20 Jun 2024 15:57:00 +0300 Subject: [PATCH 12/13] update openapi --- openapi.yaml | 51 ++++++++++++++++++++------------------------------- 1 file changed, 20 insertions(+), 31 deletions(-) diff --git a/openapi.yaml b/openapi.yaml index 71a6833..d6fc8fb 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -352,31 +352,6 @@ components: description: таймштамп времени, после которого выбирать статистику. если 0 или не передано - этого ограничения нет. нижняя граница времени type: integer responses: - Statistic: - description: - content: - 'application/json': - schema: - type: object - properties: - Count: - type: integer - format: int64 - description: Количество сессий прошедших через этот вопрос - QuestionID: - type: integer - format: int64 - description: id вопроса - PipeLineStatsResp: - description: - content: - 'application/json': - schema: - type: object - additionalProperties: - type: array - items: - $ref: '#/components/responses/Statistic' GetQuizResultsResp: description: список результатов, уместившийся на запрошенной странице content: @@ -745,6 +720,24 @@ components: deactivated: type: integer schemas: + Statistic: + type: object + properties: + Count: + type: integer + format: int64 + description: Количество прошедших сессий + QuestionID: + type: integer + format: int64 + description: Идентификатор вопроса + + PipeLineStatsResp: + type: object + additionalProperties: + type: array + items: + $ref: '#/components/schemas/Statistic' Answer: type: object properties: @@ -1564,18 +1557,14 @@ paths: schema: type: integer requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/requestBodies/GetStatisticsReq' + $ref: '#/components/requestBodies/GetStatisticsReq' responses: '200': description: Успешное получение статистики content: application/json: schema: - $ref: '#/components/responses/PipeLineStatsResp' + $ref: '#/components/schemas/PipeLineStatsResp' '400': description: Bad Request '500': From 925a15cb5413ad1aa30170787937083268e5519e Mon Sep 17 00:00:00 2001 From: skeris Date: Fri, 21 Jun 2024 00:43:57 +0300 Subject: [PATCH 13/13] uodate common --- Dockerfile | 1 + deployments/staging/docker-compose.yaml | 1 + go.mod | 2 +- go.sum | 2 ++ 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d3a769e..37a3ca9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,4 +16,5 @@ ENV IS_PROD_LOG=false ENV IS_PROD=false ENV PORT=1488 ENV PG_CRED="host=postgres port=5432 user=squiz password=Redalert2 dbname=squiz sslmode=disable" +RUN apk add tzdata CMD ["/core"] diff --git a/deployments/staging/docker-compose.yaml b/deployments/staging/docker-compose.yaml index 44ec80d..e2898da 100644 --- a/deployments/staging/docker-compose.yaml +++ b/deployments/staging/docker-compose.yaml @@ -20,6 +20,7 @@ services: GRPC_HOST: "0.0.0.0" TRASH_LOG_HOST: "10.8.0.15:7113" MODULE_LOGGER: "quiz-core-staging" + CLICK_HOUSE_CRED: "clickhouse://10.8.0.15:9000/default?sslmode=disable" ports: - 10.8.0.5:1488:1488 - 10.8.0.5:9000:9000 diff --git a/go.mod b/go.mod index 27e29cf..6faf45e 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( google.golang.org/grpc v1.64.0 google.golang.org/protobuf v1.34.2 penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240607202348-efe5f2bf3e8c - penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240620122425-3755aa4ac985 + penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240620193727-f8a59de8ed8b penahub.gitlab.yandexcloud.net/backend/quiz/worker.git v0.0.0-20240421230341-0e086fcbb990 penahub.gitlab.yandexcloud.net/external/trashlog.git v0.1.2-0.20240615192328-b2f5dffe92ae ) diff --git a/go.sum b/go.sum index beda4f5..8f2e550 100644 --- a/go.sum +++ b/go.sum @@ -290,6 +290,8 @@ penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240620093753-d25 penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240620093753-d2537b8ba40e/go.mod h1:nfZkoj8MCYaoP+xiPeUn5D0lIzinUr1qDkNfX0ng9rk= penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240620122425-3755aa4ac985 h1:i05GzLIL+iLirwoba2QihIxqKkylSiw6OQFrSAyL4GE= penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240620122425-3755aa4ac985/go.mod h1:nfZkoj8MCYaoP+xiPeUn5D0lIzinUr1qDkNfX0ng9rk= +penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240620193727-f8a59de8ed8b h1:Ka4nopPla7x3ItcBrou15lobkBmigS6hrGaZnelUnmQ= +penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240620193727-f8a59de8ed8b/go.mod h1:nfZkoj8MCYaoP+xiPeUn5D0lIzinUr1qDkNfX0ng9rk= penahub.gitlab.yandexcloud.net/backend/quiz/worker.git v0.0.0-20240421230341-0e086fcbb990 h1:jiO8GWO+3sCnDAV8/NAV8tQIUwae/I6/xiDilW7zf0o= penahub.gitlab.yandexcloud.net/backend/quiz/worker.git v0.0.0-20240421230341-0e086fcbb990/go.mod h1:zswBuTwmEsFHBVRu1nkG3/Fwylk5Vcm8OUm9iWxccSE= penahub.gitlab.yandexcloud.net/external/trashlog.git v0.1.2-0.20240605170345-09de7d822830 h1:OElRKl1ig9/X4b+TmKhiZNjzIv4QMhH4/V0IfwWoCCY=