diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 43734c2..6043251 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,11 +3,17 @@ include: file: "/templates/docker/build-template.gitlab-ci.yml" - project: "devops/pena-continuous-integration" file: "/templates/docker/deploy-template.gitlab-ci.yml" + - project: "devops/pena-continuous-integration" + file: "/templates/docker/golint.gitlab-ci.yml" stages: + - lint - build - deploy +lint: + extends: .golint_template + build-app: stage: build extends: .build_template diff --git a/Taskfile.dist.yml b/Taskfile.dist.yml new file mode 100644 index 0000000..c96d07c --- /dev/null +++ b/Taskfile.dist.yml @@ -0,0 +1,10 @@ +version: "3" + +tasks: + update-linter: + cmds: + - go get -u penahub.gitlab.yandexcloud.net/devops/linters/golang.git + lint: + cmds: + - task: update-linter + - cmd: golangci-lint run -v -c $(go list -f '{{"{{"}}.Dir{{"}}"}}' -m penahub.gitlab.yandexcloud.net/devops/linters/golang.git)/.golangci.yml diff --git a/app/app.go b/app/app.go index e8e6f11..d5c332d 100644 --- a/app/app.go +++ b/app/app.go @@ -224,7 +224,9 @@ func New(ctx context.Context, opts interface{}, ver appInit.Version) (appInit.Co pgdal.Close() } if chDal != nil { - chDal.Close(ctx) + if derr := chDal.Close(ctx); derr != nil { + fmt.Printf("error closing clickhouse: %v", derr) + } } err := grpc.Stop(ctx) err = app.Shutdown() diff --git a/benchmarks/pagination_test.go b/benchmarks/pagination_test.go new file mode 100644 index 0000000..96fb4fc --- /dev/null +++ b/benchmarks/pagination_test.go @@ -0,0 +1,329 @@ +package benchmarks + +import ( + "database/sql" + "log" + "testing" + + _ "github.com/lib/pq" +) + +const ( + accountID = "64f2cd7a7047f28fdabf6d9e" + connStr = "host=localhost port=35432 user=squiz password=Redalert2 dbname=squiz sslmode=disable" + queryTotal = ` + WITH user_data AS ( + SELECT AmoID FROM accountsAmo WHERE accountsAmo.AccountID = $1 AND accountsAmo.Deleted = false + ) + SELECT f.*, COUNT(*) OVER() as total_count + FROM fields f JOIN user_data u ON f.AccountID = u.AmoID + WHERE f.Deleted = false + ORDER BY f.ID OFFSET ($2 - 1) * $3 LIMIT $3; + ` + queryCount = ` + WITH user_data AS ( + SELECT AmoID FROM accountsAmo WHERE accountsAmo.AccountID = $1 AND accountsAmo.Deleted = false + ) + SELECT COUNT(*) + FROM fields f JOIN user_data u ON f.AccountID = u.AmoID + WHERE f.Deleted = false; + ` + queryData = ` + WITH user_data AS ( + SELECT AmoID FROM accountsAmo WHERE accountsAmo.AccountID = $1 AND accountsAmo.Deleted = false + ) + SELECT f.* + FROM fields f JOIN user_data u ON f.AccountID = u.AmoID + WHERE f.Deleted = false + ORDER BY f.ID OFFSET ($2 - 1) * $3 LIMIT $3; + ` +) + +type GetFieldsWithPaginationRow struct { + ID int64 `db:"id" json:"id"` + Amoid int32 `db:"amoid" json:"amoid"` + Code string `db:"code" json:"code"` + Accountid int32 `db:"accountid" json:"accountid"` + Name string `db:"name" json:"name"` + Entity interface{} `db:"entity" json:"entity"` + Type interface{} `db:"type" json:"type"` + Deleted bool `db:"deleted" json:"deleted"` + Createdat sql.NullTime `db:"createdat" json:"createdat"` + TotalCount int64 `db:"total_count" json:"total_count"` +} + +func initDB() *sql.DB { + db, err := sql.Open("postgres", connStr) + if err != nil { + log.Fatal(err) + } + return db +} + +// Все получаем в одном запросе не аллоцируя при этом массив +func BenchmarkAllOne(b *testing.B) { + db := initDB() + defer db.Close() + for i := 0; i < b.N; i++ { + page := 1 + size := 25 + rows, err := db.Query(queryTotal, accountID, page, size) + if err != nil { + b.Fatal(err) + } + defer rows.Close() + + var results []GetFieldsWithPaginationRow + for rows.Next() { + var row GetFieldsWithPaginationRow + if err := rows.Scan( + &row.ID, + &row.Amoid, + &row.Code, + &row.Accountid, + &row.Name, + &row.Entity, + &row.Type, + &row.Deleted, + &row.Createdat, + &row.TotalCount, + ); err != nil { + b.Fatal(err) + } + results = append(results, row) + } + + if err := rows.Err(); err != nil { + b.Fatal(err) + } + } +} + +// Все получаем в одном запросе аллоцируя при этом массив +func BenchmarkAllOnePreAllocation(b *testing.B) { + db := initDB() + defer db.Close() + for i := 0; i < b.N; i++ { + page := 1 + size := 25 + rows, err := db.Query(queryTotal, accountID, page, size) + if err != nil { + b.Fatal(err) + } + defer rows.Close() + + results := make([]GetFieldsWithPaginationRow, size) + for rows.Next() { + var row GetFieldsWithPaginationRow + if err := rows.Scan( + &row.ID, + &row.Amoid, + &row.Code, + &row.Accountid, + &row.Name, + &row.Entity, + &row.Type, + &row.Deleted, + &row.Createdat, + &row.TotalCount, + ); err != nil { + b.Fatal(err) + } + results = append(results, row) + } + + if err := rows.Err(); err != nil { + b.Fatal(err) + } + } +} + +// Считается сначала количество потом получаются данные длину и емкость массиву не меняем +func BenchmarkCountThenGetData(b *testing.B) { + db := initDB() + defer db.Close() + for i := 0; i < b.N; i++ { + page := 1 + size := 25 + + row := db.QueryRow(queryCount, accountID) + var totalCount int + if err := row.Scan(&totalCount); err != nil { + b.Fatal(err) + } + var results []GetFieldsWithPaginationRow + rows, err := db.Query(queryData, accountID, page, size) + if err != nil { + b.Fatal(err) + } + defer rows.Close() + + for rows.Next() { + var row GetFieldsWithPaginationRow + if err := rows.Scan( + &row.ID, + &row.Amoid, + &row.Code, + &row.Accountid, + &row.Name, + &row.Entity, + &row.Type, + &row.Deleted, + &row.Createdat, + ); err != nil { + b.Fatal(err) + } + results = append(results, row) + } + + if err := rows.Err(); err != nil { + b.Fatal(err) + } + } +} + +// Параллельное вычисление данных и общего количество при этом длина слайса = size +func BenchmarkParallel(b *testing.B) { + db := initDB() + defer db.Close() + + for i := 0; i < b.N; i++ { + page := 1 + size := 25 + results := make([]GetFieldsWithPaginationRow, size) + channel := make(chan error, 2) + + go func() { + row := db.QueryRow(queryCount, accountID) + var totalCount int + channel <- row.Scan(&totalCount) + }() + + go func() { + rows, err := db.Query(queryData, accountID, page, size) + if err != nil { + channel <- err + return + } + defer rows.Close() + + index := 0 + for rows.Next() { + if err := rows.Scan( + &results[index].ID, + &results[index].Amoid, + &results[index].Code, + &results[index].Accountid, + &results[index].Name, + &results[index].Entity, + &results[index].Type, + &results[index].Deleted, + &results[index].Createdat, + ); err != nil { + channel <- err + return + } + index++ + } + channel <- rows.Err() + }() + + for i := 0; i < 2; i++ { + if err := <-channel; err != nil { + b.Fatal(err) + } + } + } +} + +// Считается сначала количество потом получаются данные создаем слайс через маке указывая ему длину начальную кап = лен +func BenchmarkWithPreAllocation(b *testing.B) { + db := initDB() + defer db.Close() + + for i := 0; i < b.N; i++ { + page := 1 + size := 25 + results := make([]GetFieldsWithPaginationRow, size) + + row := db.QueryRow(queryCount, accountID) + var totalCount int + if err := row.Scan(&totalCount); err != nil { + b.Fatal(err) + } + rows, err := db.Query(queryData, accountID, page, size) + if err != nil { + b.Fatal(err) + } + defer rows.Close() + + index := 0 + for rows.Next() { + if err := rows.Scan( + &results[index].ID, + &results[index].Amoid, + &results[index].Code, + &results[index].Accountid, + &results[index].Name, + &results[index].Entity, + &results[index].Type, + &results[index].Deleted, + &results[index].Createdat, + ); err != nil { + b.Fatal(err) + } + index++ + } + + if err := rows.Err(); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkWithPreAllocationAndMonitoringTotalCount(b *testing.B) { + db := initDB() + defer db.Close() + + for i := 0; i < b.N; i++ { + page := 1 + size := 50 + + row := db.QueryRow(queryCount, accountID) + var totalCount int + if err := row.Scan(&totalCount); err != nil { + b.Fatal(err) + } + if totalCount < size { + size = totalCount + } + results := make([]GetFieldsWithPaginationRow, size) + rows, err := db.Query(queryData, accountID, page, size) + if err != nil { + b.Fatal(err) + } + defer rows.Close() + + index := 0 + for rows.Next() { + if err := rows.Scan( + &results[index].ID, + &results[index].Amoid, + &results[index].Code, + &results[index].Accountid, + &results[index].Name, + &results[index].Entity, + &results[index].Type, + &results[index].Deleted, + &results[index].Createdat, + ); err != nil { + b.Fatal(err) + } + index++ + } + + if err := rows.Err(); err != nil { + b.Fatal(err) + } + } +} diff --git a/go.sum b/go.sum index eec2bf5..3cf5969 100644 --- a/go.sum +++ b/go.sum @@ -140,8 +140,6 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tealeg/xlsx v1.0.5 h1:+f8oFmvY8Gw1iUXzPk+kz+4GpbDZPK1FhPiQRd+ypgE= -github.com/tealeg/xlsx v1.0.5/go.mod h1:btRS8dz54TDnvKNosuAqxrM1QgN1udgk9O34bDCnORM= github.com/themakers/bdd v0.0.0-20210316111417-6b1dfe326f33 h1:N9f/Q+2Ssa+yDcbfaoLTYvXmdeyUUxsJKdPUVsjSmiA= github.com/themakers/bdd v0.0.0-20210316111417-6b1dfe326f33/go.mod h1:rpcH99JknBh8seZmlOlUg51gasZH6QH34oXNsIwYT6E= github.com/themakers/hlog v0.0.0-20191205140925-235e0e4baddf h1:TJJm6KcBssmbWzplF5lzixXl1RBAi/ViPs1GaSOkhwo= @@ -262,7 +260,6 @@ google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6h google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= diff --git a/main.go b/main.go index 73c322b..4f07545 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,7 @@ package main import ( "github.com/skeris/appInit" "penahub.gitlab.yandexcloud.net/backend/quiz/core/app" + _ "penahub.gitlab.yandexcloud.net/devops/linters/golang.git/pkg/dummy" ) func main() { diff --git a/openapi.yaml b/openapi.yaml index 17349f4..13460cc 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -734,10 +734,22 @@ components: PipeLineStatsResp: type: object - additionalProperties: - type: array - items: - $ref: '#/components/schemas/Statistic' + properties: + PipelineStatistic: + type: object + additionalProperties: + type: array + items: + $ref: '#/components/schemas/Statistic' + description: Статистика по воронкам + + ContactFormStatistic: + type: object + additionalProperties: + type: integer + format: int64 + description: Количество ответов на вопрос формы контактов + description: Статистика форм контакта Answer: type: object properties: diff --git a/service/statistic_svc.go b/service/statistic_svc.go index f02fb3b..432ef2b 100644 --- a/service/statistic_svc.go +++ b/service/statistic_svc.go @@ -73,7 +73,7 @@ func (s *Service) GetQuestionsStatistics(ctx *fiber.Ctx) error { var req DeviceStatReq if err := ctx.BodyParser(&req); err != nil { - ctx.Status(fiber.StatusBadRequest).SendString("Invalid request data") + return ctx.Status(fiber.StatusBadRequest).SendString("Invalid request data") } questionsStats, err := s.dal.StatisticsRepo.GetQuestionsStatistics(ctx.Context(), statistics.DeviceStatReq{ @@ -95,7 +95,7 @@ type StatisticReq struct { func (s *Service) AllServiceStatistics(ctx *fiber.Ctx) error { var req StatisticReq if err := ctx.BodyParser(&req); err != nil { - ctx.Status(fiber.StatusBadRequest).SendString("Invalid request data") + return ctx.Status(fiber.StatusBadRequest).SendString("Invalid request data") } allSvcStats, err := s.dal.StatisticsRepo.AllServiceStatistics(ctx.Context(), req.From, req.To) @@ -109,7 +109,7 @@ func (s *Service) AllServiceStatistics(ctx *fiber.Ctx) error { func (s *Service) GetPipelinesStatistics(ctx *fiber.Ctx) error { var req StatisticReq if err := ctx.BodyParser(&req); err != nil { - ctx.Status(fiber.StatusBadRequest).SendString("Invalid request data") + return ctx.Status(fiber.StatusBadRequest).SendString("Invalid request data") } quizIDStr := ctx.Params("quizID") diff --git a/tests/utils.go b/tests/utils.go index 4d904f3..01c54a9 100644 --- a/tests/utils.go +++ b/tests/utils.go @@ -111,7 +111,11 @@ func registerUser(login string) *jwt.Token { if err != nil { panic(err) } - defer resp.Body.Close() + defer func() { + if derr := resp.Body.Close(); derr != nil { + fmt.Printf("error close response body in registerUser: %v", derr) + } + }() bytes, err := io.ReadAll(resp.Body) if err != nil { diff --git a/tools/tools.go b/tools/tools.go index 1d3ac02..c23a27c 100644 --- a/tools/tools.go +++ b/tools/tools.go @@ -187,6 +187,66 @@ func handleImage(file *excelize.File, sheet, cell, content string, count, row in fmt.Println(err.Error()) } } + count := 2 + for _, q := range questions { + if !q.Deleted && q.Type != model.TypeResult { + index := binarySearch(response, q.Id) + if index != -1 { + cell := ToAlphaString(count) + strconv.Itoa(row) + tipe := FileSearch(response[index].Content) + noAccept := make(map[string]struct{}) + todoMap := make(map[string]string) + if tipe != "Text" && q.Type == model.TypeImages || q.Type == model.TypeVarImages { + urle := ExtractImageURL(response[index].Content) + urlData := strings.Split(urle, " ") + if len(urlData) == 1 { + u, err := url.Parse(urle) + if err == nil && u.Scheme != "" && u.Host != "" { + picture, err := downloadImage(urle) + if err != nil { + fmt.Println(err.Error()) + } + err = file.SetColWidth(sheet, ToAlphaString(count), ToAlphaString(count), 50) + if err != nil { + fmt.Println(err.Error()) + } + err = file.SetRowHeight(sheet, row, 150) + if err != nil { + fmt.Println(err.Error()) + } + if err := file.AddPictureFromBytes(sheet, cell, picture); err != nil { + fmt.Println(err.Error()) + } + noAccept[response[index].Content] = struct{}{} + } else { + todoMap[response[index].Content] = cell + } + } else { + todoMap[response[index].Content] = cell + } + } else if tipe != "Text" && q.Type == model.TypeFile { + urle := ExtractImageURL(response[index].Content) + display, tooltip := urle, urle + if err := file.SetCellValue(sheet, cell, response[index].Content); err != nil { + fmt.Println(err.Error()) + } + if err := file.SetCellHyperLink(sheet, cell, urle, "External", excelize.HyperlinkOpts{ + Display: &display, + Tooltip: &tooltip, + }); err != nil { + fmt.Println(err.Error()) + } + noAccept[response[index].Content] = struct{}{} + } else { + todoMap[response[index].Content] = cell + } + for cnt, cel := range todoMap { + if _, ok := noAccept[cnt]; !ok { + if err := file.SetCellValue(sheet, cel, cnt); err != nil { + fmt.Println(err.Error()) + } + } + } mediaRow := row for i, imgContent := range multiImgArr { @@ -367,7 +427,12 @@ func downloadImage(url string) (*excelize.Picture, error) { if err != nil { return nil, err } - defer resp.Body.Close() + + defer func() { + if derr := resp.Body.Close(); derr != nil { + fmt.Printf("error close response body in downloadImage: %v", derr) + } + }() imgData, err := ioutil.ReadAll(resp.Body) if err != nil {