Merge branch 'dev' into 'leadTarget'

# Conflicts:
#   app/app.go
#   go.mod
#   go.sum
#   tools/tools.go

.gitlab-ci.yml

@@ -3,11 +3,17 @@ include:
     file: "/templates/docker/build-template.gitlab-ci.yml"
   - project: "devops/pena-continuous-integration"
     file: "/templates/docker/deploy-template.gitlab-ci.yml"
+  - project: "devops/pena-continuous-integration"
+    file: "/templates/docker/golint.gitlab-ci.yml"
 
 stages:
+  - lint
   - build
   - deploy
 
+lint:
+  extends: .golint_template
+
 build-app:
   stage: build
   extends: .build_template

Taskfile.dist.yml

@@ -0,0 +1,10 @@
+version: "3"
+
+tasks:
+  update-linter:
+    cmds:
+      - go get -u penahub.gitlab.yandexcloud.net/devops/linters/golang.git
+  lint:
+    cmds:
+      - task: update-linter
+      - cmd: golangci-lint run -v -c $(go list -f '{{"{{"}}.Dir{{"}}"}}' -m penahub.gitlab.yandexcloud.net/devops/linters/golang.git)/.golangci.yml

app/app.go

@@ -224,7 +224,9 @@ func New(ctx context.Context, opts interface{}, ver appInit.Version) (appInit.Co
 				pgdal.Close()
 			}
 			if chDal != nil {
-				chDal.Close(ctx)
+				if derr := chDal.Close(ctx); derr != nil {
+					fmt.Printf("error closing clickhouse: %v", derr)
+				}
 			}
 			err := grpc.Stop(ctx)
 			err = app.Shutdown()

benchmarks/pagination_test.go

@@ -0,0 +1,329 @@
+package benchmarks
+
+import (
+	"database/sql"
+	"log"
+	"testing"
+
+	_ "github.com/lib/pq"
+)
+
+const (
+	accountID  = "64f2cd7a7047f28fdabf6d9e"
+	connStr    = "host=localhost port=35432 user=squiz password=Redalert2 dbname=squiz sslmode=disable"
+	queryTotal = `
+		WITH user_data AS (
+			SELECT AmoID FROM accountsAmo WHERE accountsAmo.AccountID = $1 AND accountsAmo.Deleted = false
+		)
+		SELECT f.*, COUNT(*) OVER() as total_count
+		FROM fields f JOIN user_data u ON f.AccountID = u.AmoID
+		WHERE f.Deleted = false
+		ORDER BY f.ID OFFSET ($2 - 1) * $3 LIMIT $3;
+	`
+	queryCount = `
+		WITH user_data AS (
+			SELECT AmoID FROM accountsAmo WHERE accountsAmo.AccountID = $1 AND accountsAmo.Deleted = false
+		)
+		SELECT COUNT(*)
+		FROM fields f JOIN user_data u ON f.AccountID = u.AmoID
+		WHERE f.Deleted = false;
+	`
+	queryData = `
+		WITH user_data AS (
+			SELECT AmoID FROM accountsAmo WHERE accountsAmo.AccountID = $1 AND accountsAmo.Deleted = false
+		)
+		SELECT f.*
+		FROM fields f JOIN user_data u ON f.AccountID = u.AmoID
+		WHERE f.Deleted = false
+		ORDER BY f.ID OFFSET ($2 - 1) * $3 LIMIT $3;
+	`
+)
+
+type GetFieldsWithPaginationRow struct {
+	ID         int64        `db:"id" json:"id"`
+	Amoid      int32        `db:"amoid" json:"amoid"`
+	Code       string       `db:"code" json:"code"`
+	Accountid  int32        `db:"accountid" json:"accountid"`
+	Name       string       `db:"name" json:"name"`
+	Entity     interface{}  `db:"entity" json:"entity"`
+	Type       interface{}  `db:"type" json:"type"`
+	Deleted    bool         `db:"deleted" json:"deleted"`
+	Createdat  sql.NullTime `db:"createdat" json:"createdat"`
+	TotalCount int64        `db:"total_count" json:"total_count"`
+}
+
+func initDB() *sql.DB {
+	db, err := sql.Open("postgres", connStr)
+	if err != nil {
+		log.Fatal(err)
+	}
+	return db
+}
+
+// Все получаем в одном запросе не аллоцируя при этом массив
+func BenchmarkAllOne(b *testing.B) {
+	db := initDB()
+	defer db.Close()
+	for i := 0; i < b.N; i++ {
+		page := 1
+		size := 25
+		rows, err := db.Query(queryTotal, accountID, page, size)
+		if err != nil {
+			b.Fatal(err)
+		}
+		defer rows.Close()
+
+		var results []GetFieldsWithPaginationRow
+		for rows.Next() {
+			var row GetFieldsWithPaginationRow
+			if err := rows.Scan(
+				&row.ID,
+				&row.Amoid,
+				&row.Code,
+				&row.Accountid,
+				&row.Name,
+				&row.Entity,
+				&row.Type,
+				&row.Deleted,
+				&row.Createdat,
+				&row.TotalCount,
+			); err != nil {
+				b.Fatal(err)
+			}
+			results = append(results, row)
+		}
+
+		if err := rows.Err(); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+// Все получаем в одном запросе аллоцируя при этом массив
+func BenchmarkAllOnePreAllocation(b *testing.B) {
+	db := initDB()
+	defer db.Close()
+	for i := 0; i < b.N; i++ {
+		page := 1
+		size := 25
+		rows, err := db.Query(queryTotal, accountID, page, size)
+		if err != nil {
+			b.Fatal(err)
+		}
+		defer rows.Close()
+
+		results := make([]GetFieldsWithPaginationRow, size)
+		for rows.Next() {
+			var row GetFieldsWithPaginationRow
+			if err := rows.Scan(
+				&row.ID,
+				&row.Amoid,
+				&row.Code,
+				&row.Accountid,
+				&row.Name,
+				&row.Entity,
+				&row.Type,
+				&row.Deleted,
+				&row.Createdat,
+				&row.TotalCount,
+			); err != nil {
+				b.Fatal(err)
+			}
+			results = append(results, row)
+		}
+
+		if err := rows.Err(); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+// Считается сначала количество потом получаются данные длину и емкость массиву не меняем
+func BenchmarkCountThenGetData(b *testing.B) {
+	db := initDB()
+	defer db.Close()
+	for i := 0; i < b.N; i++ {
+		page := 1
+		size := 25
+
+		row := db.QueryRow(queryCount, accountID)
+		var totalCount int
+		if err := row.Scan(&totalCount); err != nil {
+			b.Fatal(err)
+		}
+		var results []GetFieldsWithPaginationRow
+		rows, err := db.Query(queryData, accountID, page, size)
+		if err != nil {
+			b.Fatal(err)
+		}
+		defer rows.Close()
+
+		for rows.Next() {
+			var row GetFieldsWithPaginationRow
+			if err := rows.Scan(
+				&row.ID,
+				&row.Amoid,
+				&row.Code,
+				&row.Accountid,
+				&row.Name,
+				&row.Entity,
+				&row.Type,
+				&row.Deleted,
+				&row.Createdat,
+			); err != nil {
+				b.Fatal(err)
+			}
+			results = append(results, row)
+		}
+
+		if err := rows.Err(); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+// Параллельное вычисление данных и общего количество при этом длина слайса = size
+func BenchmarkParallel(b *testing.B) {
+	db := initDB()
+	defer db.Close()
+
+	for i := 0; i < b.N; i++ {
+		page := 1
+		size := 25
+		results := make([]GetFieldsWithPaginationRow, size)
+		channel := make(chan error, 2)
+
+		go func() {
+			row := db.QueryRow(queryCount, accountID)
+			var totalCount int
+			channel <- row.Scan(&totalCount)
+		}()
+
+		go func() {
+			rows, err := db.Query(queryData, accountID, page, size)
+			if err != nil {
+				channel <- err
+				return
+			}
+			defer rows.Close()
+
+			index := 0
+			for rows.Next() {
+				if err := rows.Scan(
+					&results[index].ID,
+					&results[index].Amoid,
+					&results[index].Code,
+					&results[index].Accountid,
+					&results[index].Name,
+					&results[index].Entity,
+					&results[index].Type,
+					&results[index].Deleted,
+					&results[index].Createdat,
+				); err != nil {
+					channel <- err
+					return
+				}
+				index++
+			}
+			channel <- rows.Err()
+		}()
+
+		for i := 0; i < 2; i++ {
+			if err := <-channel; err != nil {
+				b.Fatal(err)
+			}
+		}
+	}
+}
+
+// Считается сначала количество потом получаются данные создаем слайс через маке указывая ему длину начальную кап = лен
+func BenchmarkWithPreAllocation(b *testing.B) {
+	db := initDB()
+	defer db.Close()
+
+	for i := 0; i < b.N; i++ {
+		page := 1
+		size := 25
+		results := make([]GetFieldsWithPaginationRow, size)
+
+		row := db.QueryRow(queryCount, accountID)
+		var totalCount int
+		if err := row.Scan(&totalCount); err != nil {
+			b.Fatal(err)
+		}
+		rows, err := db.Query(queryData, accountID, page, size)
+		if err != nil {
+			b.Fatal(err)
+		}
+		defer rows.Close()
+
+		index := 0
+		for rows.Next() {
+			if err := rows.Scan(
+				&results[index].ID,
+				&results[index].Amoid,
+				&results[index].Code,
+				&results[index].Accountid,
+				&results[index].Name,
+				&results[index].Entity,
+				&results[index].Type,
+				&results[index].Deleted,
+				&results[index].Createdat,
+			); err != nil {
+				b.Fatal(err)
+			}
+			index++
+		}
+
+		if err := rows.Err(); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkWithPreAllocationAndMonitoringTotalCount(b *testing.B) {
+	db := initDB()
+	defer db.Close()
+
+	for i := 0; i < b.N; i++ {
+		page := 1
+		size := 50
+
+		row := db.QueryRow(queryCount, accountID)
+		var totalCount int
+		if err := row.Scan(&totalCount); err != nil {
+			b.Fatal(err)
+		}
+		if totalCount < size {
+			size = totalCount
+		}
+		results := make([]GetFieldsWithPaginationRow, size)
+		rows, err := db.Query(queryData, accountID, page, size)
+		if err != nil {
+			b.Fatal(err)
+		}
+		defer rows.Close()
+
+		index := 0
+		for rows.Next() {
+			if err := rows.Scan(
+				&results[index].ID,
+				&results[index].Amoid,
+				&results[index].Code,
+				&results[index].Accountid,
+				&results[index].Name,
+				&results[index].Entity,
+				&results[index].Type,
+				&results[index].Deleted,
+				&results[index].Createdat,
+			); err != nil {
+				b.Fatal(err)
+			}
+			index++
+		}
+
+		if err := rows.Err(); err != nil {
+			b.Fatal(err)
+		}
+	}
+}

go.sum

@@ -140,8 +140,6 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/tealeg/xlsx v1.0.5 h1:+f8oFmvY8Gw1iUXzPk+kz+4GpbDZPK1FhPiQRd+ypgE=
-github.com/tealeg/xlsx v1.0.5/go.mod h1:btRS8dz54TDnvKNosuAqxrM1QgN1udgk9O34bDCnORM=
 github.com/themakers/bdd v0.0.0-20210316111417-6b1dfe326f33 h1:N9f/Q+2Ssa+yDcbfaoLTYvXmdeyUUxsJKdPUVsjSmiA=
 github.com/themakers/bdd v0.0.0-20210316111417-6b1dfe326f33/go.mod h1:rpcH99JknBh8seZmlOlUg51gasZH6QH34oXNsIwYT6E=
 github.com/themakers/hlog v0.0.0-20191205140925-235e0e4baddf h1:TJJm6KcBssmbWzplF5lzixXl1RBAi/ViPs1GaSOkhwo=
@@ -262,7 +260,6 @@ google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6h
 google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=

main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"github.com/skeris/appInit"
 	"penahub.gitlab.yandexcloud.net/backend/quiz/core/app"
+	_ "penahub.gitlab.yandexcloud.net/devops/linters/golang.git/pkg/dummy"
 )
 
 func main() {

openapi.yaml

@@ -734,10 +734,22 @@ components:
 
     PipeLineStatsResp:
       type: object
-      additionalProperties:
-        type: array
-        items:
-          $ref: '#/components/schemas/Statistic'
+      properties:
+        PipelineStatistic:
+          type: object
+          additionalProperties:
+            type: array
+            items:
+              $ref: '#/components/schemas/Statistic'
+          description: Статистика по воронкам
+
+        ContactFormStatistic:
+          type: object
+          additionalProperties:
+            type: integer
+            format: int64
+            description: Количество ответов на вопрос формы контактов
+          description: Статистика форм контакта
     Answer:
       type: object
       properties:

service/statistic_svc.go

@@ -73,7 +73,7 @@ func (s *Service) GetQuestionsStatistics(ctx *fiber.Ctx) error {
 
 	var req DeviceStatReq
 	if err := ctx.BodyParser(&req); err != nil {
-		ctx.Status(fiber.StatusBadRequest).SendString("Invalid request data")
+		return ctx.Status(fiber.StatusBadRequest).SendString("Invalid request data")
 	}
 
 	questionsStats, err := s.dal.StatisticsRepo.GetQuestionsStatistics(ctx.Context(), statistics.DeviceStatReq{
@@ -95,7 +95,7 @@ type StatisticReq struct {
 func (s *Service) AllServiceStatistics(ctx *fiber.Ctx) error {
 	var req StatisticReq
 	if err := ctx.BodyParser(&req); err != nil {
-		ctx.Status(fiber.StatusBadRequest).SendString("Invalid request data")
+		return ctx.Status(fiber.StatusBadRequest).SendString("Invalid request data")
 	}
 
 	allSvcStats, err := s.dal.StatisticsRepo.AllServiceStatistics(ctx.Context(), req.From, req.To)
@@ -109,7 +109,7 @@ func (s *Service) AllServiceStatistics(ctx *fiber.Ctx) error {
 func (s *Service) GetPipelinesStatistics(ctx *fiber.Ctx) error {
 	var req StatisticReq
 	if err := ctx.BodyParser(&req); err != nil {
-		ctx.Status(fiber.StatusBadRequest).SendString("Invalid request data")
+		return ctx.Status(fiber.StatusBadRequest).SendString("Invalid request data")
 	}
 
 	quizIDStr := ctx.Params("quizID")

tests/utils.go

@@ -111,7 +111,11 @@ func registerUser(login string) *jwt.Token {
 	if err != nil {
 		panic(err)
 	}
-	defer resp.Body.Close()
+	defer func() {
+		if derr := resp.Body.Close(); derr != nil {
+			fmt.Printf("error close response body in registerUser: %v", derr)
+		}
+	}()
 
 	bytes, err := io.ReadAll(resp.Body)
 	if err != nil {

tools/tools.go

@@ -187,6 +187,66 @@ func handleImage(file *excelize.File, sheet, cell, content string, count, row in
 				fmt.Println(err.Error())
 			}
 		}
+		count := 2
+		for _, q := range questions {
+			if !q.Deleted && q.Type != model.TypeResult {
+				index := binarySearch(response, q.Id)
+				if index != -1 {
+					cell := ToAlphaString(count) + strconv.Itoa(row)
+					tipe := FileSearch(response[index].Content)
+					noAccept := make(map[string]struct{})
+					todoMap := make(map[string]string)
+					if tipe != "Text" && q.Type == model.TypeImages || q.Type == model.TypeVarImages {
+						urle := ExtractImageURL(response[index].Content)
+						urlData := strings.Split(urle, " ")
+						if len(urlData) == 1 {
+							u, err := url.Parse(urle)
+							if err == nil && u.Scheme != "" && u.Host != "" {
+								picture, err := downloadImage(urle)
+								if err != nil {
+									fmt.Println(err.Error())
+								}
+								err = file.SetColWidth(sheet, ToAlphaString(count), ToAlphaString(count), 50)
+								if err != nil {
+									fmt.Println(err.Error())
+								}
+								err = file.SetRowHeight(sheet, row, 150)
+								if err != nil {
+									fmt.Println(err.Error())
+								}
+								if err := file.AddPictureFromBytes(sheet, cell, picture); err != nil {
+									fmt.Println(err.Error())
+								}
+								noAccept[response[index].Content] = struct{}{}
+							} else {
+								todoMap[response[index].Content] = cell
+							}
+						} else {
+							todoMap[response[index].Content] = cell
+						}
+					} else if tipe != "Text" && q.Type == model.TypeFile {
+						urle := ExtractImageURL(response[index].Content)
+						display, tooltip := urle, urle
+						if err := file.SetCellValue(sheet, cell, response[index].Content); err != nil {
+							fmt.Println(err.Error())
+						}
+						if err := file.SetCellHyperLink(sheet, cell, urle, "External", excelize.HyperlinkOpts{
+							Display: &display,
+							Tooltip: &tooltip,
+						}); err != nil {
+							fmt.Println(err.Error())
+						}
+						noAccept[response[index].Content] = struct{}{}
+					} else {
+						todoMap[response[index].Content] = cell
+					}
+					for cnt, cel := range todoMap {
+						if _, ok := noAccept[cnt]; !ok {
+							if err := file.SetCellValue(sheet, cel, cnt); err != nil {
+								fmt.Println(err.Error())
+							}
+						}
+					}
 
 		mediaRow := row
 		for i, imgContent := range multiImgArr {
@@ -367,7 +427,12 @@ func downloadImage(url string) (*excelize.Picture, error) {
 	if err != nil {
 		return nil, err
 	}
-	defer resp.Body.Close()
+
+	defer func() {
+		if derr := resp.Body.Close(); derr != nil {
+			fmt.Printf("error close response body in downloadImage: %v", derr)
+		}
+	}()
 
 	imgData, err := ioutil.ReadAll(resp.Body)
 	if err != nil {