From 1e1d489e437a4959aed81694cd4b9b0c22ec3ec3 Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 12 Jun 2024 15:15:14 +0300 Subject: [PATCH 01/59] add migrate files for separate users table --- dal/schema/000014_init.down.sql | 20 ++++++++++++++++++++ dal/schema/000014_init.up.sql | 29 +++++++++++++++++++++++++++++ sqlc.yaml | 2 ++ 3 files changed, 51 insertions(+) create mode 100644 dal/schema/000014_init.down.sql create mode 100644 dal/schema/000014_init.up.sql diff --git a/dal/schema/000014_init.down.sql b/dal/schema/000014_init.down.sql new file mode 100644 index 0000000..5ddb16f --- /dev/null +++ b/dal/schema/000014_init.down.sql @@ -0,0 +1,20 @@ +DROP TABLE IF EXISTS usersAmo; +DROP INDEX IF EXISTS idx_unique_accountsAmo; +DROP TABLE IF EXISTS accountsAmo; + +CREATE TABLE IF NOT EXISTS users ( + ID BIGSERIAL UNIQUE NOT NULL PRIMARY KEY, + AccountID VARCHAR(30) NOT NULL DEFAULT '', -- id квизе из токена + AmoID INT NOT NULL , -- id в амо + Name VARCHAR(512) NOT NULL DEFAULT '', -- имя в амо + Email VARCHAR(50) NOT NULL DEFAULT '', -- почта в амо + Role INT NOT NULL DEFAULT 0, -- роль в амо + "Group" INT NOT NULL DEFAULT 0, -- вложенная структура так как в амо группы хранятся массивом структур + Deleted BOOLEAN NOT NULL DEFAULT FALSE, + CreatedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + Subdomain VARCHAR(50) NOT NULL DEFAULT '', + AmoUserID INT NOT NULL , -- id пользователя который подключал интеграцию + Country VARCHAR(50) NOT NULL DEFAULT '' -- страна в амо +); + +CREATE UNIQUE INDEX idx_unique_users ON users (amoID) WHERE Deleted = false; \ No newline at end of file diff --git a/dal/schema/000014_init.up.sql b/dal/schema/000014_init.up.sql new file mode 100644 index 0000000..9cce9b1 --- /dev/null +++ b/dal/schema/000014_init.up.sql @@ -0,0 +1,29 @@ +DROP TABLE IF EXISTS users; +DROP INDEX IF EXISTS idx_unique_users; + +CREATE TABLE IF NOT EXISTS accountsAmo ( + ID BIGSERIAL UNIQUE NOT NULL PRIMARY KEY, + AccountID VARCHAR(30) NOT NULL DEFAULT '', -- ID аккаунта у нас + AmoID INT NOT NULL, -- ID "компании" в амо + Name VARCHAR(512) NOT NULL DEFAULT '', + Deleted BOOLEAN NOT NULL DEFAULT FALSE, + CreatedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + Subdomain VARCHAR(50) NOT NULL DEFAULT '', -- поддомен - пример https://penadigitaltech.amocrm.ru + Country VARCHAR(50) NOT NULL DEFAULT '', + DriveURL VARCHAR(255) DEFAULT '' -- URL объктного хранилища +); + +CREATE UNIQUE INDEX idx_unique_accountsAmo ON accountsAmo (amoID) WHERE Deleted = false; + +CREATE TABLE IF NOT EXISTS usersAmo ( + ID BIGSERIAL UNIQUE NOT NULL PRIMARY KEY, + AmoID INT NOT NULL, -- ID компании в амо (внешний ключ) + AmoUserID INT NOT NULL, -- ID пользователя в амо + Name VARCHAR(512) NOT NULL DEFAULT '', + Email VARCHAR(50) NOT NULL DEFAULT '', + Role INT NOT NULL DEFAULT 0, + "Group" INT NOT NULL DEFAULT 0, + Deleted BOOLEAN NOT NULL DEFAULT FALSE, + CreatedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (AmoID) REFERENCES accountsAmo(AmoID) +); \ No newline at end of file diff --git a/sqlc.yaml b/sqlc.yaml index 9658197..d367be4 100644 --- a/sqlc.yaml +++ b/sqlc.yaml @@ -30,6 +30,8 @@ packages: - "./dal/schema/000012_init.down.sql" - "./dal/schema/000013_init.up.sql" - "./dal/schema/000013_init.down.sql" + - "./dal/schema/000014_init.up.sql" + - "./dal/schema/000014_init.down.sql" engine: "postgresql" emit_json_tags: true emit_db_tags: true From c2b5e87d2f901432430a017a735b8dfc97c5d17b Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 12 Jun 2024 16:26:49 +0300 Subject: [PATCH 02/59] update amo queries --- dal/db_query/queries.sql | 135 ++++++++++++++-------------------- dal/schema/000014_init.up.sql | 5 +- model/amo.go | 51 ++++++------- 3 files changed, 82 insertions(+), 109 deletions(-) diff --git a/dal/db_query/queries.sql b/dal/db_query/queries.sql index 47181e4..c79806f 100644 --- a/dal/db_query/queries.sql +++ b/dal/db_query/queries.sql @@ -686,8 +686,8 @@ RETURNING quiz_id; -- amo methods: -- name: CreateAmoAccount :exec -INSERT INTO users (AccountID, AmoID, Name, Email, Role, "Group", Deleted, CreatedAt, Subdomain, AmoUserID, Country,DriveURL) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12); +INSERT INTO accountsAmo (AccountID, AmoID,Name, Subdomain, Country,DriveURL) +VALUES ($1, $2, $3, $4, $5, $6); -- name: CreateWebHook :exec INSERT INTO tokens (AccountID, RefreshToken, AccessToken, AuthCode, Expiration, CreatedAt) @@ -704,57 +704,73 @@ SELECT * FROM tokens; SELECT * FROM tokens WHERE Expiration <= TO_TIMESTAMP(EXTRACT(EPOCH FROM NOW()) + (10 * 60)); -- name: WebhookDelete :exec -WITH userd AS ( - UPDATE users SET Deleted = true WHERE AmoUserID = $1 RETURNING AccountID +WITH companyDel AS ( + UPDATE accountsAmo SET Deleted = true WHERE AmoID = $1 RETURNING AccountID +), +userDel AS ( +UPDATE usersAmo SET Deleted = true WHERE AmoID = $1 ) -DELETE FROM tokens WHERE AccountID IN (SELECT AccountID FROM userd); +DELETE FROM tokens WHERE AccountID IN (SELECT AccountID FROM companyDel); -- name: SoftDeleteAccount :exec -WITH userd AS ( - SELECT AmoUserID FROM users WHERE users.AccountID = $1 -), - tokend AS ( - UPDATE users SET Deleted = true WHERE AmoUserID IN (SELECT AmoUserID FROM userd) - ) +WITH amoCompany AS ( + SELECT AmoID FROM accountsAmo WHERE accountsAmo.AccountID = $1 +),usersDel AS ( + UPDATE usersAmo SET Deleted = true WHERE AmoID = (SELECT AmoID FROM amoCompany) + ), + companyDel AS ( UPDATE accountsAmo SET Deleted = true WHERE AmoID = (SELECT AmoID FROM amoCompany) + ) DELETE FROM tokens WHERE tokens.AccountID = $1; --- name: GetCurrentAccount :one -SELECT * FROM users WHERE AccountID = $1 AND Deleted = false; +-- name: GetCurrentCompany :one +SELECT * FROM accountsAmo WHERE AccountID = $1 AND Deleted = false; --- name: CheckMainUser :exec -UPDATE users SET Name = $1, "Group" = $2, Email = $3, Role = $4 WHERE AmoID = $5; +-- name: GetAllCompanyUsers :many +SELECT * FROM usersamo WHERE amoid = $2 AND deleted = false; -- name: GetUsersWithPagination :many WITH user_data AS ( - SELECT AmoID FROM users WHERE users.AccountID = $1 AND Deleted = false + SELECT AmoID FROM accountsAmo WHERE accountsAmo.AccountID = $1 AND accountsAmo.Deleted = false ) -SELECT u.*, COUNT(*) OVER() as total_count -FROM users u -JOIN user_data a ON u.AmoUserID = a.AmoID +SELECT u.*, COUNT(*) OVER() as total_count +FROM usersAmo u + JOIN user_data a ON u.AmoID = a.AmoID WHERE u.Deleted = false ORDER BY u.ID OFFSET ($2 - 1) * $3 LIMIT $3; -- name: GetTagsWithPagination :many +WITH user_data AS ( + SELECT AmoID FROM accountsAmo WHERE accountsAmo.AccountID = $1 AND accountsAmo.Deleted = false +) SELECT t.*, COUNT(*) OVER() as total_count -FROM tags t JOIN (SELECT AmoID FROM users WHERE users.AccountID = $1 AND Deleted = false) u ON t.AccountID = u.AmoID +FROM tags t JOIN user_data u ON t.AccountID = u.AmoID WHERE t.Deleted = false ORDER BY t.ID OFFSET ($2 - 1) * $3 LIMIT $3; -- name: GetStepsWithPagination :many +WITH user_data AS ( + SELECT AmoID FROM accountsAmo WHERE accountsAmo.AccountID = $1 AND accountsAmo.Deleted = false +) SELECT s.*, COUNT(*) OVER() as total_count -FROM steps s JOIN (SELECT AmoID FROM users WHERE users.AccountID = $1 AND Deleted = false) u ON s.AccountID = u.AmoID +FROM steps s JOIN user_data u ON s.AccountID = u.AmoID WHERE s.Deleted = false AND PipelineID = $4 ORDER BY s.ID OFFSET ($2 - 1) * $3 LIMIT $3; -- name: GetPipelinesWithPagination :many +WITH user_data AS ( + SELECT AmoID FROM accountsAmo WHERE accountsAmo.AccountID = $1 AND accountsAmo.Deleted = false +) SELECT p.*, COUNT(*) OVER() as total_count -FROM pipelines p JOIN (SELECT AmoID FROM users WHERE users.AccountID = $1 AND Deleted = false) u ON p.AccountID = u.AmoID +FROM pipelines p JOIN user_data u ON p.AccountID = u.AmoID WHERE p.Deleted = false ORDER BY p.ID OFFSET ($2 - 1) * $3 LIMIT $3; -- name: GetFieldsWithPagination :many +WITH user_data AS ( + SELECT AmoID FROM accountsAmo WHERE accountsAmo.AccountID = $1 AND accountsAmo.Deleted = false +) SELECT f.*, COUNT(*) OVER() as total_count -FROM fields f JOIN (SELECT AmoID FROM users WHERE users.AccountID = $1 AND Deleted = false) u ON f.AccountID = u.AmoID +FROM fields f JOIN user_data u ON f.AccountID = u.AmoID WHERE f.Deleted = false ORDER BY f.ID OFFSET ($2 - 1) * $3 LIMIT $3; @@ -796,9 +812,7 @@ WHERE f.amoID = (update_data ->> 'AmoID')::INT -- name: CheckTags :many WITH user_data AS ( - SELECT AmoID - FROM users - WHERE users.AccountID = $1 + SELECT AmoID FROM accountsAmo WHERE accountsAmo.AccountID = $1 AND accountsAmo.Deleted = false ), new_tags AS ( SELECT (tag->>'AmoID')::INT AS amoID, (tag->>'Entity')::entitytype AS Entity, @@ -857,9 +871,7 @@ WHERE NOT EXISTS ( -- name: CheckFields :many WITH user_data AS ( - SELECT AmoID - FROM users - WHERE users.AccountID = $1 + SELECT AmoID FROM accountsAmo WHERE accountsAmo.AccountID = $1 AND accountsAmo.Deleted = false ), new_fields AS ( SELECT (field->>'AmoID')::INT AS amoID, COALESCE(field->>'Code', '')::varchar(255) AS code, @@ -930,13 +942,13 @@ SELECT * FROM rules WHERE QuizID = $1 AND Deleted = false; -- name: SetQuizSettings :one INSERT INTO rules (AccountID, QuizID, PerformerID, PipelineID, StepID, FieldsRule,TagsToAdd) SELECT u.AmoID AS AccountID,$1 AS QuizID,$2 AS PerformerID,$3 AS PipelineID, - $4 AS StepID,$5 AS FieldsRule,$6 AS TagsToAdd FROM users u WHERE u.AccountID = $7 AND u.Deleted = false + $4 AS StepID,$5 AS FieldsRule,$6 AS TagsToAdd FROM accountsamo u WHERE u.AccountID = $7 AND u.Deleted = false RETURNING id; -- name: ChangeQuizSettings :one UPDATE rules SET PerformerID = $1,PipelineID = $2,StepID = $3,FieldsRule = $4, TagsToAdd=$5 -WHERE AccountID = (SELECT AmoID FROM users WHERE users.AccountID = $6 AND users.Deleted = false) AND QuizID = $7 AND Deleted = false +WHERE AccountID = (SELECT AmoID FROM accountsAmo WHERE accountsAmo.AccountID = $6 AND accountsAmo.Deleted = false) AND QuizID = $7 AND Deleted = false RETURNING id; -- name: GetQuestionListByIDs :many @@ -944,48 +956,15 @@ SELECT * FROM question WHERE id = ANY($1::int[]) AND deleted = FALSE; -- name: UpdateFieldRules :exec UPDATE rules SET FieldsRule = $1 -WHERE AccountID = (SELECT AmoID FROM users WHERE users.AccountID = $2 AND users.Deleted = false) AND QuizID = $3 AND Deleted = false; +WHERE AccountID = (SELECT AmoID FROM accountsAmo WHERE accountsAmo.AccountID = $2 AND accountsAmo.Deleted = false) AND QuizID = $3 AND Deleted = false; --- name: UpdateUsers :exec -UPDATE users AS u -SET Name = (update_data ->> 'Name')::varchar(512), - Email = (update_data ->> 'Email')::varchar(50), - Role = (update_data ->> 'Role')::INT, - "Group" = (update_data ->> 'Group')::INT, - AmoUserID= (update_data ->> 'AmoUserID')::INT -FROM json_array_elements($1::json) AS update_data -WHERE u.AmoID = (update_data ->> 'AmocrmID')::INT; +-- name: UpdateAmoAccountUser :exec +UPDATE usersAmo SET Name = $3, Email = $4, Role = $5, "Group" = $6 +WHERE AmoID = $1 AND AmoUserID = $2 AND deleted = false; --- name: CheckUsers :many -WITH new_users AS ( - SELECT (u->>'AmocrmID')::INT AS AmoID, - (u->>'Name')::VARCHAR(512) AS Name, - (u->>'Group')::INT AS "Group", - (u->>'Role')::INT AS Role, - (u->>'Email')::VARCHAR(50) AS Email, - (u->>'AmoUserID')::INT AS AmoUserID, - CURRENT_TIMESTAMP AS createdAt - FROM json_array_elements($1::json) AS u -), inserted_users AS ( - INSERT INTO users (AmoID, Name, "Group", Role, Email, AmoUserID,createdAt) - SELECT nu.AmoID, - nu.Name, - nu."Group", - nu.Role, - nu.Email, - nu.AmoUserID, - nu.createdAt - FROM new_users nu - ON CONFLICT (amoID) DO NOTHING - RETURNING * -) -SELECT nu.* -FROM new_users nu -WHERE NOT EXISTS ( - SELECT * - FROM inserted_users ins - WHERE ins.amoID = nu.amoID -); +-- name: AddAmoAccountUser :exec +INSERT INTO usersAmo (AmoID, AmoUserID, Name, Email, Role, "Group", Deleted, CreatedAt) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8); -- name: GettingAmoUsersTrueResults :many SELECT a.quiz_id,a.id,a.result,a.question_id,a.content,a.session, @@ -993,13 +972,13 @@ SELECT a.quiz_id,a.id,a.result,a.question_id,a.content,a.session, FROM answer a2 WHERE a2.start = true AND a2.session = a.session LIMIT 1) AS utm -,t.accesstoken,r.accountid,r.fieldsrule,r.tagstoadd,r.performerid,r.stepid,r.pipelineid,(SELECT u.name FROM users u WHERE u.amoid = r.performerid) AS performer_name,u.subdomain,u.accountid,u.driveurl +,t.accesstoken,r.accountid,r.fieldsrule,r.tagstoadd,r.performerid,r.stepid,r.pipelineid,(SELECT u.name FROM usersAmo u WHERE u.AmoUserID = r.performerid) AS performer_name,u.subdomain,u.accountid,u.driveurl FROM answer a INNER JOIN quiz q ON a.quiz_id = q.id LEFT JOIN amoCRMStatuses s ON a.id = s.AnswerID INNER JOIN rules r ON q.id = r.QuizID INNER JOIN tokens t ON q.accountid = t.AccountID - INNER JOIN users u ON q.accountid = u.accountid AND u.amoid = r.accountid + INNER JOIN accountsAmo u ON q.accountid = u.accountid AND u.amoid = r.accountid WHERE a.result = true AND s.id IS NULL AND a.deleted = false @@ -1011,12 +990,12 @@ WHERE a.result = true INSERT INTO amoCRMStatuses (AccountID, DealID, AnswerID, Status) SELECT u.AmoID, $1, $2, $3 FROM tokens AS t - JOIN users AS u ON t.AccountID = u.AccountID + JOIN accountsAmo AS u ON t.AccountID = u.AccountID WHERE t.AccessToken = $4 AND u.Deleted = false; -- name: UpdatingDealAmoStatus :exec UPDATE amoCRMStatuses SET Status = $1 -WHERE DealID = $2 AND AccountID = (SELECT u.AmoID FROM tokens AS t JOIN users AS u ON t.AccountID = u.AccountID WHERE t.AccessToken = $3 AND u.Deleted = false); +WHERE DealID = $2 AND AccountID = (SELECT u.AmoID FROM tokens AS t JOIN accountsAmo AS u ON t.AccountID = u.AccountID WHERE t.AccessToken = $3 AND u.Deleted = false); -- name: DeleteFields :exec UPDATE fields SET Deleted = true WHERE ID = ANY($1::bigint[]); @@ -1031,7 +1010,7 @@ UPDATE steps SET Deleted = true WHERE ID = ANY($1::bigint[]); UPDATE pipelines SET Deleted = true WHERE ID = ANY($1::bigint[]); -- name: DeleteUsers :exec -UPDATE users SET Deleted = true WHERE ID = ANY($1::bigint[]); +UPDATE usersAmo SET Deleted = true WHERE ID = ANY($1::bigint[]); -- name: GetUserTagsByID :many SELECT ID,AmoID,AccountID,Name,Entity,Color @@ -1049,9 +1028,7 @@ FROM pipelines WHERE AccountID = $1 AND Deleted = false; -- name: GetUserUsersByID :many -SELECT ID,AccountID,AmoID,Name,Email,Role,"Group",Subdomain,AmoUserID,Country -FROM users -WHERE AmoUserID = $1 AND Deleted = false; +SELECT * FROM usersAmo WHERE amoid = $1 AND Deleted = false; -- name: GetFieldByAmoID :one SELECT * FROM fields WHERE AmoID = $1 AND Deleted = false; diff --git a/dal/schema/000014_init.up.sql b/dal/schema/000014_init.up.sql index 9cce9b1..b0ba982 100644 --- a/dal/schema/000014_init.up.sql +++ b/dal/schema/000014_init.up.sql @@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS accountsAmo ( AmoID INT NOT NULL, -- ID "компании" в амо Name VARCHAR(512) NOT NULL DEFAULT '', Deleted BOOLEAN NOT NULL DEFAULT FALSE, - CreatedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + CreatedAt TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, Subdomain VARCHAR(50) NOT NULL DEFAULT '', -- поддомен - пример https://penadigitaltech.amocrm.ru Country VARCHAR(50) NOT NULL DEFAULT '', DriveURL VARCHAR(255) DEFAULT '' -- URL объктного хранилища @@ -24,6 +24,5 @@ CREATE TABLE IF NOT EXISTS usersAmo ( Role INT NOT NULL DEFAULT 0, "Group" INT NOT NULL DEFAULT 0, Deleted BOOLEAN NOT NULL DEFAULT FALSE, - CreatedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - FOREIGN KEY (AmoID) REFERENCES accountsAmo(AmoID) + CreatedAt TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ); \ No newline at end of file diff --git a/model/amo.go b/model/amo.go index b5bf0ea..2c9287e 100644 --- a/model/amo.go +++ b/model/amo.go @@ -1,32 +1,29 @@ package model -type User struct { - /* - айдишник в нашей системе Primary Key*/ - ID int64 `json:"ID"` - /* - id пользователя из токена в нашей системе*/ - Accountid string `json:"AccountID"` - /* - айдишник пользователя в амо*/ - AmoID int32 `json:"AmocrmID"` - /* - имя аккаунта в амо*/ - Name string `json:"Name"` - /* - почта пользователя из амо*/ - Email string `json:"Email"` - /* - роль пользователя в амо*/ - Role int32 `json:"Role"` - /* - группы пользователя в амо*/ - Group int32 `json:"Group"` - /* - флаг мягкого удаления*/ - Deleted bool `json:"Deleted"` - /* - таймштамп создания аккаунта*/ - Createdat int64 `json:"CreatedAt"` - /* - поддомен организации в амо*/ - Subdomain string `json:"Subdomain"` - /* - айдишник пользвателя, который подключал интеграцию*/ - Amouserid int32 `json:"AmoUserID"` - /* - страна указанная в настройках амо*/ - Country string `json:"Country"` - // урл объектного хранилища пользователя в амо - DriveURL string `json:"DriveURL"` +import "time" + +type AmoAccountUser struct { + ID int64 `json:"id"` // ID пользователя + AmoID int32 `json:"amoID"` // ID компании в амо, к которой пользователь принадлежит + AmoUserID int32 `json:"amoUserID"` // ID пользователя в амо + Name string `json:"name"` // Имя + Email string `json:"email"` // Email + Role int32 `json:"role"` // Роль + Group int32 `json:"group"` // Группа + Deleted bool `json:"deleted"` + CreatedAt time.Time `json:"createdAt"` +} + +type AmoAccount struct { + ID int64 `json:"id"` // ID компании + AccountID string `json:"accountID"` // ID аккаунта нас + AmoID int32 `json:"amoID"` // ID компании в амо + Name string `json:"name"` // Название + Deleted bool `json:"deleted"` + CreatedAt time.Time `json:"createdAt"` + Subdomain string `json:"subdomain"` // поддомен + Country string `json:"country"` // Страна + DriveURL string `json:"driveURL"` // URL объктного хранилища } type UserGroups struct { From 92b755fde54a88c0a0c2344c57d54bd13d5431a6 Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 12 Jun 2024 16:30:10 +0300 Subject: [PATCH 03/59] update sqlc gen --- dal/db_query/queries.sql | 6 +- dal/sqlcgen/models.go | 36 ++-- dal/sqlcgen/queries.sql.go | 389 ++++++++++++++++--------------------- 3 files changed, 196 insertions(+), 235 deletions(-) diff --git a/dal/db_query/queries.sql b/dal/db_query/queries.sql index c79806f..d172f23 100644 --- a/dal/db_query/queries.sql +++ b/dal/db_query/queries.sql @@ -705,7 +705,7 @@ SELECT * FROM tokens WHERE Expiration <= TO_TIMESTAMP(EXTRACT(EPOCH FROM NOW()) -- name: WebhookDelete :exec WITH companyDel AS ( - UPDATE accountsAmo SET Deleted = true WHERE AmoID = $1 RETURNING AccountID + UPDATE accountsAmo SET Deleted = true WHERE accountsAmo.AmoID = $1 RETURNING AccountID ), userDel AS ( UPDATE usersAmo SET Deleted = true WHERE AmoID = $1 @@ -726,7 +726,7 @@ DELETE FROM tokens WHERE tokens.AccountID = $1; SELECT * FROM accountsAmo WHERE AccountID = $1 AND Deleted = false; -- name: GetAllCompanyUsers :many -SELECT * FROM usersamo WHERE amoid = $2 AND deleted = false; +SELECT * FROM usersamo WHERE amoid = $1 AND deleted = false; -- name: GetUsersWithPagination :many WITH user_data AS ( @@ -1041,4 +1041,4 @@ WHERE AccountID = $1 AND Deleted = false; -- name: DecrementManual :one UPDATE privileges p SET amount = amount - 1 FROM account a WHERE p.account_id = a.id AND a.user_id = $1 AND p.privilegeID = $2 AND p.amount > 0 -RETURNING p.id, p.privilegeID, p.account_id, p.privilege_name, p.amount, p.created_at;; +RETURNING p.id, p.privilegeID, p.account_id, p.privilege_name, p.amount, p.created_at; diff --git a/dal/sqlcgen/models.go b/dal/sqlcgen/models.go index d6e25c0..fa9e952 100644 --- a/dal/sqlcgen/models.go +++ b/dal/sqlcgen/models.go @@ -20,6 +20,18 @@ type Account struct { Deleted sql.NullBool `db:"deleted" json:"deleted"` } +type Accountsamo struct { + ID int64 `db:"id" json:"id"` + Accountid string `db:"accountid" json:"accountid"` + Amoid int32 `db:"amoid" json:"amoid"` + Name string `db:"name" json:"name"` + Deleted bool `db:"deleted" json:"deleted"` + Createdat time.Time `db:"createdat" json:"createdat"` + Subdomain string `db:"subdomain" json:"subdomain"` + Country string `db:"country" json:"country"` + Driveurl sql.NullString `db:"driveurl" json:"driveurl"` +} + type Amocrmstatus struct { ID int64 `db:"id" json:"id"` Accountid int32 `db:"accountid" json:"accountid"` @@ -173,18 +185,14 @@ type Token struct { Createdat sql.NullTime `db:"createdat" json:"createdat"` } -type User struct { - ID int64 `db:"id" json:"id"` - Accountid string `db:"accountid" json:"accountid"` - Amoid int32 `db:"amoid" json:"amoid"` - Name string `db:"name" json:"name"` - Email string `db:"email" json:"email"` - Role int32 `db:"role" json:"role"` - Group int32 `db:"Group" json:"Group"` - Deleted bool `db:"deleted" json:"deleted"` - Createdat sql.NullTime `db:"createdat" json:"createdat"` - Subdomain string `db:"subdomain" json:"subdomain"` - Amouserid int32 `db:"amouserid" json:"amouserid"` - Country string `db:"country" json:"country"` - Driveurl string `db:"driveurl" json:"driveurl"` +type Usersamo struct { + ID int64 `db:"id" json:"id"` + Amoid int32 `db:"amoid" json:"amoid"` + Amouserid int32 `db:"amouserid" json:"amouserid"` + Name string `db:"name" json:"name"` + Email string `db:"email" json:"email"` + Role int32 `db:"role" json:"role"` + Group int32 `db:"Group" json:"Group"` + Deleted bool `db:"deleted" json:"deleted"` + Createdat time.Time `db:"createdat" json:"createdat"` } diff --git a/dal/sqlcgen/queries.sql.go b/dal/sqlcgen/queries.sql.go index 1d2ff0b..7f8b2bf 100644 --- a/dal/sqlcgen/queries.sql.go +++ b/dal/sqlcgen/queries.sql.go @@ -60,6 +60,36 @@ func (q *Queries) AccountPagination(ctx context.Context, arg AccountPaginationPa return items, nil } +const addAmoAccountUser = `-- name: AddAmoAccountUser :exec +INSERT INTO usersAmo (AmoID, AmoUserID, Name, Email, Role, "Group", Deleted, CreatedAt) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8) +` + +type AddAmoAccountUserParams struct { + Amoid int32 `db:"amoid" json:"amoid"` + Amouserid int32 `db:"amouserid" json:"amouserid"` + Name string `db:"name" json:"name"` + Email string `db:"email" json:"email"` + Role int32 `db:"role" json:"role"` + Group int32 `db:"Group" json:"Group"` + Deleted bool `db:"deleted" json:"deleted"` + Createdat time.Time `db:"createdat" json:"createdat"` +} + +func (q *Queries) AddAmoAccountUser(ctx context.Context, arg AddAmoAccountUserParams) error { + _, err := q.db.ExecContext(ctx, addAmoAccountUser, + arg.Amoid, + arg.Amouserid, + arg.Name, + arg.Email, + arg.Role, + arg.Group, + arg.Deleted, + arg.Createdat, + ) + return err +} + const allServiceStatistics = `-- name: AllServiceStatistics :one WITH Registrations AS ( SELECT COUNT(*) AS registration_count @@ -117,7 +147,7 @@ func (q *Queries) ArchiveQuiz(ctx context.Context, arg ArchiveQuizParams) error const changeQuizSettings = `-- name: ChangeQuizSettings :one UPDATE rules SET PerformerID = $1,PipelineID = $2,StepID = $3,FieldsRule = $4, TagsToAdd=$5 -WHERE AccountID = (SELECT AmoID FROM users WHERE users.AccountID = $6 AND users.Deleted = false) AND QuizID = $7 AND Deleted = false +WHERE AccountID = (SELECT AmoID FROM accountsAmo WHERE accountsAmo.AccountID = $6 AND accountsAmo.Deleted = false) AND QuizID = $7 AND Deleted = false RETURNING id ` @@ -200,9 +230,7 @@ func (q *Queries) CheckExpired(ctx context.Context) ([]Token, error) { const checkFields = `-- name: CheckFields :many WITH user_data AS ( - SELECT AmoID - FROM users - WHERE users.AccountID = $1 + SELECT AmoID FROM accountsAmo WHERE accountsAmo.AccountID = $1 AND accountsAmo.Deleted = false ), new_fields AS ( SELECT (field->>'AmoID')::INT AS amoID, COALESCE(field->>'Code', '')::varchar(255) AS code, @@ -282,29 +310,6 @@ func (q *Queries) CheckFields(ctx context.Context, arg CheckFieldsParams) ([]Che return items, nil } -const checkMainUser = `-- name: CheckMainUser :exec -UPDATE users SET Name = $1, "Group" = $2, Email = $3, Role = $4 WHERE AmoID = $5 -` - -type CheckMainUserParams struct { - Name string `db:"name" json:"name"` - Group int32 `db:"Group" json:"Group"` - Email string `db:"email" json:"email"` - Role int32 `db:"role" json:"role"` - Amoid int32 `db:"amoid" json:"amoid"` -} - -func (q *Queries) CheckMainUser(ctx context.Context, arg CheckMainUserParams) error { - _, err := q.db.ExecContext(ctx, checkMainUser, - arg.Name, - arg.Group, - arg.Email, - arg.Role, - arg.Amoid, - ) - return err -} - const checkPipelines = `-- name: CheckPipelines :many WITH new_pipelines AS ( SELECT (pipeline->>'AmoID')::INT AS amoID, @@ -487,9 +492,7 @@ func (q *Queries) CheckSteps(ctx context.Context, dollar_1 json.RawMessage) ([]C const checkTags = `-- name: CheckTags :many WITH user_data AS ( - SELECT AmoID - FROM users - WHERE users.AccountID = $1 + SELECT AmoID FROM accountsAmo WHERE accountsAmo.AccountID = $1 AND accountsAmo.Deleted = false ), new_tags AS ( SELECT (tag->>'AmoID')::INT AS amoID, (tag->>'Entity')::entitytype AS Entity, @@ -562,79 +565,6 @@ func (q *Queries) CheckTags(ctx context.Context, arg CheckTagsParams) ([]CheckTa return items, nil } -const checkUsers = `-- name: CheckUsers :many -WITH new_users AS ( - SELECT (u->>'AmocrmID')::INT AS AmoID, - (u->>'Name')::VARCHAR(512) AS Name, - (u->>'Group')::INT AS "Group", - (u->>'Role')::INT AS Role, - (u->>'Email')::VARCHAR(50) AS Email, - (u->>'AmoUserID')::INT AS AmoUserID, - CURRENT_TIMESTAMP AS createdAt - FROM json_array_elements($1::json) AS u -), inserted_users AS ( - INSERT INTO users (AmoID, Name, "Group", Role, Email, AmoUserID,createdAt) - SELECT nu.AmoID, - nu.Name, - nu."Group", - nu.Role, - nu.Email, - nu.AmoUserID, - nu.createdAt - FROM new_users nu - ON CONFLICT (amoID) DO NOTHING - RETURNING id, accountid, amoid, name, email, role, "Group", deleted, createdat, subdomain, amouserid, country, driveurl -) -SELECT nu.amoid, nu.name, nu."Group", nu.role, nu.email, nu.amouserid, nu.createdat -FROM new_users nu -WHERE NOT EXISTS ( - SELECT id, accountid, amoid, name, email, role, "Group", deleted, createdat, subdomain, amouserid, country, driveurl - FROM inserted_users ins - WHERE ins.amoID = nu.amoID -) -` - -type CheckUsersRow struct { - Amoid int32 `db:"amoid" json:"amoid"` - Name string `db:"name" json:"name"` - Group int32 `db:"Group" json:"Group"` - Role int32 `db:"role" json:"role"` - Email string `db:"email" json:"email"` - Amouserid int32 `db:"amouserid" json:"amouserid"` - Createdat interface{} `db:"createdat" json:"createdat"` -} - -func (q *Queries) CheckUsers(ctx context.Context, dollar_1 json.RawMessage) ([]CheckUsersRow, error) { - rows, err := q.db.QueryContext(ctx, checkUsers, dollar_1) - if err != nil { - return nil, err - } - defer rows.Close() - var items []CheckUsersRow - for rows.Next() { - var i CheckUsersRow - if err := rows.Scan( - &i.Amoid, - &i.Name, - &i.Group, - &i.Role, - &i.Email, - &i.Amouserid, - &i.Createdat, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Close(); err != nil { - return nil, err - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - const copyQuestion = `-- name: CopyQuestion :one INSERT INTO question( quiz_id, title, description, questiontype, required, @@ -780,23 +710,17 @@ func (q *Queries) CreateAccount(ctx context.Context, arg CreateAccountParams) (A const createAmoAccount = `-- name: CreateAmoAccount :exec -INSERT INTO users (AccountID, AmoID, Name, Email, Role, "Group", Deleted, CreatedAt, Subdomain, AmoUserID, Country,DriveURL) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) +INSERT INTO accountsAmo (AccountID, AmoID,Name, Subdomain, Country,DriveURL) +VALUES ($1, $2, $3, $4, $5, $6) ` type CreateAmoAccountParams struct { - Accountid string `db:"accountid" json:"accountid"` - Amoid int32 `db:"amoid" json:"amoid"` - Name string `db:"name" json:"name"` - Email string `db:"email" json:"email"` - Role int32 `db:"role" json:"role"` - Group int32 `db:"Group" json:"Group"` - Deleted bool `db:"deleted" json:"deleted"` - Createdat sql.NullTime `db:"createdat" json:"createdat"` - Subdomain string `db:"subdomain" json:"subdomain"` - Amouserid int32 `db:"amouserid" json:"amouserid"` - Country string `db:"country" json:"country"` - Driveurl string `db:"driveurl" json:"driveurl"` + Accountid string `db:"accountid" json:"accountid"` + Amoid int32 `db:"amoid" json:"amoid"` + Name string `db:"name" json:"name"` + Subdomain string `db:"subdomain" json:"subdomain"` + Country string `db:"country" json:"country"` + Driveurl sql.NullString `db:"driveurl" json:"driveurl"` } // amo methods: @@ -805,13 +729,7 @@ func (q *Queries) CreateAmoAccount(ctx context.Context, arg CreateAmoAccountPara arg.Accountid, arg.Amoid, arg.Name, - arg.Email, - arg.Role, - arg.Group, - arg.Deleted, - arg.Createdat, arg.Subdomain, - arg.Amouserid, arg.Country, arg.Driveurl, ) @@ -1004,7 +922,7 @@ func (q *Queries) DeleteTags(ctx context.Context, dollar_1 []int64) error { } const deleteUsers = `-- name: DeleteUsers :exec -UPDATE users SET Deleted = true WHERE ID = ANY($1::bigint[]) +UPDATE usersAmo SET Deleted = true WHERE ID = ANY($1::bigint[]) ` func (q *Queries) DeleteUsers(ctx context.Context, dollar_1 []int64) error { @@ -1459,6 +1377,43 @@ func (q *Queries) GetAllAnswersByQuizID(ctx context.Context, session sql.NullStr return items, nil } +const getAllCompanyUsers = `-- name: GetAllCompanyUsers :many +SELECT id, amoid, amouserid, name, email, role, "Group", deleted, createdat FROM usersamo WHERE amoid = $1 AND deleted = false +` + +func (q *Queries) GetAllCompanyUsers(ctx context.Context, amoid int32) ([]Usersamo, error) { + rows, err := q.db.QueryContext(ctx, getAllCompanyUsers, amoid) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Usersamo + for rows.Next() { + var i Usersamo + if err := rows.Scan( + &i.ID, + &i.Amoid, + &i.Amouserid, + &i.Name, + &i.Email, + &i.Role, + &i.Group, + &i.Deleted, + &i.Createdat, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const getAllTokens = `-- name: GetAllTokens :many SELECT accountid, refreshtoken, accesstoken, authcode, expiration, createdat FROM tokens ` @@ -1493,25 +1448,21 @@ func (q *Queries) GetAllTokens(ctx context.Context) ([]Token, error) { return items, nil } -const getCurrentAccount = `-- name: GetCurrentAccount :one -SELECT id, accountid, amoid, name, email, role, "Group", deleted, createdat, subdomain, amouserid, country, driveurl FROM users WHERE AccountID = $1 AND Deleted = false +const getCurrentCompany = `-- name: GetCurrentCompany :one +SELECT id, accountid, amoid, name, deleted, createdat, subdomain, country, driveurl FROM accountsAmo WHERE AccountID = $1 AND Deleted = false ` -func (q *Queries) GetCurrentAccount(ctx context.Context, accountid string) (User, error) { - row := q.db.QueryRowContext(ctx, getCurrentAccount, accountid) - var i User +func (q *Queries) GetCurrentCompany(ctx context.Context, accountid string) (Accountsamo, error) { + row := q.db.QueryRowContext(ctx, getCurrentCompany, accountid) + var i Accountsamo err := row.Scan( &i.ID, &i.Accountid, &i.Amoid, &i.Name, - &i.Email, - &i.Role, - &i.Group, &i.Deleted, &i.Createdat, &i.Subdomain, - &i.Amouserid, &i.Country, &i.Driveurl, ) @@ -1634,8 +1585,11 @@ func (q *Queries) GetFieldByAmoID(ctx context.Context, amoid int32) (Field, erro } const getFieldsWithPagination = `-- name: GetFieldsWithPagination :many +WITH user_data AS ( + SELECT AmoID FROM accountsAmo WHERE accountsAmo.AccountID = $1 AND accountsAmo.Deleted = false +) SELECT f.id, f.amoid, f.code, f.accountid, f.name, f.entity, f.type, f.deleted, f.createdat, COUNT(*) OVER() as total_count -FROM fields f JOIN (SELECT AmoID FROM users WHERE users.AccountID = $1 AND Deleted = false) u ON f.AccountID = u.AmoID +FROM fields f JOIN user_data u ON f.AccountID = u.AmoID WHERE f.Deleted = false ORDER BY f.ID OFFSET ($2 - 1) * $3 LIMIT $3 ` @@ -1750,8 +1704,11 @@ func (q *Queries) GetListStartQuiz(ctx context.Context, accountid string) ([]int } const getPipelinesWithPagination = `-- name: GetPipelinesWithPagination :many +WITH user_data AS ( + SELECT AmoID FROM accountsAmo WHERE accountsAmo.AccountID = $1 AND accountsAmo.Deleted = false +) SELECT p.id, p.amoid, p.accountid, p.name, p.isarchive, p.deleted, p.createdat, COUNT(*) OVER() as total_count -FROM pipelines p JOIN (SELECT AmoID FROM users WHERE users.AccountID = $1 AND Deleted = false) u ON p.AccountID = u.AmoID +FROM pipelines p JOIN user_data u ON p.AccountID = u.AmoID WHERE p.Deleted = false ORDER BY p.ID OFFSET ($2 - 1) * $3 LIMIT $3 ` @@ -2368,8 +2325,11 @@ func (q *Queries) GetResultAnswers(ctx context.Context, id int64) ([]GetResultAn } const getStepsWithPagination = `-- name: GetStepsWithPagination :many +WITH user_data AS ( + SELECT AmoID FROM accountsAmo WHERE accountsAmo.AccountID = $1 AND accountsAmo.Deleted = false +) SELECT s.id, s.amoid, s.pipelineid, s.accountid, s.name, s.color, s.deleted, s.createdat, COUNT(*) OVER() as total_count -FROM steps s JOIN (SELECT AmoID FROM users WHERE users.AccountID = $1 AND Deleted = false) u ON s.AccountID = u.AmoID +FROM steps s JOIN user_data u ON s.AccountID = u.AmoID WHERE s.Deleted = false AND PipelineID = $4 ORDER BY s.ID OFFSET ($2 - 1) * $3 LIMIT $3 ` @@ -2432,8 +2392,11 @@ func (q *Queries) GetStepsWithPagination(ctx context.Context, arg GetStepsWithPa } const getTagsWithPagination = `-- name: GetTagsWithPagination :many +WITH user_data AS ( + SELECT AmoID FROM accountsAmo WHERE accountsAmo.AccountID = $1 AND accountsAmo.Deleted = false +) SELECT t.id, t.amoid, t.accountid, t.entity, t.name, t.color, t.deleted, t.createdat, COUNT(*) OVER() as total_count -FROM tags t JOIN (SELECT AmoID FROM users WHERE users.AccountID = $1 AND Deleted = false) u ON t.AccountID = u.AmoID +FROM tags t JOIN user_data u ON t.AccountID = u.AmoID WHERE t.Deleted = false ORDER BY t.ID OFFSET ($2 - 1) * $3 LIMIT $3 ` @@ -2688,44 +2651,28 @@ func (q *Queries) GetUserTagsByID(ctx context.Context, accountid int32) ([]GetUs } const getUserUsersByID = `-- name: GetUserUsersByID :many -SELECT ID,AccountID,AmoID,Name,Email,Role,"Group",Subdomain,AmoUserID,Country -FROM users -WHERE AmoUserID = $1 AND Deleted = false +SELECT id, amoid, amouserid, name, email, role, "Group", deleted, createdat FROM usersAmo WHERE amoid = $1 AND Deleted = false ` -type GetUserUsersByIDRow struct { - ID int64 `db:"id" json:"id"` - Accountid string `db:"accountid" json:"accountid"` - Amoid int32 `db:"amoid" json:"amoid"` - Name string `db:"name" json:"name"` - Email string `db:"email" json:"email"` - Role int32 `db:"role" json:"role"` - Group int32 `db:"Group" json:"Group"` - Subdomain string `db:"subdomain" json:"subdomain"` - Amouserid int32 `db:"amouserid" json:"amouserid"` - Country string `db:"country" json:"country"` -} - -func (q *Queries) GetUserUsersByID(ctx context.Context, amouserid int32) ([]GetUserUsersByIDRow, error) { - rows, err := q.db.QueryContext(ctx, getUserUsersByID, amouserid) +func (q *Queries) GetUserUsersByID(ctx context.Context, amoid int32) ([]Usersamo, error) { + rows, err := q.db.QueryContext(ctx, getUserUsersByID, amoid) if err != nil { return nil, err } defer rows.Close() - var items []GetUserUsersByIDRow + var items []Usersamo for rows.Next() { - var i GetUserUsersByIDRow + var i Usersamo if err := rows.Scan( &i.ID, - &i.Accountid, &i.Amoid, + &i.Amouserid, &i.Name, &i.Email, &i.Role, &i.Group, - &i.Subdomain, - &i.Amouserid, - &i.Country, + &i.Deleted, + &i.Createdat, ); err != nil { return nil, err } @@ -2742,11 +2689,11 @@ func (q *Queries) GetUserUsersByID(ctx context.Context, amouserid int32) ([]GetU const getUsersWithPagination = `-- name: GetUsersWithPagination :many WITH user_data AS ( - SELECT AmoID FROM users WHERE users.AccountID = $1 AND Deleted = false + SELECT AmoID FROM accountsAmo WHERE accountsAmo.AccountID = $1 AND accountsAmo.Deleted = false ) -SELECT u.id, u.accountid, u.amoid, u.name, u.email, u.role, u."Group", u.deleted, u.createdat, u.subdomain, u.amouserid, u.country, u.driveurl, COUNT(*) OVER() as total_count -FROM users u -JOIN user_data a ON u.AmoUserID = a.AmoID +SELECT u.id, u.amoid, u.amouserid, u.name, u.email, u.role, u."Group", u.deleted, u.createdat, COUNT(*) OVER() as total_count +FROM usersAmo u + JOIN user_data a ON u.AmoID = a.AmoID WHERE u.Deleted = false ORDER BY u.ID OFFSET ($2 - 1) * $3 LIMIT $3 ` @@ -2758,20 +2705,16 @@ type GetUsersWithPaginationParams struct { } type GetUsersWithPaginationRow struct { - ID int64 `db:"id" json:"id"` - Accountid string `db:"accountid" json:"accountid"` - Amoid int32 `db:"amoid" json:"amoid"` - Name string `db:"name" json:"name"` - Email string `db:"email" json:"email"` - Role int32 `db:"role" json:"role"` - Group int32 `db:"Group" json:"Group"` - Deleted bool `db:"deleted" json:"deleted"` - Createdat sql.NullTime `db:"createdat" json:"createdat"` - Subdomain string `db:"subdomain" json:"subdomain"` - Amouserid int32 `db:"amouserid" json:"amouserid"` - Country string `db:"country" json:"country"` - Driveurl string `db:"driveurl" json:"driveurl"` - TotalCount int64 `db:"total_count" json:"total_count"` + ID int64 `db:"id" json:"id"` + Amoid int32 `db:"amoid" json:"amoid"` + Amouserid int32 `db:"amouserid" json:"amouserid"` + Name string `db:"name" json:"name"` + Email string `db:"email" json:"email"` + Role int32 `db:"role" json:"role"` + Group int32 `db:"Group" json:"Group"` + Deleted bool `db:"deleted" json:"deleted"` + Createdat time.Time `db:"createdat" json:"createdat"` + TotalCount int64 `db:"total_count" json:"total_count"` } func (q *Queries) GetUsersWithPagination(ctx context.Context, arg GetUsersWithPaginationParams) ([]GetUsersWithPaginationRow, error) { @@ -2785,18 +2728,14 @@ func (q *Queries) GetUsersWithPagination(ctx context.Context, arg GetUsersWithPa var i GetUsersWithPaginationRow if err := rows.Scan( &i.ID, - &i.Accountid, &i.Amoid, + &i.Amouserid, &i.Name, &i.Email, &i.Role, &i.Group, &i.Deleted, &i.Createdat, - &i.Subdomain, - &i.Amouserid, - &i.Country, - &i.Driveurl, &i.TotalCount, ); err != nil { return nil, err @@ -2818,13 +2757,13 @@ SELECT a.quiz_id,a.id,a.result,a.question_id,a.content,a.session, FROM answer a2 WHERE a2.start = true AND a2.session = a.session LIMIT 1) AS utm -,t.accesstoken,r.accountid,r.fieldsrule,r.tagstoadd,r.performerid,r.stepid,r.pipelineid,(SELECT u.name FROM users u WHERE u.amoid = r.performerid) AS performer_name,u.subdomain,u.accountid,u.driveurl +,t.accesstoken,r.accountid,r.fieldsrule,r.tagstoadd,r.performerid,r.stepid,r.pipelineid,(SELECT u.name FROM usersAmo u WHERE u.AmoUserID = r.performerid) AS performer_name,u.subdomain,u.accountid,u.driveurl FROM answer a INNER JOIN quiz q ON a.quiz_id = q.id LEFT JOIN amoCRMStatuses s ON a.id = s.AnswerID INNER JOIN rules r ON q.id = r.QuizID INNER JOIN tokens t ON q.accountid = t.AccountID - INNER JOIN users u ON q.accountid = u.accountid AND u.amoid = r.accountid + INNER JOIN accountsAmo u ON q.accountid = u.accountid AND u.amoid = r.accountid WHERE a.result = true AND s.id IS NULL AND a.deleted = false @@ -2851,7 +2790,7 @@ type GettingAmoUsersTrueResultsRow struct { PerformerName string `db:"performer_name" json:"performer_name"` Subdomain string `db:"subdomain" json:"subdomain"` Accountid_2 string `db:"accountid_2" json:"accountid_2"` - Driveurl string `db:"driveurl" json:"driveurl"` + Driveurl sql.NullString `db:"driveurl" json:"driveurl"` } func (q *Queries) GettingAmoUsersTrueResults(ctx context.Context) ([]GettingAmoUsersTrueResultsRow, error) { @@ -3375,7 +3314,7 @@ func (q *Queries) QuizCopyQid(ctx context.Context, arg QuizCopyQidParams) (QuizC const setQuizSettings = `-- name: SetQuizSettings :one INSERT INTO rules (AccountID, QuizID, PerformerID, PipelineID, StepID, FieldsRule,TagsToAdd) SELECT u.AmoID AS AccountID,$1 AS QuizID,$2 AS PerformerID,$3 AS PipelineID, - $4 AS StepID,$5 AS FieldsRule,$6 AS TagsToAdd FROM users u WHERE u.AccountID = $7 AND u.Deleted = false + $4 AS StepID,$5 AS FieldsRule,$6 AS TagsToAdd FROM accountsamo u WHERE u.AccountID = $7 AND u.Deleted = false RETURNING id ` @@ -3408,7 +3347,7 @@ const settingDealAmoStatus = `-- name: SettingDealAmoStatus :exec INSERT INTO amoCRMStatuses (AccountID, DealID, AnswerID, Status) SELECT u.AmoID, $1, $2, $3 FROM tokens AS t - JOIN users AS u ON t.AccountID = u.AccountID + JOIN accountsAmo AS u ON t.AccountID = u.AccountID WHERE t.AccessToken = $4 AND u.Deleted = false ` @@ -3430,12 +3369,13 @@ func (q *Queries) SettingDealAmoStatus(ctx context.Context, arg SettingDealAmoSt } const softDeleteAccount = `-- name: SoftDeleteAccount :exec -WITH userd AS ( - SELECT AmoUserID FROM users WHERE users.AccountID = $1 -), - tokend AS ( - UPDATE users SET Deleted = true WHERE AmoUserID IN (SELECT AmoUserID FROM userd) - ) +WITH amoCompany AS ( + SELECT AmoID FROM accountsAmo WHERE accountsAmo.AccountID = $1 +),usersDel AS ( + UPDATE usersAmo SET Deleted = true WHERE AmoID = (SELECT AmoID FROM amoCompany) + ), + companyDel AS ( UPDATE accountsAmo SET Deleted = true WHERE AmoID = (SELECT AmoID FROM amoCompany) + ) DELETE FROM tokens WHERE tokens.AccountID = $1 ` @@ -3481,9 +3421,35 @@ func (q *Queries) TemplateCopy(ctx context.Context, arg TemplateCopyParams) (int return quiz_id, err } +const updateAmoAccountUser = `-- name: UpdateAmoAccountUser :exec +UPDATE usersAmo SET Name = $3, Email = $4, Role = $5, "Group" = $6 +WHERE AmoID = $1 AND AmoUserID = $2 AND deleted = false +` + +type UpdateAmoAccountUserParams struct { + Amoid int32 `db:"amoid" json:"amoid"` + Amouserid int32 `db:"amouserid" json:"amouserid"` + Name string `db:"name" json:"name"` + Email string `db:"email" json:"email"` + Role int32 `db:"role" json:"role"` + Group int32 `db:"Group" json:"Group"` +} + +func (q *Queries) UpdateAmoAccountUser(ctx context.Context, arg UpdateAmoAccountUserParams) error { + _, err := q.db.ExecContext(ctx, updateAmoAccountUser, + arg.Amoid, + arg.Amouserid, + arg.Name, + arg.Email, + arg.Role, + arg.Group, + ) + return err +} + const updateFieldRules = `-- name: UpdateFieldRules :exec UPDATE rules SET FieldsRule = $1 -WHERE AccountID = (SELECT AmoID FROM users WHERE users.AccountID = $2 AND users.Deleted = false) AND QuizID = $3 AND Deleted = false +WHERE AccountID = (SELECT AmoID FROM accountsAmo WHERE accountsAmo.AccountID = $2 AND accountsAmo.Deleted = false) AND QuizID = $3 AND Deleted = false ` type UpdateFieldRulesParams struct { @@ -3592,25 +3558,9 @@ func (q *Queries) UpdateTags(ctx context.Context, dollar_1 json.RawMessage) erro return err } -const updateUsers = `-- name: UpdateUsers :exec -UPDATE users AS u -SET Name = (update_data ->> 'Name')::varchar(512), - Email = (update_data ->> 'Email')::varchar(50), - Role = (update_data ->> 'Role')::INT, - "Group" = (update_data ->> 'Group')::INT, - AmoUserID= (update_data ->> 'AmoUserID')::INT -FROM json_array_elements($1::json) AS update_data -WHERE u.AmoID = (update_data ->> 'AmocrmID')::INT -` - -func (q *Queries) UpdateUsers(ctx context.Context, dollar_1 json.RawMessage) error { - _, err := q.db.ExecContext(ctx, updateUsers, dollar_1) - return err -} - const updatingDealAmoStatus = `-- name: UpdatingDealAmoStatus :exec UPDATE amoCRMStatuses SET Status = $1 -WHERE DealID = $2 AND AccountID = (SELECT u.AmoID FROM tokens AS t JOIN users AS u ON t.AccountID = u.AccountID WHERE t.AccessToken = $3 AND u.Deleted = false) +WHERE DealID = $2 AND AccountID = (SELECT u.AmoID FROM tokens AS t JOIN accountsAmo AS u ON t.AccountID = u.AccountID WHERE t.AccessToken = $3 AND u.Deleted = false) ` type UpdatingDealAmoStatusParams struct { @@ -3625,14 +3575,17 @@ func (q *Queries) UpdatingDealAmoStatus(ctx context.Context, arg UpdatingDealAmo } const webhookDelete = `-- name: WebhookDelete :exec -WITH userd AS ( - UPDATE users SET Deleted = true WHERE AmoUserID = $1 RETURNING AccountID +WITH companyDel AS ( + UPDATE accountsAmo SET Deleted = true WHERE accountsAmo.AmoID = $1 RETURNING AccountID +), +userDel AS ( +UPDATE usersAmo SET Deleted = true WHERE AmoID = $1 ) -DELETE FROM tokens WHERE AccountID IN (SELECT AccountID FROM userd) +DELETE FROM tokens WHERE AccountID IN (SELECT AccountID FROM companyDel) ` -func (q *Queries) WebhookDelete(ctx context.Context, amouserid int32) error { - _, err := q.db.ExecContext(ctx, webhookDelete, amouserid) +func (q *Queries) WebhookDelete(ctx context.Context, amoid int32) error { + _, err := q.db.ExecContext(ctx, webhookDelete, amoid) return err } From 2fe0379a677655fbce9e82ad6e1e3295990caec6 Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 12 Jun 2024 16:36:36 +0300 Subject: [PATCH 04/59] update amo queries --- dal/schema/000014_init.up.sql | 2 +- model/amoResp.go | 2 +- repository/amo/amo.go | 38 +++++++++++++---------------------- 3 files changed, 16 insertions(+), 26 deletions(-) diff --git a/dal/schema/000014_init.up.sql b/dal/schema/000014_init.up.sql index b0ba982..9542bbb 100644 --- a/dal/schema/000014_init.up.sql +++ b/dal/schema/000014_init.up.sql @@ -10,7 +10,7 @@ CREATE TABLE IF NOT EXISTS accountsAmo ( CreatedAt TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, Subdomain VARCHAR(50) NOT NULL DEFAULT '', -- поддомен - пример https://penadigitaltech.amocrm.ru Country VARCHAR(50) NOT NULL DEFAULT '', - DriveURL VARCHAR(255) DEFAULT '' -- URL объктного хранилища + DriveURL VARCHAR(255) NOT NULL DEFAULT '' -- URL объктного хранилища ); CREATE UNIQUE INDEX idx_unique_accountsAmo ON accountsAmo (amoID) WHERE Deleted = false; diff --git a/model/amoResp.go b/model/amoResp.go index fd49975..1a65002 100644 --- a/model/amoResp.go +++ b/model/amoResp.go @@ -55,7 +55,7 @@ type UserListResp struct { /* - общее количество юзеров, которые у нас закешированы для этого пользователя*/ Count int64 `json:"count"` /* - список юзеров, которые были закешированы нашим сервисом*/ - Items []User `json:"items"` + Items []AmoAccountUser `json:"items"` } type UserListStepsResp struct { diff --git a/repository/amo/amo.go b/repository/amo/amo.go index 76c0cb6..9f9f472 100644 --- a/repository/amo/amo.go +++ b/repository/amo/amo.go @@ -39,20 +39,18 @@ func (r *AmoRepository) GettingUserWithPagination(ctx context.Context, req *mode return nil, err } var count int64 - var users []model.User + var users []model.AmoAccountUser for _, row := range rows { - user := model.User{ + user := model.AmoAccountUser{ ID: row.ID, - Accountid: row.Accountid, AmoID: row.Amoid, + AmoUserID: row.Amouserid, Name: row.Name, Email: row.Email, Group: row.Group, Role: row.Role, - Createdat: row.Createdat.Time.Unix(), - Subdomain: row.Subdomain, - Amouserid: row.Amouserid, - Country: row.Country, + Deleted: row.Deleted, + CreatedAt: row.Createdat, } count = row.TotalCount @@ -75,43 +73,35 @@ func (r *AmoRepository) SoftDeleteAccount(ctx context.Context, accountID string) return nil } -func (r *AmoRepository) GetCurrentAccount(ctx context.Context, accountID string) (*model.User, error) { - row, err := r.queries.GetCurrentAccount(ctx, accountID) +func (r *AmoRepository) GetCurrentAccount(ctx context.Context, accountID string) (*model.AmoAccount, error) { + row, err := r.queries.GetCurrentCompany(ctx, accountID) if err != nil { return nil, err } - user := model.User{ + user := model.AmoAccount{ ID: row.ID, - Accountid: row.Accountid, + AccountID: row.Accountid, AmoID: row.Amoid, Name: row.Name, - Email: row.Email, - Role: row.Role, - Group: row.Group, - Createdat: row.Createdat.Time.Unix(), + Deleted: row.Deleted, + CreatedAt: row.Createdat, Subdomain: row.Subdomain, - Amouserid: row.Amouserid, Country: row.Country, - DriveURL: row.Driveurl, + DriveURL: row.Driveurl.String, } return &user, nil } -func (r *AmoRepository) CreateAccount(ctx context.Context, accountID string, userInfo model.User) error { +func (r *AmoRepository) CreateAccount(ctx context.Context, accountID string, userInfo model.AmoAccount) error { err := r.queries.CreateAmoAccount(ctx, sqlcgen.CreateAmoAccountParams{ Accountid: accountID, Amoid: userInfo.AmoID, Name: userInfo.Name, - Email: userInfo.Email, - Role: userInfo.Role, - Group: userInfo.Group, - Createdat: sql.NullTime{Time: time.Now(), Valid: true}, Subdomain: userInfo.Subdomain, - Amouserid: userInfo.Amouserid, Country: userInfo.Country, - Driveurl: userInfo.DriveURL, + Driveurl: sql.NullString{String: userInfo.DriveURL, Valid: true}, }) if err != nil { From 405cd05c98ffe1f2a1ef6c1e9400b44a357b5cb1 Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 12 Jun 2024 16:37:25 +0300 Subject: [PATCH 05/59] update sqlc gen --- dal/sqlcgen/models.go | 18 +++++++++--------- dal/sqlcgen/queries.sql.go | 14 +++++++------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/dal/sqlcgen/models.go b/dal/sqlcgen/models.go index fa9e952..c6d928d 100644 --- a/dal/sqlcgen/models.go +++ b/dal/sqlcgen/models.go @@ -21,15 +21,15 @@ type Account struct { } type Accountsamo struct { - ID int64 `db:"id" json:"id"` - Accountid string `db:"accountid" json:"accountid"` - Amoid int32 `db:"amoid" json:"amoid"` - Name string `db:"name" json:"name"` - Deleted bool `db:"deleted" json:"deleted"` - Createdat time.Time `db:"createdat" json:"createdat"` - Subdomain string `db:"subdomain" json:"subdomain"` - Country string `db:"country" json:"country"` - Driveurl sql.NullString `db:"driveurl" json:"driveurl"` + ID int64 `db:"id" json:"id"` + Accountid string `db:"accountid" json:"accountid"` + Amoid int32 `db:"amoid" json:"amoid"` + Name string `db:"name" json:"name"` + Deleted bool `db:"deleted" json:"deleted"` + Createdat time.Time `db:"createdat" json:"createdat"` + Subdomain string `db:"subdomain" json:"subdomain"` + Country string `db:"country" json:"country"` + Driveurl string `db:"driveurl" json:"driveurl"` } type Amocrmstatus struct { diff --git a/dal/sqlcgen/queries.sql.go b/dal/sqlcgen/queries.sql.go index 7f8b2bf..9902b06 100644 --- a/dal/sqlcgen/queries.sql.go +++ b/dal/sqlcgen/queries.sql.go @@ -715,12 +715,12 @@ VALUES ($1, $2, $3, $4, $5, $6) ` type CreateAmoAccountParams struct { - Accountid string `db:"accountid" json:"accountid"` - Amoid int32 `db:"amoid" json:"amoid"` - Name string `db:"name" json:"name"` - Subdomain string `db:"subdomain" json:"subdomain"` - Country string `db:"country" json:"country"` - Driveurl sql.NullString `db:"driveurl" json:"driveurl"` + Accountid string `db:"accountid" json:"accountid"` + Amoid int32 `db:"amoid" json:"amoid"` + Name string `db:"name" json:"name"` + Subdomain string `db:"subdomain" json:"subdomain"` + Country string `db:"country" json:"country"` + Driveurl string `db:"driveurl" json:"driveurl"` } // amo methods: @@ -2790,7 +2790,7 @@ type GettingAmoUsersTrueResultsRow struct { PerformerName string `db:"performer_name" json:"performer_name"` Subdomain string `db:"subdomain" json:"subdomain"` Accountid_2 string `db:"accountid_2" json:"accountid_2"` - Driveurl sql.NullString `db:"driveurl" json:"driveurl"` + Driveurl string `db:"driveurl" json:"driveurl"` } func (q *Queries) GettingAmoUsersTrueResults(ctx context.Context) ([]GettingAmoUsersTrueResultsRow, error) { From 391f9df6241c7f55e596502fffac90073b54a001 Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 12 Jun 2024 16:49:31 +0300 Subject: [PATCH 06/59] update account and users amo repo methods --- dal/db_query/queries.sql | 4 +-- repository/amo/amo.go | 75 +++++++++++++++------------------------- 2 files changed, 29 insertions(+), 50 deletions(-) diff --git a/dal/db_query/queries.sql b/dal/db_query/queries.sql index d172f23..0aae5f9 100644 --- a/dal/db_query/queries.sql +++ b/dal/db_query/queries.sql @@ -963,8 +963,8 @@ UPDATE usersAmo SET Name = $3, Email = $4, Role = $5, "Group" = $6 WHERE AmoID = $1 AND AmoUserID = $2 AND deleted = false; -- name: AddAmoAccountUser :exec -INSERT INTO usersAmo (AmoID, AmoUserID, Name, Email, Role, "Group", Deleted, CreatedAt) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8); +INSERT INTO usersAmo (AmoID, AmoUserID, Name, Email, Role, "Group") +VALUES ($1, $2, $3, $4, $5, $6); -- name: GettingAmoUsersTrueResults :many SELECT a.quiz_id,a.id,a.result,a.question_id,a.content,a.session, diff --git a/repository/amo/amo.go b/repository/amo/amo.go index 9f9f472..35a8622 100644 --- a/repository/amo/amo.go +++ b/repository/amo/amo.go @@ -88,7 +88,7 @@ func (r *AmoRepository) GetCurrentAccount(ctx context.Context, accountID string) CreatedAt: row.Createdat, Subdomain: row.Subdomain, Country: row.Country, - DriveURL: row.Driveurl.String, + DriveURL: row.Driveurl, } return &user, nil @@ -101,7 +101,7 @@ func (r *AmoRepository) CreateAccount(ctx context.Context, accountID string, use Name: userInfo.Name, Subdomain: userInfo.Subdomain, Country: userInfo.Country, - Driveurl: sql.NullString{String: userInfo.DriveURL, Valid: true}, + Driveurl: userInfo.DriveURL, }) if err != nil { @@ -111,15 +111,17 @@ func (r *AmoRepository) CreateAccount(ctx context.Context, accountID string, use return nil } -func (r *AmoRepository) CheckMainUser(ctx context.Context, user model.User) error { - err := r.queries.CheckMainUser(ctx, sqlcgen.CheckMainUserParams{ - Name: user.Name, - Group: user.Group, - Email: user.Email, - Role: user.Role, - Amoid: user.AmoID, - }) +// todo возможно стоит обновлять еще и компанию пока не знаю +func (r *AmoRepository) AddAmoAccountUser(ctx context.Context, user model.AmoAccountUser) error { + err := r.queries.AddAmoAccountUser(ctx, sqlcgen.AddAmoAccountUserParams{ + Amoid: user.AmoID, + Amouserid: user.AmoUserID, + Name: user.Name, + Email: user.Email, + Role: user.Role, + Group: user.Group, + }) if err != nil { return err } @@ -127,39 +129,18 @@ func (r *AmoRepository) CheckMainUser(ctx context.Context, user model.User) erro return nil } -func (r *AmoRepository) CheckAndUpdateUsers(ctx context.Context, users []model.User) error { - dollar1, err := json.Marshal(users) +func (r *AmoRepository) UpdateAmoAccountUser(ctx context.Context, user model.AmoAccountUser) error { + err := r.queries.UpdateAmoAccountUser(ctx, sqlcgen.UpdateAmoAccountUserParams{ + Amoid: user.AmoID, + Amouserid: user.AmoUserID, + Name: user.Name, + Email: user.Email, + Role: user.Role, + Group: user.Group, + }) if err != nil { return err } - rows, err := r.queries.CheckUsers(ctx, dollar1) - if err != nil { - return err - } - - if rows != nil { - var toUpdate []model.User - for _, row := range rows { - to := model.User{ - AmoID: row.Amoid, - Name: row.Name, - Group: row.Group, - Role: row.Role, - Email: row.Email, - Amouserid: row.Amouserid, - } - toUpdate = append(toUpdate, to) - } - dollar1, err := json.Marshal(toUpdate) - if err != nil { - return err - } - - err = r.queries.UpdateUsers(ctx, dollar1) - if err != nil { - return err - } - } return nil } @@ -185,27 +166,25 @@ func (r *AmoRepository) DeleteUsers(ctx context.Context, ids []int64) error { return nil } -func (r *AmoRepository) GetUserUsersByID(ctx context.Context, amoUserID int32) ([]model.User, error) { +func (r *AmoRepository) GetUserUsersByID(ctx context.Context, amoUserID int32) ([]model.AmoAccountUser, error) { rows, err := r.queries.GetUserUsersByID(ctx, amoUserID) if err != nil { return nil, err } - var users []model.User + var users []model.AmoAccountUser for _, row := range rows { - user := model.User{ + user := model.AmoAccountUser{ ID: row.ID, - Accountid: row.Accountid, AmoID: row.Amoid, + AmoUserID: row.Amouserid, Name: row.Name, Email: row.Email, Group: row.Group, Role: row.Role, - Subdomain: row.Subdomain, - Amouserid: row.Amouserid, - Country: row.Country, + Deleted: row.Deleted, + CreatedAt: row.Createdat, } - users = append(users, user) } From 429052cf2e7d9243bedc75b8f3b6447ed2fa33fa Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 12 Jun 2024 16:50:15 +0300 Subject: [PATCH 07/59] update sqlc gen --- dal/sqlcgen/queries.sql.go | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/dal/sqlcgen/queries.sql.go b/dal/sqlcgen/queries.sql.go index 9902b06..1d80a1c 100644 --- a/dal/sqlcgen/queries.sql.go +++ b/dal/sqlcgen/queries.sql.go @@ -61,19 +61,17 @@ func (q *Queries) AccountPagination(ctx context.Context, arg AccountPaginationPa } const addAmoAccountUser = `-- name: AddAmoAccountUser :exec -INSERT INTO usersAmo (AmoID, AmoUserID, Name, Email, Role, "Group", Deleted, CreatedAt) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8) +INSERT INTO usersAmo (AmoID, AmoUserID, Name, Email, Role, "Group") +VALUES ($1, $2, $3, $4, $5, $6) ` type AddAmoAccountUserParams struct { - Amoid int32 `db:"amoid" json:"amoid"` - Amouserid int32 `db:"amouserid" json:"amouserid"` - Name string `db:"name" json:"name"` - Email string `db:"email" json:"email"` - Role int32 `db:"role" json:"role"` - Group int32 `db:"Group" json:"Group"` - Deleted bool `db:"deleted" json:"deleted"` - Createdat time.Time `db:"createdat" json:"createdat"` + Amoid int32 `db:"amoid" json:"amoid"` + Amouserid int32 `db:"amouserid" json:"amouserid"` + Name string `db:"name" json:"name"` + Email string `db:"email" json:"email"` + Role int32 `db:"role" json:"role"` + Group int32 `db:"Group" json:"Group"` } func (q *Queries) AddAmoAccountUser(ctx context.Context, arg AddAmoAccountUserParams) error { @@ -84,8 +82,6 @@ func (q *Queries) AddAmoAccountUser(ctx context.Context, arg AddAmoAccountUserPa arg.Email, arg.Role, arg.Group, - arg.Deleted, - arg.Createdat, ) return err } From 7764e7618cca1b0d7b93563a2c7e17759a86f885 Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 12 Jun 2024 17:03:27 +0300 Subject: [PATCH 08/59] update amo CreateAccount --- repository/amo/amo.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repository/amo/amo.go b/repository/amo/amo.go index 35a8622..b67a328 100644 --- a/repository/amo/amo.go +++ b/repository/amo/amo.go @@ -94,9 +94,9 @@ func (r *AmoRepository) GetCurrentAccount(ctx context.Context, accountID string) return &user, nil } -func (r *AmoRepository) CreateAccount(ctx context.Context, accountID string, userInfo model.AmoAccount) error { +func (r *AmoRepository) CreateAccount(ctx context.Context, userInfo model.AmoAccount) error { err := r.queries.CreateAmoAccount(ctx, sqlcgen.CreateAmoAccountParams{ - Accountid: accountID, + Accountid: userInfo.AccountID, Amoid: userInfo.AmoID, Name: userInfo.Name, Subdomain: userInfo.Subdomain, From ab69a20b1cc5355a88be890e46f68bd9c94108f7 Mon Sep 17 00:00:00 2001 From: Pavel Date: Tue, 18 Jun 2024 11:58:15 +0300 Subject: [PATCH 09/59] add query for checked expired tokens and update head accounnt amo --- dal/db_query/queries.sql | 7 +++++-- model/amo.go | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/dal/db_query/queries.sql b/dal/db_query/queries.sql index d512c5a..517880f 100644 --- a/dal/db_query/queries.sql +++ b/dal/db_query/queries.sql @@ -700,8 +700,11 @@ WHERE accountID = $5; -- name: GetAllTokens :many SELECT * FROM tokens; --- name: CheckExpired :many -SELECT * FROM tokens WHERE Expiration <= TO_TIMESTAMP(EXTRACT(EPOCH FROM NOW()) + (10 * 60)); +-- name: CheckExpiredToken :one +SELECT * FROM tokens WHERE AccountID = $1 AND Expiration <= NOW(); + +-- name: UpdateAmoAccount :exec +UPDATE accountsAmo SET AmoID = $2, Name = $3, Subdomain = $4, Country = $5, DriveURL = $6 WHERE AccountID = $1 AND Deleted = false; -- name: WebhookDelete :exec WITH companyDel AS ( diff --git a/model/amo.go b/model/amo.go index 2c9287e..bdcbcf1 100644 --- a/model/amo.go +++ b/model/amo.go @@ -24,6 +24,7 @@ type AmoAccount struct { Subdomain string `json:"subdomain"` // поддомен Country string `json:"country"` // Страна DriveURL string `json:"driveURL"` // URL объктного хранилища + Stale bool `json:"stale"` // флаг "не свежести" если с токенами все в порядке - false, если просрочились то true } type UserGroups struct { From 42340d9ff3642366508fd4bd8b61449c4cb2d982 Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 12 Jun 2024 18:25:16 +0300 Subject: [PATCH 10/59] update sqlc gen --- dal/sqlcgen/queries.sql.go | 69 +++++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 30 deletions(-) diff --git a/dal/sqlcgen/queries.sql.go b/dal/sqlcgen/queries.sql.go index fb0e9be..2769ffa 100644 --- a/dal/sqlcgen/queries.sql.go +++ b/dal/sqlcgen/queries.sql.go @@ -190,38 +190,22 @@ func (q *Queries) CheckAndAddDefault(ctx context.Context, arg CheckAndAddDefault return err } -const checkExpired = `-- name: CheckExpired :many -SELECT accountid, refreshtoken, accesstoken, authcode, expiration, createdat FROM tokens WHERE Expiration <= TO_TIMESTAMP(EXTRACT(EPOCH FROM NOW()) + (10 * 60)) +const checkExpiredToken = `-- name: CheckExpiredToken :one +SELECT accountid, refreshtoken, accesstoken, authcode, expiration, createdat FROM tokens WHERE AccountID = $1 AND Expiration <= NOW() ` -func (q *Queries) CheckExpired(ctx context.Context) ([]Token, error) { - rows, err := q.db.QueryContext(ctx, checkExpired) - if err != nil { - return nil, err - } - defer rows.Close() - var items []Token - for rows.Next() { - var i Token - if err := rows.Scan( - &i.Accountid, - &i.Refreshtoken, - &i.Accesstoken, - &i.Authcode, - &i.Expiration, - &i.Createdat, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Close(); err != nil { - return nil, err - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil +func (q *Queries) CheckExpiredToken(ctx context.Context, accountid string) (Token, error) { + row := q.db.QueryRowContext(ctx, checkExpiredToken, accountid) + var i Token + err := row.Scan( + &i.Accountid, + &i.Refreshtoken, + &i.Accesstoken, + &i.Authcode, + &i.Expiration, + &i.Createdat, + ) + return i, err } const checkFields = `-- name: CheckFields :many @@ -3417,6 +3401,31 @@ func (q *Queries) TemplateCopy(ctx context.Context, arg TemplateCopyParams) (int return quiz_id, err } +const updateAmoAccount = `-- name: UpdateAmoAccount :exec +UPDATE accountsAmo SET AmoID = $2, Name = $3, Subdomain = $4, Country = $5, DriveURL = $6 WHERE AccountID = $1 AND Deleted = false +` + +type UpdateAmoAccountParams struct { + Accountid string `db:"accountid" json:"accountid"` + Amoid int32 `db:"amoid" json:"amoid"` + Name string `db:"name" json:"name"` + Subdomain string `db:"subdomain" json:"subdomain"` + Country string `db:"country" json:"country"` + Driveurl string `db:"driveurl" json:"driveurl"` +} + +func (q *Queries) UpdateAmoAccount(ctx context.Context, arg UpdateAmoAccountParams) error { + _, err := q.db.ExecContext(ctx, updateAmoAccount, + arg.Accountid, + arg.Amoid, + arg.Name, + arg.Subdomain, + arg.Country, + arg.Driveurl, + ) + return err +} + const updateAmoAccountUser = `-- name: UpdateAmoAccountUser :exec UPDATE usersAmo SET Name = $3, Email = $4, Role = $5, "Group" = $6 WHERE AmoID = $1 AND AmoUserID = $2 AND deleted = false From 5b621f8a9b2134f036125e1d216e60f4c8a7e338 Mon Sep 17 00:00:00 2001 From: Pavel Date: Tue, 18 Jun 2024 12:12:46 +0300 Subject: [PATCH 11/59] added amo repo methods for check account stale and update head account data --- dal/db_query/queries.sql | 2 +- repository/amo/amo.go | 50 ++++++++++++++++++++-------------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/dal/db_query/queries.sql b/dal/db_query/queries.sql index 517880f..5f23d2f 100644 --- a/dal/db_query/queries.sql +++ b/dal/db_query/queries.sql @@ -704,7 +704,7 @@ SELECT * FROM tokens; SELECT * FROM tokens WHERE AccountID = $1 AND Expiration <= NOW(); -- name: UpdateAmoAccount :exec -UPDATE accountsAmo SET AmoID = $2, Name = $3, Subdomain = $4, Country = $5, DriveURL = $6 WHERE AccountID = $1 AND Deleted = false; +UPDATE accountsAmo SET Name = $2, Subdomain = $3, Country = $4, DriveURL = $5 WHERE AccountID = $1 AND Deleted = false; -- name: WebhookDelete :exec WITH companyDel AS ( diff --git a/repository/amo/amo.go b/repository/amo/amo.go index b67a328..e6c99e1 100644 --- a/repository/amo/amo.go +++ b/repository/amo/amo.go @@ -91,9 +91,34 @@ func (r *AmoRepository) GetCurrentAccount(ctx context.Context, accountID string) DriveURL: row.Driveurl, } + _, err = r.queries.CheckExpiredToken(ctx, accountID) + if err != nil { + if err == sql.ErrNoRows { + user.Stale = false + return &user, nil + } + return nil, err + } + + user.Stale = true + return &user, nil } +func (r *AmoRepository) UpdateCurrentAccount(ctx context.Context, user model.AmoAccount) error { + err := r.queries.UpdateAmoAccount(ctx, sqlcgen.UpdateAmoAccountParams{ + Accountid: user.AccountID, + Name: user.Name, + Subdomain: user.Subdomain, + Country: user.Country, + Driveurl: user.DriveURL, + }) + if err != nil { + return err + } + return nil +} + func (r *AmoRepository) CreateAccount(ctx context.Context, userInfo model.AmoAccount) error { err := r.queries.CreateAmoAccount(ctx, sqlcgen.CreateAmoAccountParams{ Accountid: userInfo.AccountID, @@ -225,31 +250,6 @@ func (r *AmoRepository) WebhookUpdate(ctx context.Context, token model.Token) er return nil } -// воркер запускается каждые 5 минут, поэтомму ищем токены котторые исекают менее чем через 10 минут отдаем их на обноление -func (r *AmoRepository) CheckExpired(ctx context.Context) ([]model.Token, error) { - rows, err := r.queries.CheckExpired(ctx) - if err != nil { - return nil, err - } - - var tokens []model.Token - - for _, row := range rows { - token := model.Token{ - AccountID: row.Accountid, - AccessToken: row.Accesstoken, - RefreshToken: row.Refreshtoken, - AuthCode: row.Authcode, - Expiration: row.Expiration.Unix(), - CreatedAt: row.Createdat.Time.Unix(), - } - - tokens = append(tokens, token) - } - - return tokens, nil -} - func (r *AmoRepository) GetAllTokens(ctx context.Context) ([]model.Token, error) { rows, err := r.queries.GetAllTokens(ctx) if err != nil { From 6a7e3f9ea52fa5d30bc2a237bae1c52f6154fae3 Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 12 Jun 2024 18:38:45 +0300 Subject: [PATCH 12/59] update sqlc gen --- dal/sqlcgen/queries.sql.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dal/sqlcgen/queries.sql.go b/dal/sqlcgen/queries.sql.go index 2769ffa..c134333 100644 --- a/dal/sqlcgen/queries.sql.go +++ b/dal/sqlcgen/queries.sql.go @@ -3402,12 +3402,11 @@ func (q *Queries) TemplateCopy(ctx context.Context, arg TemplateCopyParams) (int } const updateAmoAccount = `-- name: UpdateAmoAccount :exec -UPDATE accountsAmo SET AmoID = $2, Name = $3, Subdomain = $4, Country = $5, DriveURL = $6 WHERE AccountID = $1 AND Deleted = false +UPDATE accountsAmo SET Name = $2, Subdomain = $3, Country = $4, DriveURL = $5 WHERE AccountID = $1 AND Deleted = false ` type UpdateAmoAccountParams struct { Accountid string `db:"accountid" json:"accountid"` - Amoid int32 `db:"amoid" json:"amoid"` Name string `db:"name" json:"name"` Subdomain string `db:"subdomain" json:"subdomain"` Country string `db:"country" json:"country"` @@ -3417,7 +3416,6 @@ type UpdateAmoAccountParams struct { func (q *Queries) UpdateAmoAccount(ctx context.Context, arg UpdateAmoAccountParams) error { _, err := q.db.ExecContext(ctx, updateAmoAccount, arg.Accountid, - arg.Amoid, arg.Name, arg.Subdomain, arg.Country, From 1c53cb3c1c371db297b142f1129fd75fcf5a7df6 Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 13 Jun 2024 15:25:11 +0300 Subject: [PATCH 13/59] add init connection with clickHouse --- dal/dal.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/dal/dal.go b/dal/dal.go index 4d0c38d..9ee3a72 100644 --- a/dal/dal.go +++ b/dal/dal.go @@ -5,6 +5,7 @@ import ( "database/sql" _ "embed" "errors" + "fmt" _ "github.com/lib/pq" "github.com/minio/minio-go/v7" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/dal/sqlcgen" @@ -177,3 +178,29 @@ func (d *AmoDal) Close(ctx context.Context) error { } return nil } + +type ClickHouseDAL struct { + conn *sql.DB +} + +func NewClickHouseDAL(ctx context.Context, cred string) (*ClickHouseDAL, error) { + conn, err := sql.Open("clickhouse", cred) + if err != nil { + return nil, fmt.Errorf("error open database connection: %w", err) + } + + timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + if err := conn.PingContext(timeoutCtx); err != nil { + return nil, fmt.Errorf("error ping database: %w", err) + } + + return &ClickHouseDAL{ + conn: conn, + }, nil +} + +func (d *ClickHouseDAL) Close(ctx context.Context) error { + return d.conn.Close() +} From 8eb1473d6472e474b8a8b37f5bf1df75a0ea083f Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 13 Jun 2024 15:39:18 +0300 Subject: [PATCH 14/59] init stats click repo --- dal/dal.go | 10 ++++++++-- repository/statistics/click_statistics.go | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 repository/statistics/click_statistics.go diff --git a/dal/dal.go b/dal/dal.go index 9ee3a72..9217abe 100644 --- a/dal/dal.go +++ b/dal/dal.go @@ -180,7 +180,8 @@ func (d *AmoDal) Close(ctx context.Context) error { } type ClickHouseDAL struct { - conn *sql.DB + conn *sql.DB + StatisticClickRepo *statistics.StatisticClick } func NewClickHouseDAL(ctx context.Context, cred string) (*ClickHouseDAL, error) { @@ -196,8 +197,13 @@ func NewClickHouseDAL(ctx context.Context, cred string) (*ClickHouseDAL, error) return nil, fmt.Errorf("error ping database: %w", err) } + statsClickRepo := statistics.NewClickStatistic(statistics.DepsClick{ + Conn: conn, + }) + return &ClickHouseDAL{ - conn: conn, + conn: conn, + StatisticClickRepo: statsClickRepo, }, nil } diff --git a/repository/statistics/click_statistics.go b/repository/statistics/click_statistics.go new file mode 100644 index 0000000..df2774b --- /dev/null +++ b/repository/statistics/click_statistics.go @@ -0,0 +1,17 @@ +package statistics + +import "database/sql" + +type DepsClick struct { + Conn *sql.DB +} + +type StatisticClick struct { + conn *sql.DB +} + +func NewClickStatistic(deps DepsClick) *StatisticClick { + return &StatisticClick{ + conn: deps.Conn, + } +} From ec052f5f152b7247d50e7afb3834cfae083d2ba6 Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 13 Jun 2024 18:37:58 +0300 Subject: [PATCH 15/59] add first try test with mat view --- dal/dal.go | 5 +- repository/statistics/click_statistics.go | 96 ++++++++++++++++++++++- 2 files changed, 97 insertions(+), 4 deletions(-) diff --git a/dal/dal.go b/dal/dal.go index 9217abe..cf2ad7d 100644 --- a/dal/dal.go +++ b/dal/dal.go @@ -197,9 +197,12 @@ func NewClickHouseDAL(ctx context.Context, cred string) (*ClickHouseDAL, error) return nil, fmt.Errorf("error ping database: %w", err) } - statsClickRepo := statistics.NewClickStatistic(statistics.DepsClick{ + statsClickRepo, err := statistics.NewClickStatistic(ctx, statistics.DepsClick{ Conn: conn, }) + if err != nil { + return nil, err + } return &ClickHouseDAL{ conn: conn, diff --git a/repository/statistics/click_statistics.go b/repository/statistics/click_statistics.go index df2774b..3622e56 100644 --- a/repository/statistics/click_statistics.go +++ b/repository/statistics/click_statistics.go @@ -1,6 +1,10 @@ package statistics -import "database/sql" +import ( + "context" + "database/sql" + "fmt" +) type DepsClick struct { Conn *sql.DB @@ -10,8 +14,94 @@ type StatisticClick struct { conn *sql.DB } -func NewClickStatistic(deps DepsClick) *StatisticClick { - return &StatisticClick{ +func NewClickStatistic(ctx context.Context, deps DepsClick) (*StatisticClick, error) { + s := &StatisticClick{ conn: deps.Conn, } + + err := s.checkMW(ctx) + if err != nil { + fmt.Println("error check material view existing") + return nil, err + } + + return s, nil +} + +func (s *StatisticClick) checkMW(ctx context.Context) error { + query := ` +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_last_answers_events +ENGINE = MergeTree() +PARTITION BY toStartOfDay(event_time) +ORDER BY (ctxsession, event_time) AS +SELECT + event_time,ctxsession,ctxquizid,ctxquestionid,ctxidint,message,keyos,keydevice,keydevicetype, + keybrowser,ctxuserip,ctxuserport,keydomain,keypath,ctxquiz,ctxreferrer +FROM statistics WHERE message IN ('InfoQuizOpen', 'InfoAnswer', 'InfoResult') + AND event_level = 'info' AND create_time = (SELECT max(create_time) FROM statistics AS inner_table + WHERE inner_table.ctxsession = statistics.ctxsession); +` + _, err := s.conn.ExecContext(ctx, query) + if err != nil { + return err + } + return nil +} + +type PipeLineStatsResp [][]struct { + Count int64 + QuestionID int64 +} + +func (s *StatisticClick) GetPipelinesStatistics(ctx context.Context, from uint64, to uint64) ([]PipeLineStatsResp, error) { + query := ` +SELECT + ctxsession,ctxquestionid,count(*) as session_count +FROM mv_last_answers_events WHERE event_time BETWEEN ? AND ? +GROUP BY ctxsession, ctxquestionid ORDER BY ctxsession, ctxquestionid +` + rows, err := s.conn.QueryContext(ctx, query, from, to) + if err != nil { + return nil, err + } + defer rows.Close() + + var pipelines []PipeLineStatsResp + var currentPipeline PipeLineStatsResp + var lastSession string + + for rows.Next() { + var session string + var questionID int64 + var count int64 + err := rows.Scan(&session, &questionID, &count) + if err != nil { + return nil, err + } + + // новая сессия - новая воронка + if session != lastSession && lastSession != "" { + pipelines = append(pipelines, currentPipeline) + currentPipeline = PipeLineStatsResp{} + } + + // текущая статистика в текущую воронку + currentPipeline = append(currentPipeline, []struct { + Count int64 + QuestionID int64 + }{{Count: count, QuestionID: questionID}}) + + lastSession = session + } + + // последня воронка если есть то добавляем + if len(currentPipeline) > 0 { + pipelines = append(pipelines, currentPipeline) + } + + if err := rows.Err(); err != nil { + return nil, err + } + + return pipelines, nil } From a0bae358aacd6b5f351c3fae45105a952026f228 Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 13 Jun 2024 18:44:06 +0300 Subject: [PATCH 16/59] add second try test with mat view --- repository/statistics/click_statistics.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/repository/statistics/click_statistics.go b/repository/statistics/click_statistics.go index 3622e56..19265ad 100644 --- a/repository/statistics/click_statistics.go +++ b/repository/statistics/click_statistics.go @@ -53,14 +53,14 @@ type PipeLineStatsResp [][]struct { QuestionID int64 } -func (s *StatisticClick) GetPipelinesStatistics(ctx context.Context, from uint64, to uint64) ([]PipeLineStatsResp, error) { +func (s *StatisticClick) GetPipelinesStatistics(ctx context.Context, quizID int64, from uint64, to uint64) ([]PipeLineStatsResp, error) { query := ` SELECT ctxsession,ctxquestionid,count(*) as session_count -FROM mv_last_answers_events WHERE event_time BETWEEN ? AND ? +FROM mv_last_answers_events WHERE ctxquizid = ? AND event_time BETWEEN ? AND ? GROUP BY ctxsession, ctxquestionid ORDER BY ctxsession, ctxquestionid ` - rows, err := s.conn.QueryContext(ctx, query, from, to) + rows, err := s.conn.QueryContext(ctx, query, quizID, from, to) if err != nil { return nil, err } From 46fac5b68652cdf4f24feba5dcf75f1982317f6e Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 13 Jun 2024 19:01:44 +0300 Subject: [PATCH 17/59] add third try test with mat view --- repository/statistics/click_statistics.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repository/statistics/click_statistics.go b/repository/statistics/click_statistics.go index 19265ad..1321fa6 100644 --- a/repository/statistics/click_statistics.go +++ b/repository/statistics/click_statistics.go @@ -28,6 +28,7 @@ func NewClickStatistic(ctx context.Context, deps DepsClick) (*StatisticClick, er return s, nil } +// todo toanaliz for keydevice,keydevicetype,keybrowser, func (s *StatisticClick) checkMW(ctx context.Context) error { query := ` CREATE MATERIALIZED VIEW IF NOT EXISTS mv_last_answers_events @@ -35,8 +36,7 @@ ENGINE = MergeTree() PARTITION BY toStartOfDay(event_time) ORDER BY (ctxsession, event_time) AS SELECT - event_time,ctxsession,ctxquizid,ctxquestionid,ctxidint,message,keyos,keydevice,keydevicetype, - keybrowser,ctxuserip,ctxuserport,keydomain,keypath,ctxquiz,ctxreferrer + event_time,ctxsession,ctxquizid,ctxquestionid,ctxidint,message,keyos,ctxuserip,ctxuserport,keydomain,keypath,ctxquiz,ctxreferrer FROM statistics WHERE message IN ('InfoQuizOpen', 'InfoAnswer', 'InfoResult') AND event_level = 'info' AND create_time = (SELECT max(create_time) FROM statistics AS inner_table WHERE inner_table.ctxsession = statistics.ctxsession); From a111b3bc3c439e1c84de446adcea6235bd932b03 Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 13 Jun 2024 22:03:41 +0300 Subject: [PATCH 18/59] update clickhouse query --- repository/statistics/click_statistics.go | 26 +++++++++++++---------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/repository/statistics/click_statistics.go b/repository/statistics/click_statistics.go index 1321fa6..1c7664b 100644 --- a/repository/statistics/click_statistics.go +++ b/repository/statistics/click_statistics.go @@ -48,12 +48,14 @@ FROM statistics WHERE message IN ('InfoQuizOpen', 'InfoAnswer', 'InfoResult') return nil } -type PipeLineStatsResp [][]struct { +type Statistic struct { Count int64 QuestionID int64 } -func (s *StatisticClick) GetPipelinesStatistics(ctx context.Context, quizID int64, from uint64, to uint64) ([]PipeLineStatsResp, error) { +type PipeLineStatsResp [][]Statistic + +func (s *StatisticClick) GetPipelinesStatistics(ctx context.Context, quizID int64, from uint64, to uint64) (PipeLineStatsResp, error) { query := ` SELECT ctxsession,ctxquestionid,count(*) as session_count @@ -66,8 +68,8 @@ GROUP BY ctxsession, ctxquestionid ORDER BY ctxsession, ctxquestionid } defer rows.Close() - var pipelines []PipeLineStatsResp - var currentPipeline PipeLineStatsResp + var pipelines PipeLineStatsResp + var currentPipeline []Statistic var lastSession string for rows.Next() { @@ -80,16 +82,18 @@ GROUP BY ctxsession, ctxquestionid ORDER BY ctxsession, ctxquestionid } // новая сессия - новая воронка - if session != lastSession && lastSession != "" { - pipelines = append(pipelines, currentPipeline) - currentPipeline = PipeLineStatsResp{} + if session != lastSession { + if lastSession != "" { + pipelines = append(pipelines, currentPipeline) + } + currentPipeline = []Statistic{} } // текущая статистика в текущую воронку - currentPipeline = append(currentPipeline, []struct { - Count int64 - QuestionID int64 - }{{Count: count, QuestionID: questionID}}) + currentPipeline = append(currentPipeline, Statistic{ + Count: count, + QuestionID: questionID, + }) lastSession = session } From 465cc5e6e0b8ae5e375a2af2351abbe67844988e Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 14 Jun 2024 12:01:37 +0300 Subject: [PATCH 19/59] del partition by days from mw --- repository/statistics/click_statistics.go | 1 - 1 file changed, 1 deletion(-) diff --git a/repository/statistics/click_statistics.go b/repository/statistics/click_statistics.go index 1c7664b..cdad23f 100644 --- a/repository/statistics/click_statistics.go +++ b/repository/statistics/click_statistics.go @@ -33,7 +33,6 @@ func (s *StatisticClick) checkMW(ctx context.Context) error { query := ` CREATE MATERIALIZED VIEW IF NOT EXISTS mv_last_answers_events ENGINE = MergeTree() -PARTITION BY toStartOfDay(event_time) ORDER BY (ctxsession, event_time) AS SELECT event_time,ctxsession,ctxquizid,ctxquestionid,ctxidint,message,keyos,ctxuserip,ctxuserport,keydomain,keypath,ctxquiz,ctxreferrer From d62217cf39170358423c55889cae448219f64f9a Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 14 Jun 2024 13:58:29 +0300 Subject: [PATCH 20/59] update mat view, now save only last answer for session --- repository/statistics/click_statistics.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/repository/statistics/click_statistics.go b/repository/statistics/click_statistics.go index cdad23f..d31df41 100644 --- a/repository/statistics/click_statistics.go +++ b/repository/statistics/click_statistics.go @@ -35,10 +35,15 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS mv_last_answers_events ENGINE = MergeTree() ORDER BY (ctxsession, event_time) AS SELECT - event_time,ctxsession,ctxquizid,ctxquestionid,ctxidint,message,keyos,ctxuserip,ctxuserport,keydomain,keypath,ctxquiz,ctxreferrer -FROM statistics WHERE message IN ('InfoQuizOpen', 'InfoAnswer', 'InfoResult') - AND event_level = 'info' AND create_time = (SELECT max(create_time) FROM statistics AS inner_table - WHERE inner_table.ctxsession = statistics.ctxsession); + event_time, ctxsession, ctxquizid, ctxquestionid, ctxidint, message, keyos, + ctxuserip, ctxuserport, keydomain, keypath, ctxquiz, ctxreferrer +FROM (SELECT + event_time, ctxsession, ctxquizid, ctxquestionid, ctxidint, message, keyos, + ctxuserip, ctxuserport, keydomain, keypath, ctxquiz, ctxreferrer, + row_number() OVER (PARTITION BY ctxsession ORDER BY create_time DESC) as row_num + FROM statistics + WHERE message IN ('InfoQuizOpen', 'InfoAnswer', 'InfoResult') AND event_level = 'info') AS sorted +WHERE row_num = 1; ` _, err := s.conn.ExecContext(ctx, query) if err != nil { From 2e031cecdee21c83e18c898ff687b1516ffebc44 Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 14 Jun 2024 14:22:18 +0300 Subject: [PATCH 21/59] update mat view --- repository/statistics/click_statistics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/statistics/click_statistics.go b/repository/statistics/click_statistics.go index d31df41..20eb3ce 100644 --- a/repository/statistics/click_statistics.go +++ b/repository/statistics/click_statistics.go @@ -40,7 +40,7 @@ SELECT FROM (SELECT event_time, ctxsession, ctxquizid, ctxquestionid, ctxidint, message, keyos, ctxuserip, ctxuserport, keydomain, keypath, ctxquiz, ctxreferrer, - row_number() OVER (PARTITION BY ctxsession ORDER BY create_time DESC) as row_num + row_number() OVER (PARTITION BY ctxsession ORDER BY event_time DESC) as row_num FROM statistics WHERE message IN ('InfoQuizOpen', 'InfoAnswer', 'InfoResult') AND event_level = 'info') AS sorted WHERE row_num = 1; From 4c6649bd30b31085b1a8cb3f6a257a5a26369a80 Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 14 Jun 2024 16:51:47 +0300 Subject: [PATCH 22/59] need rework full --- repository/statistics/click_statistics.go | 127 +++++++++++++++++----- 1 file changed, 97 insertions(+), 30 deletions(-) diff --git a/repository/statistics/click_statistics.go b/repository/statistics/click_statistics.go index 20eb3ce..a4afd17 100644 --- a/repository/statistics/click_statistics.go +++ b/repository/statistics/click_statistics.go @@ -29,6 +29,7 @@ func NewClickStatistic(ctx context.Context, deps DepsClick) (*StatisticClick, er } // todo toanaliz for keydevice,keydevicetype,keybrowser, +// todo получается пока какая то фигня func (s *StatisticClick) checkMW(ctx context.Context) error { query := ` CREATE MATERIALIZED VIEW IF NOT EXISTS mv_last_answers_events @@ -59,57 +60,123 @@ type Statistic struct { type PipeLineStatsResp [][]Statistic -func (s *StatisticClick) GetPipelinesStatistics(ctx context.Context, quizID int64, from uint64, to uint64) (PipeLineStatsResp, error) { +type FunnelData struct { + QuestionID int64 + Count int64 +} + +func (s *StatisticClick) funnelData(ctx context.Context, quizID int64, from uint64, to uint64) ([]FunnelData, error) { query := ` -SELECT - ctxsession,ctxquestionid,count(*) as session_count -FROM mv_last_answers_events WHERE ctxquizid = ? AND event_time BETWEEN ? AND ? -GROUP BY ctxsession, ctxquestionid ORDER BY ctxsession, ctxquestionid +SELECT ctxquestionid, uniqExact(ctxsession) AS count +FROM mv_last_answers_events +WHERE ctxquizid = ? AND event_time BETWEEN ? AND ? +GROUP BY ctxquestionid ` rows, err := s.conn.QueryContext(ctx, query, quizID, from, to) if err != nil { return nil, err } defer rows.Close() - - var pipelines PipeLineStatsResp - var currentPipeline []Statistic - var lastSession string - + var data []FunnelData for rows.Next() { - var session string var questionID int64 var count int64 - err := rows.Scan(&session, &questionID, &count) + + err := rows.Scan(&questionID, &count) if err != nil { return nil, err } - // новая сессия - новая воронка - if session != lastSession { - if lastSession != "" { - pipelines = append(pipelines, currentPipeline) - } - currentPipeline = []Statistic{} - } - - // текущая статистика в текущую воронку - currentPipeline = append(currentPipeline, Statistic{ - Count: count, + data = append(data, FunnelData{ QuestionID: questionID, + Count: count, }) - - lastSession = session - } - - // последня воронка если есть то добавляем - if len(currentPipeline) > 0 { - pipelines = append(pipelines, currentPipeline) } if err := rows.Err(); err != nil { return nil, err } + return data, nil +} + +// выборка вопросов с одной воронкой +func (s *StatisticClick) getOneFunnelQuestions(ctx context.Context, quizID int64, from uint64, to uint64) ([]int64, error) { + var result []int64 + + questionCounts, err := s.funnelData(ctx, quizID, from, to) + if err != nil { + return nil, err + } + + for _, qc := range questionCounts { + if qc.Count == 1 { + result = append(result, qc.QuestionID) + } + } + + return result, nil +} + +func (s *StatisticClick) GetPipelinesStatistics(ctx context.Context, quizID int64, from uint64, to uint64) (PipeLineStatsResp, error) { + var pipelines PipeLineStatsResp + idS, err := s.getOneFunnelQuestions(ctx, quizID, from, to) + if err != nil { + return nil, err + } + + query := ` +SELECT + ctxsession,ctxquestionid,count(*) as session_count +FROM mv_last_answers_events WHERE ctxquizid = ? AND ctxquestionid = ? AND event_time BETWEEN ? AND ? +GROUP BY ctxsession, ctxquestionid ORDER BY ctxsession, ctxquestionid +` + + for _, quiestionID := range idS { + rows, err := s.conn.QueryContext(ctx, query, quizID, quiestionID, from, to) + if err != nil { + return nil, err + } + defer rows.Close() + + var currentPipeline []Statistic + var lastSession string + + for rows.Next() { + var session string + var questionID int64 + var count int64 + err := rows.Scan(&session, &questionID, &count) + if err != nil { + return nil, err + } + + // новая сессия - новая воронка + if session != lastSession { + if lastSession != "" { + pipelines = append(pipelines, currentPipeline) + } + currentPipeline = []Statistic{} + } + + // текущая статистика в текущую воронку + currentPipeline = append(currentPipeline, Statistic{ + Count: count, + QuestionID: questionID, + }) + + lastSession = session + } + + // последня воронка если есть то добавляем + if len(currentPipeline) > 0 { + pipelines = append(pipelines, currentPipeline) + } + + if err := rows.Err(); err != nil { + return nil, err + } + } + return pipelines, nil } From fac78b12563be6c1e7d28341579c8126351aa71f Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 14 Jun 2024 21:06:01 +0300 Subject: [PATCH 23/59] need rework --- repository/statistics/click_statistics.go | 134 +++++++++++++--------- 1 file changed, 79 insertions(+), 55 deletions(-) diff --git a/repository/statistics/click_statistics.go b/repository/statistics/click_statistics.go index a4afd17..d71aec6 100644 --- a/repository/statistics/click_statistics.go +++ b/repository/statistics/click_statistics.go @@ -29,8 +29,10 @@ func NewClickStatistic(ctx context.Context, deps DepsClick) (*StatisticClick, er } // todo toanaliz for keydevice,keydevicetype,keybrowser, -// todo получается пока какая то фигня func (s *StatisticClick) checkMW(ctx context.Context) error { + // в мат вью получаем последний ссесионный координат, то есть тот момент на котором сессия в квизе завершилась + // не зависит какой это тип вопроса, страт, обычный, резулт, получаем всегда последний у сессии + // он определят конечное положение пользователя в опросе query := ` CREATE MATERIALIZED VIEW IF NOT EXISTS mv_last_answers_events ENGINE = MergeTree() @@ -61,57 +63,82 @@ type Statistic struct { type PipeLineStatsResp [][]Statistic type FunnelData struct { - QuestionID int64 - Count int64 + Session string + QuestionIDs []int64 } func (s *StatisticClick) funnelData(ctx context.Context, quizID int64, from uint64, to uint64) ([]FunnelData, error) { - query := ` -SELECT ctxquestionid, uniqExact(ctxsession) AS count -FROM mv_last_answers_events -WHERE ctxquizid = ? AND event_time BETWEEN ? AND ? -GROUP BY ctxquestionid -` - rows, err := s.conn.QueryContext(ctx, query, quizID, from, to) + // получили все уникальные сессии из мат вью + sessionQuery := ` + SELECT DISTINCT ctxsession + FROM mv_last_answers_events + WHERE ctxquizid = ? AND event_time BETWEEN ? AND ? + ` + sessionRows, err := s.conn.QueryContext(ctx, sessionQuery, quizID, from, to) if err != nil { return nil, err } - defer rows.Close() - var data []FunnelData - for rows.Next() { - var questionID int64 - var count int64 + defer sessionRows.Close() - err := rows.Scan(&questionID, &count) + var funnelData []FunnelData + for sessionRows.Next() { + var ctxsession string + err := sessionRows.Scan(&ctxsession) if err != nil { return nil, err } + // получаем вопросы уникальные, последние в рамках сессии + questionQuery := ` + SELECT DISTINCT ctxquestionid + FROM statistics WHERE ctxsession = ? ORDER BY event_time DESC; + ` + questionRows, err := s.conn.QueryContext(ctx, questionQuery, ctxsession) + if err != nil { + return nil, err + } + defer questionRows.Close() - data = append(data, FunnelData{ - QuestionID: questionID, - Count: count, + var questions []int64 + for questionRows.Next() { + var ctxquestionid int64 + err := questionRows.Scan(&ctxquestionid) + if err != nil { + return nil, err + } + questions = append(questions, ctxquestionid) + } + + fmt.Println("questions", questions) + + if err := questionRows.Err(); err != nil { + return nil, err + } + + funnelData = append(funnelData, FunnelData{ + Session: ctxsession, + QuestionIDs: questions, }) } - if err := rows.Err(); err != nil { + if err := sessionRows.Err(); err != nil { return nil, err } - return data, nil + return funnelData, nil } -// выборка вопросов с одной воронкой +// todo еще раз понять подумать осознать как подписывать воронку сейчас если массив соддержит 1 элемент это старт ответ func (s *StatisticClick) getOneFunnelQuestions(ctx context.Context, quizID int64, from uint64, to uint64) ([]int64, error) { var result []int64 - questionCounts, err := s.funnelData(ctx, quizID, from, to) + funnelData, err := s.funnelData(ctx, quizID, from, to) if err != nil { return nil, err } - for _, qc := range questionCounts { - if qc.Count == 1 { - result = append(result, qc.QuestionID) + for _, fd := range funnelData { + if len(fd.QuestionIDs) == 1 { + result = append(result, fd.QuestionIDs...) } } @@ -119,63 +146,60 @@ func (s *StatisticClick) getOneFunnelQuestions(ctx context.Context, quizID int64 } func (s *StatisticClick) GetPipelinesStatistics(ctx context.Context, quizID int64, from uint64, to uint64) (PipeLineStatsResp, error) { + // тут нужно подписать все вопросы, которые принадлежат окончанию сессии, то есть массив ответов до окончания сессии + // это делается для каждой из сессии которая имеется в мат вью, это отношение к результату которого достиг респондент + + // для того чтобы получить индентификатор воронки, то есть тот который последний этап прохождения был, то на чем все закончилось + // нужно из всех подписанных которые подписываются массивом и нужно выбрать из всех подписанных ответов на вопросы, выбрать те + // которые имеет всего один элемент в списке, то есть достижение никакого другого вопроса или результата не проходило через + // этот ответа кроме попытки достигнуть этотго оттвета или результата, это и являются конечные точки прохождения вопросов и от них считаются воронки и являются индентификаторами воронки + // именно те которые с массивом с 1 элементом + + // нужно вернуть в каждом списке пару, id вопроса и количество ответов на этот вопрос, count - уникальные сессии которые прошли через этот вопрос + var pipelines PipeLineStatsResp idS, err := s.getOneFunnelQuestions(ctx, quizID, from, to) if err != nil { return nil, err } - query := ` -SELECT - ctxsession,ctxquestionid,count(*) as session_count -FROM mv_last_answers_events WHERE ctxquizid = ? AND ctxquestionid = ? AND event_time BETWEEN ? AND ? -GROUP BY ctxsession, ctxquestionid ORDER BY ctxsession, ctxquestionid -` + for _, questionID := range idS { + query := ` + SELECT ctxquestionid, COUNT(DISTINCT ctxsession) AS session_count + FROM mv_last_answers_events + WHERE ctxquizid = ? AND ctxquestionid = ? AND event_time BETWEEN ? AND ? + GROUP BY ctxquestionid + ` - for _, quiestionID := range idS { - rows, err := s.conn.QueryContext(ctx, query, quizID, quiestionID, from, to) + rows, err := s.conn.QueryContext(ctx, query, quizID, questionID, from, to) if err != nil { return nil, err } defer rows.Close() var currentPipeline []Statistic - var lastSession string for rows.Next() { - var session string - var questionID int64 + var id int64 var count int64 - err := rows.Scan(&session, &questionID, &count) + err := rows.Scan(&id, &count) if err != nil { return nil, err } - // новая сессия - новая воронка - if session != lastSession { - if lastSession != "" { - pipelines = append(pipelines, currentPipeline) - } - currentPipeline = []Statistic{} - } - - // текущая статистика в текущую воронку currentPipeline = append(currentPipeline, Statistic{ + QuestionID: id, Count: count, - QuestionID: questionID, }) - - lastSession = session - } - - // последня воронка если есть то добавляем - if len(currentPipeline) > 0 { - pipelines = append(pipelines, currentPipeline) } if err := rows.Err(); err != nil { return nil, err } + + if len(currentPipeline) > 0 { + pipelines = append(pipelines, currentPipeline) + } } return pipelines, nil From 4a1517c84f3435b06d233939d4a424605f01fcb4 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Fri, 14 Jun 2024 21:36:58 +0000 Subject: [PATCH 24/59] Update queries.sql --- dal/db_query/queries.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dal/db_query/queries.sql b/dal/db_query/queries.sql index 0aae5f9..d512c5a 100644 --- a/dal/db_query/queries.sql +++ b/dal/db_query/queries.sql @@ -972,7 +972,7 @@ SELECT a.quiz_id,a.id,a.result,a.question_id,a.content,a.session, FROM answer a2 WHERE a2.start = true AND a2.session = a.session LIMIT 1) AS utm -,t.accesstoken,r.accountid,r.fieldsrule,r.tagstoadd,r.performerid,r.stepid,r.pipelineid,(SELECT u.name FROM usersAmo u WHERE u.AmoUserID = r.performerid) AS performer_name,u.subdomain,u.accountid,u.driveurl +,t.accesstoken,r.accountid,r.fieldsrule,r.tagstoadd,r.performerid,r.stepid,r.pipelineid,(SELECT u.name FROM usersAmo u WHERE u.AmoUserID = r.performerid AND u.deleted = false) AS performer_name,u.subdomain,u.accountid,u.driveurl FROM answer a INNER JOIN quiz q ON a.quiz_id = q.id LEFT JOIN amoCRMStatuses s ON a.id = s.AnswerID From 01abb94a5e3fa6853ae9dce41d811d8fce2865a0 Mon Sep 17 00:00:00 2001 From: skeris Date: Sat, 15 Jun 2024 00:57:55 +0300 Subject: [PATCH 25/59] D-- --- dal/sqlcgen/queries.sql.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dal/sqlcgen/queries.sql.go b/dal/sqlcgen/queries.sql.go index 1d80a1c..fb0e9be 100644 --- a/dal/sqlcgen/queries.sql.go +++ b/dal/sqlcgen/queries.sql.go @@ -2753,7 +2753,7 @@ SELECT a.quiz_id,a.id,a.result,a.question_id,a.content,a.session, FROM answer a2 WHERE a2.start = true AND a2.session = a.session LIMIT 1) AS utm -,t.accesstoken,r.accountid,r.fieldsrule,r.tagstoadd,r.performerid,r.stepid,r.pipelineid,(SELECT u.name FROM usersAmo u WHERE u.AmoUserID = r.performerid) AS performer_name,u.subdomain,u.accountid,u.driveurl +,t.accesstoken,r.accountid,r.fieldsrule,r.tagstoadd,r.performerid,r.stepid,r.pipelineid,(SELECT u.name FROM usersAmo u WHERE u.AmoUserID = r.performerid AND u.deleted = false) AS performer_name,u.subdomain,u.accountid,u.driveurl FROM answer a INNER JOIN quiz q ON a.quiz_id = q.id LEFT JOIN amoCRMStatuses s ON a.id = s.AnswerID From 9cb375b02385a7bbec265a369ffba11edc7e2475 Mon Sep 17 00:00:00 2001 From: Pavel Date: Mon, 17 Jun 2024 14:37:03 +0300 Subject: [PATCH 26/59] add reworked,need test and ome more checks --- repository/statistics/click_statistics.go | 214 ++++++++++------------ 1 file changed, 94 insertions(+), 120 deletions(-) diff --git a/repository/statistics/click_statistics.go b/repository/statistics/click_statistics.go index d71aec6..32b80db 100644 --- a/repository/statistics/click_statistics.go +++ b/repository/statistics/click_statistics.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "strings" ) type DepsClick struct { @@ -21,18 +22,14 @@ func NewClickStatistic(ctx context.Context, deps DepsClick) (*StatisticClick, er err := s.checkMW(ctx) if err != nil { - fmt.Println("error check material view existing") + fmt.Println("error check material view existing", err) return nil, err } return s, nil } -// todo toanaliz for keydevice,keydevicetype,keybrowser, func (s *StatisticClick) checkMW(ctx context.Context) error { - // в мат вью получаем последний ссесионный координат, то есть тот момент на котором сессия в квизе завершилась - // не зависит какой это тип вопроса, страт, обычный, резулт, получаем всегда последний у сессии - // он определят конечное положение пользователя в опросе query := ` CREATE MATERIALIZED VIEW IF NOT EXISTS mv_last_answers_events ENGINE = MergeTree() @@ -62,145 +59,122 @@ type Statistic struct { type PipeLineStatsResp [][]Statistic -type FunnelData struct { - Session string - QuestionIDs []int64 -} +func (s *StatisticClick) getFunnel(ctx context.Context, quizID int64, from uint64, to uint64) (map[int64][]string, error) { + // берем из матвью все что принадлежит quizID в указанном интервале времени + // выбираем самыые поздние по роу набер - 1 + // группируем по ид вопроса и для каждого ид вопроса формируем массив сессий которые были последнимим для этого вопроса + // выбираем только те где длина массива = 1 + query := ` + SELECT ctxquestionid, arrayJoin(endsession) AS session + FROM (SELECT ctxquestionid, groupArray(ctxsession) AS endsession + FROM (SELECT ctxsession,ctxquestionid,row_number() OVER (PARTITION BY ctxsession ORDER BY event_time DESC) AS row_num + FROM mv_last_answers_events WHERE ctxquizid = ? AND event_time BETWEEN ? AND ? + ) AS rows + WHERE row_num = 1 GROUP BY ctxquestionid + ) AS group_sessions WHERE length(endsession) = 1; + ` -func (s *StatisticClick) funnelData(ctx context.Context, quizID int64, from uint64, to uint64) ([]FunnelData, error) { - // получили все уникальные сессии из мат вью - sessionQuery := ` - SELECT DISTINCT ctxsession - FROM mv_last_answers_events - WHERE ctxquizid = ? AND event_time BETWEEN ? AND ? - ` - sessionRows, err := s.conn.QueryContext(ctx, sessionQuery, quizID, from, to) + rows, err := s.conn.QueryContext(ctx, query, quizID, from, to) if err != nil { return nil, err } - defer sessionRows.Close() + defer rows.Close() - var funnelData []FunnelData - for sessionRows.Next() { - var ctxsession string - err := sessionRows.Scan(&ctxsession) + funnel := make(map[int64][]string) + + for rows.Next() { + var questionID int64 + var sessionID string + err := rows.Scan(&questionID, &sessionID) if err != nil { return nil, err } - // получаем вопросы уникальные, последние в рамках сессии - questionQuery := ` - SELECT DISTINCT ctxquestionid - FROM statistics WHERE ctxsession = ? ORDER BY event_time DESC; - ` - questionRows, err := s.conn.QueryContext(ctx, questionQuery, ctxsession) - if err != nil { - return nil, err - } - defer questionRows.Close() - - var questions []int64 - for questionRows.Next() { - var ctxquestionid int64 - err := questionRows.Scan(&ctxquestionid) - if err != nil { - return nil, err - } - questions = append(questions, ctxquestionid) - } - - fmt.Println("questions", questions) - - if err := questionRows.Err(); err != nil { - return nil, err - } - - funnelData = append(funnelData, FunnelData{ - Session: ctxsession, - QuestionIDs: questions, - }) + funnel[questionID] = append(funnel[questionID], sessionID) } - if err := sessionRows.Err(); err != nil { + if err := rows.Err(); err != nil { return nil, err } - return funnelData, nil -} - -// todo еще раз понять подумать осознать как подписывать воронку сейчас если массив соддержит 1 элемент это старт ответ -func (s *StatisticClick) getOneFunnelQuestions(ctx context.Context, quizID int64, from uint64, to uint64) ([]int64, error) { - var result []int64 - - funnelData, err := s.funnelData(ctx, quizID, from, to) - if err != nil { - return nil, err - } - - for _, fd := range funnelData { - if len(fd.QuestionIDs) == 1 { - result = append(result, fd.QuestionIDs...) - } - } - - return result, nil + return funnel, nil } func (s *StatisticClick) GetPipelinesStatistics(ctx context.Context, quizID int64, from uint64, to uint64) (PipeLineStatsResp, error) { - // тут нужно подписать все вопросы, которые принадлежат окончанию сессии, то есть массив ответов до окончания сессии - // это делается для каждой из сессии которая имеется в мат вью, это отношение к результату которого достиг респондент - - // для того чтобы получить индентификатор воронки, то есть тот который последний этап прохождения был, то на чем все закончилось - // нужно из всех подписанных которые подписываются массивом и нужно выбрать из всех подписанных ответов на вопросы, выбрать те - // которые имеет всего один элемент в списке, то есть достижение никакого другого вопроса или результата не проходило через - // этот ответа кроме попытки достигнуть этотго оттвета или результата, это и являются конечные точки прохождения вопросов и от них считаются воронки и являются индентификаторами воронки - // именно те которые с массивом с 1 элементом - - // нужно вернуть в каждом списке пару, id вопроса и количество ответов на этот вопрос, count - уникальные сессии которые прошли через этот вопрос - var pipelines PipeLineStatsResp - idS, err := s.getOneFunnelQuestions(ctx, quizID, from, to) + + // получили id вопросов воронок где массив состоит из 1 элемента + funnel, err := s.getFunnel(ctx, quizID, from, to) + if err != nil { + return nil, err + } + var idS []int64 + for queID := range funnel { + idS = append(idS, queID) + } + + if len(idS) == 0 { + return nil, nil + } + // тут считаем количество ответов на эти вопросы по уникальным сессиям + sesCount, err := s.countSession(ctx, quizID, from, to, idS) if err != nil { return nil, err } - for _, questionID := range idS { - query := ` - SELECT ctxquestionid, COUNT(DISTINCT ctxsession) AS session_count - FROM mv_last_answers_events - WHERE ctxquizid = ? AND ctxquestionid = ? AND event_time BETWEEN ? AND ? - GROUP BY ctxquestionid - ` - - rows, err := s.conn.QueryContext(ctx, query, quizID, questionID, from, to) - if err != nil { - return nil, err - } - defer rows.Close() - - var currentPipeline []Statistic - - for rows.Next() { - var id int64 - var count int64 - err := rows.Scan(&id, &count) - if err != nil { - return nil, err + for questionID := range funnel { + if sessionCount, ok := sesCount[questionID]; ok { + pipeline := []Statistic{ + { + QuestionID: questionID, + Count: sessionCount, + }, } - - currentPipeline = append(currentPipeline, Statistic{ - QuestionID: id, - Count: count, - }) - } - - if err := rows.Err(); err != nil { - return nil, err - } - - if len(currentPipeline) > 0 { - pipelines = append(pipelines, currentPipeline) + pipelines = append(pipelines, pipeline) } } return pipelines, nil } + +func (s *StatisticClick) countSession(ctx context.Context, quizID int64, from uint64, to uint64, questionIDs []int64) (map[int64]int64, error) { + placeholders := make([]string, len(questionIDs)) + args := make([]interface{}, len(questionIDs)+3) + args[0] = quizID + for i, id := range questionIDs { + placeholders[i] = "?" + args[i+1] = id + } + args[len(args)-2] = from + args[len(args)-1] = to + + query := fmt.Sprintf(` + SELECT ctxquestionid, COUNT(DISTINCT ctxsession) AS session_count + FROM statistics + WHERE ctxquizid = ? AND ctxquestionid IN (%s) AND event_time BETWEEN ? AND ? + GROUP BY ctxquestionid; + `, strings.Join(placeholders, ",")) + + rows, err := s.conn.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + counts := make(map[int64]int64) + + for rows.Next() { + var questionID int64 + var count int64 + err := rows.Scan(&questionID, &count) + if err != nil { + return nil, err + } + counts[questionID] = count + } + + if err := rows.Err(); err != nil { + return nil, err + } + + return counts, nil +} From 2fd95695743c98b4c983bdae9aa46dfad76dd576 Mon Sep 17 00:00:00 2001 From: Pavel Date: Mon, 17 Jun 2024 17:33:23 +0300 Subject: [PATCH 27/59] add migrate file for delete html from answer content (copy from core branch - cleaningContent) --- dal/schema/000015_init.down.sql | 0 dal/schema/000015_init.up.sql | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 dal/schema/000015_init.down.sql create mode 100644 dal/schema/000015_init.up.sql diff --git a/dal/schema/000015_init.down.sql b/dal/schema/000015_init.down.sql new file mode 100644 index 0000000..e69de29 diff --git a/dal/schema/000015_init.up.sql b/dal/schema/000015_init.up.sql new file mode 100644 index 0000000..3985517 --- /dev/null +++ b/dal/schema/000015_init.up.sql @@ -0,0 +1,19 @@ +UPDATE answer +SET content = + CASE + WHEN content ~ '|' THEN + regexp_replace(content, '<\/?tr[^>]*>|<\/?td[^>]*>', '', 'g') + WHEN content ~ '[^<]+' THEN + regexp_replace(content, '([^<]+)', '\1', 'g') + WHEN content ~ ']*src="([^"]*)"[^>]*' THEN + regexp_replace(content, ']*src="\s*"[^>]*', '', 'g') + ELSE content + END; + +UPDATE answer +SET content = + CASE + WHEN content ~ ']*src=["'']?([^"''>]+)["'']?[^>]*>)', '\1\3', 'g') + ELSE content + END; From e1322a679dd10eb3bed8a0fedc56f283a544103b Mon Sep 17 00:00:00 2001 From: Pavel Date: Mon, 17 Jun 2024 18:18:47 +0300 Subject: [PATCH 28/59] separate ans repo for 2 repo, worker and answer, now worker if que type == fie getting file url and content for render img in mailbox --- dal/dal.go | 9 ++++- repository/answer/answer.go | 21 +++--------- repository/answer/worker_answer.go | 55 ++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 18 deletions(-) create mode 100644 repository/answer/worker_answer.go diff --git a/dal/dal.go b/dal/dal.go index 4d0c38d..d7ded03 100644 --- a/dal/dal.go +++ b/dal/dal.go @@ -31,6 +31,7 @@ type DAL struct { ResultRepo *result.ResultRepository WorkerRepo *workers.WorkerRepository StatisticsRepo *statistics.StatisticsRepository + WorkerAnsRepo *answer.WorkerAnswerRepository } func New(ctx context.Context, cred string, minioClient *minio.Client) (*DAL, error) { @@ -62,12 +63,17 @@ func New(ctx context.Context, cred string, minioClient *minio.Client) (*DAL, err } } - answerRepo := answer.NewAnswerRepository(answer.Deps{ + workerAnsRepo := answer.NewWorkerAnswerRepo(answer.Deps{ Queries: queries, Pool: pool, AnswerMinio: storerAnswer, }) + answerRepo := answer.NewAnswerRepository(answer.Deps{ + Queries: queries, + Pool: pool, + }) + questionRepo := question.NewQuestionRepository(question.Deps{ Queries: queries, Pool: pool, @@ -102,6 +108,7 @@ func New(ctx context.Context, cred string, minioClient *minio.Client) (*DAL, err ResultRepo: resultRepo, WorkerRepo: workerRepo, StatisticsRepo: statisticsRepo, + WorkerAnsRepo: workerAnsRepo, }, nil } diff --git a/repository/answer/answer.go b/repository/answer/answer.go index c473ce0..893dbb9 100644 --- a/repository/answer/answer.go +++ b/repository/answer/answer.go @@ -15,16 +15,14 @@ type Deps struct { } type AnswerRepository struct { - queries *sqlcgen.Queries - pool *sql.DB - answerMinio *StorerAnswer + queries *sqlcgen.Queries + pool *sql.DB } func NewAnswerRepository(deps Deps) *AnswerRepository { return &AnswerRepository{ - queries: deps.Queries, - pool: deps.Pool, - answerMinio: deps.AnswerMinio, + queries: deps.Queries, + pool: deps.Pool, } } @@ -111,17 +109,6 @@ func (r *AnswerRepository) GetAllAnswersByQuizID(ctx context.Context, session st } for _, row := range rows { - //todo тут забыл добавить проверку на то что minio !=nil - /*if row.Questiontype == model.TypeFile { - fmt.Println("GALL", row.Qid, row.QuestionID, row.Content) - fileURL, err := r.answerMinio.GetAnswerURL(ctx, row.Qid.UUID.String(), row.QuestionID, row.Content.String) - if err != nil { - fmt.Println("GetAnswerURL dal answer minio answer", err) - return nil, err - } - row.Content = sql.NullString{String: fmt.Sprintf("%s|%s", fileURL, row.Content.String), Valid: true} - }*/ - resultAnswer := model.ResultAnswer{ Content: row.Content.String, CreatedAt: row.CreatedAt.Time, diff --git a/repository/answer/worker_answer.go b/repository/answer/worker_answer.go new file mode 100644 index 0000000..3bd1995 --- /dev/null +++ b/repository/answer/worker_answer.go @@ -0,0 +1,55 @@ +package answer + +import ( + "context" + "database/sql" + "fmt" + "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/dal/sqlcgen" + "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/model" +) + +type WorkerAnswerRepository struct { + queries *sqlcgen.Queries + pool *sql.DB + answerMinio *StorerAnswer +} + +func NewWorkerAnswerRepo(deps Deps) *WorkerAnswerRepository { + return &WorkerAnswerRepository{ + queries: deps.Queries, + pool: deps.Pool, + answerMinio: deps.AnswerMinio, + } +} + +func (r *WorkerAnswerRepository) GetAllAnswersByQuizID(ctx context.Context, session string) ([]model.ResultAnswer, error) { + var results []model.ResultAnswer + + rows, err := r.queries.GetAllAnswersByQuizID(ctx, sql.NullString{String: session, Valid: true}) + if err != nil { + return nil, err + } + + for _, row := range rows { + if row.Questiontype == model.TypeFile && r.answerMinio != nil { + fmt.Println("GALL", row.Qid, row.QuestionID, row.Content) + fileURL, err := r.answerMinio.GetAnswerURL(ctx, row.Qid.UUID.String(), row.QuestionID, row.Content.String) + if err != nil { + fmt.Println("GetAnswerURL dal answer minio answer", err) + return nil, err + } + row.Content = sql.NullString{String: fmt.Sprintf("%s|%s", fileURL, row.Content.String), Valid: true} + } + + resultAnswer := model.ResultAnswer{ + Content: row.Content.String, + CreatedAt: row.CreatedAt.Time, + QuestionID: uint64(row.QuestionID), + AnswerID: uint64(row.ID), + } + + results = append(results, resultAnswer) + } + + return results, nil +} From 2835b15071214481ef9809752ec032b94c8717b2 Mon Sep 17 00:00:00 2001 From: Pavel Date: Tue, 18 Jun 2024 11:20:02 +0300 Subject: [PATCH 29/59] add populate to mv --- repository/statistics/click_statistics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/statistics/click_statistics.go b/repository/statistics/click_statistics.go index 32b80db..9580e42 100644 --- a/repository/statistics/click_statistics.go +++ b/repository/statistics/click_statistics.go @@ -33,7 +33,7 @@ func (s *StatisticClick) checkMW(ctx context.Context) error { query := ` CREATE MATERIALIZED VIEW IF NOT EXISTS mv_last_answers_events ENGINE = MergeTree() -ORDER BY (ctxsession, event_time) AS +ORDER BY (ctxsession, event_time) POPULATE AS SELECT event_time, ctxsession, ctxquizid, ctxquestionid, ctxidint, message, keyos, ctxuserip, ctxuserport, keydomain, keypath, ctxquiz, ctxreferrer From ac92017da11e01b79871563b821a0d5d48fadfa6 Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 19 Jun 2024 20:21:48 +0300 Subject: [PATCH 30/59] add coalesce for GettingAmoUsersTrueResults --- dal/db_query/queries.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dal/db_query/queries.sql b/dal/db_query/queries.sql index 5f23d2f..87c7fa0 100644 --- a/dal/db_query/queries.sql +++ b/dal/db_query/queries.sql @@ -971,10 +971,10 @@ VALUES ($1, $2, $3, $4, $5, $6); -- name: GettingAmoUsersTrueResults :many SELECT a.quiz_id,a.id,a.result,a.question_id,a.content,a.session, - (SELECT a2.utm + COALESCE((SELECT a2.utm FROM answer a2 WHERE a2.start = true AND a2.session = a.session - LIMIT 1) AS utm + LIMIT 1), '{}') AS utm ,t.accesstoken,r.accountid,r.fieldsrule,r.tagstoadd,r.performerid,r.stepid,r.pipelineid,(SELECT u.name FROM usersAmo u WHERE u.AmoUserID = r.performerid AND u.deleted = false) AS performer_name,u.subdomain,u.accountid,u.driveurl FROM answer a INNER JOIN quiz q ON a.quiz_id = q.id From 302d5e0ab543daf8ec59be7c51dffadb29792cb6 Mon Sep 17 00:00:00 2001 From: Pavel Date: Tue, 18 Jun 2024 15:33:51 +0300 Subject: [PATCH 31/59] update sqlc gen --- dal/sqlcgen/queries.sql.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dal/sqlcgen/queries.sql.go b/dal/sqlcgen/queries.sql.go index c134333..27770ba 100644 --- a/dal/sqlcgen/queries.sql.go +++ b/dal/sqlcgen/queries.sql.go @@ -2733,10 +2733,10 @@ func (q *Queries) GetUsersWithPagination(ctx context.Context, arg GetUsersWithPa const gettingAmoUsersTrueResults = `-- name: GettingAmoUsersTrueResults :many SELECT a.quiz_id,a.id,a.result,a.question_id,a.content,a.session, - (SELECT a2.utm + COALESCE((SELECT a2.utm FROM answer a2 WHERE a2.start = true AND a2.session = a.session - LIMIT 1) AS utm + LIMIT 1), '{}') AS utm ,t.accesstoken,r.accountid,r.fieldsrule,r.tagstoadd,r.performerid,r.stepid,r.pipelineid,(SELECT u.name FROM usersAmo u WHERE u.AmoUserID = r.performerid AND u.deleted = false) AS performer_name,u.subdomain,u.accountid,u.driveurl FROM answer a INNER JOIN quiz q ON a.quiz_id = q.id @@ -2759,7 +2759,7 @@ type GettingAmoUsersTrueResultsRow struct { QuestionID int64 `db:"question_id" json:"question_id"` Content sql.NullString `db:"content" json:"content"` Session sql.NullString `db:"session" json:"session"` - Utm json.RawMessage `db:"utm" json:"utm"` + Utm interface{} `db:"utm" json:"utm"` Accesstoken string `db:"accesstoken" json:"accesstoken"` Accountid int32 `db:"accountid" json:"accountid"` Fieldsrule json.RawMessage `db:"fieldsrule" json:"fieldsrule"` From 7b56647a864fd98d4149cb05c37dc84cfae82216 Mon Sep 17 00:00:00 2001 From: Pavel Date: Tue, 18 Jun 2024 21:19:49 +0300 Subject: [PATCH 32/59] add todo --- repository/statistics/click_statistics.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/repository/statistics/click_statistics.go b/repository/statistics/click_statistics.go index 9580e42..a786ea4 100644 --- a/repository/statistics/click_statistics.go +++ b/repository/statistics/click_statistics.go @@ -59,6 +59,24 @@ type Statistic struct { type PipeLineStatsResp [][]Statistic +// todo нужно исключить множества из подмножеств, пока что получаются все уникальные векторы респондентов по опросу +// пример: +//"[0, 116783, 116810]" +//"[0, 116783, 116798]" +//"[0, 116783, 116798, 116831]" +//"[0, 116783, 116810, 116849]" +//[0] +//"[0, 116783]" +//"[0, 116783, 116810, 116843]" + +//SELECT DISTINCT final FROM ( +//SELECT groupArray(ctxquestionid) AS final FROM (SELECT DISTINCT f.ctxsession, a.ctxquestionid +//FROM (SELECT ctxsession, max(event_time) AS max_time +//FROM statistics WHERE ctxquizid = 26276 GROUP BY ctxsession ) f +//JOIN ( SELECT ctxsession, ctxquestionid, event_time +//FROM statistics WHERE ctxquizid = 26276 ) a ON f.ctxsession = a.ctxsession) +//GROUP BY ctxsession); + func (s *StatisticClick) getFunnel(ctx context.Context, quizID int64, from uint64, to uint64) (map[int64][]string, error) { // берем из матвью все что принадлежит quizID в указанном интервале времени // выбираем самыые поздние по роу набер - 1 From 4b23ef459808486c6c82e6b042f4e09c5f4fd0d1 Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 19 Jun 2024 17:29:20 +0300 Subject: [PATCH 33/59] reworked --- repository/statistics/click_statistics.go | 72 ++++++++++++++--------- utils/binary_search.go | 20 +++++++ 2 files changed, 64 insertions(+), 28 deletions(-) create mode 100644 utils/binary_search.go diff --git a/repository/statistics/click_statistics.go b/repository/statistics/click_statistics.go index a786ea4..76c980c 100644 --- a/repository/statistics/click_statistics.go +++ b/repository/statistics/click_statistics.go @@ -4,6 +4,8 @@ import ( "context" "database/sql" "fmt" + "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/utils" + "sort" "strings" ) @@ -59,7 +61,6 @@ type Statistic struct { type PipeLineStatsResp [][]Statistic -// todo нужно исключить множества из подмножеств, пока что получаются все уникальные векторы респондентов по опросу // пример: //"[0, 116783, 116810]" //"[0, 116783, 116798]" @@ -69,27 +70,15 @@ type PipeLineStatsResp [][]Statistic //"[0, 116783]" //"[0, 116783, 116810, 116843]" -//SELECT DISTINCT final FROM ( -//SELECT groupArray(ctxquestionid) AS final FROM (SELECT DISTINCT f.ctxsession, a.ctxquestionid -//FROM (SELECT ctxsession, max(event_time) AS max_time -//FROM statistics WHERE ctxquizid = 26276 GROUP BY ctxsession ) f -//JOIN ( SELECT ctxsession, ctxquestionid, event_time -//FROM statistics WHERE ctxquizid = 26276 ) a ON f.ctxsession = a.ctxsession) -//GROUP BY ctxsession); +//SELECT DISTINCT last_que, reversed +//FROM ( SELECT groupArray(ctxquestionid) AS reversed, arraySlice(arrayReverse(groupArray(ctxquestionid)), 1, 1)[1] AS last_que +//FROM statistics WHERE ctxquizid = 26276 GROUP BY ctxsession ) AS sub; -func (s *StatisticClick) getFunnel(ctx context.Context, quizID int64, from uint64, to uint64) (map[int64][]string, error) { - // берем из матвью все что принадлежит quizID в указанном интервале времени - // выбираем самыые поздние по роу набер - 1 - // группируем по ид вопроса и для каждого ид вопроса формируем массив сессий которые были последнимим для этого вопроса - // выбираем только те где длина массива = 1 +func (s *StatisticClick) getFunnel(ctx context.Context, quizID int64, from uint64, to uint64) (map[int64][]int64, error) { query := ` - SELECT ctxquestionid, arrayJoin(endsession) AS session - FROM (SELECT ctxquestionid, groupArray(ctxsession) AS endsession - FROM (SELECT ctxsession,ctxquestionid,row_number() OVER (PARTITION BY ctxsession ORDER BY event_time DESC) AS row_num - FROM mv_last_answers_events WHERE ctxquizid = ? AND event_time BETWEEN ? AND ? - ) AS rows - WHERE row_num = 1 GROUP BY ctxquestionid - ) AS group_sessions WHERE length(endsession) = 1; + SELECT DISTINCT last_que, reversed + FROM ( SELECT groupArray(ctxquestionid) AS reversed, arraySlice(arrayReverse(groupArray(ctxquestionid)), 1, 1)[1] AS last_que + FROM statistics WHERE ctxquizid = ? AND event_time BETWEEN ? AND ? GROUP BY ctxsession ) AS sub; ` rows, err := s.conn.QueryContext(ctx, query, quizID, from, to) @@ -98,23 +87,50 @@ func (s *StatisticClick) getFunnel(ctx context.Context, quizID int64, from uint6 } defer rows.Close() - funnel := make(map[int64][]string) - + funnel := make(map[int64][]int64) for rows.Next() { - var questionID int64 - var sessionID string - err := rows.Scan(&questionID, &sessionID) - if err != nil { + var lastQue int64 + var reversed []int64 + if err := rows.Scan(&lastQue, &reversed); err != nil { return nil, err } - funnel[questionID] = append(funnel[questionID], sessionID) + funnel[lastQue] = reversed } if err := rows.Err(); err != nil { return nil, err } - return funnel, nil + result := make(map[int64][]int64) + keys := make([]int64, 0, len(funnel)) + for key := range funnel { + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { + return keys[i] < keys[j] + }) + + for _, lastQue := range keys { + reversed := funnel[lastQue] + found := false + + for _, otherLastQue := range keys { + if otherLastQue != lastQue { + otherReversed := funnel[otherLastQue] + index := utils.BinarySearch(lastQue, otherReversed) + if index { + found = true + break + } + } + } + + if !found { + result[lastQue] = reversed + } + } + + return result, nil } func (s *StatisticClick) GetPipelinesStatistics(ctx context.Context, quizID int64, from uint64, to uint64) (PipeLineStatsResp, error) { diff --git a/utils/binary_search.go b/utils/binary_search.go new file mode 100644 index 0000000..8e91039 --- /dev/null +++ b/utils/binary_search.go @@ -0,0 +1,20 @@ +package utils + +func BinarySearch(target int64, array []int64) bool { + left := 0 + right := len(array) - 1 + + for left <= right { + mid := (left + right) / 2 + + if array[mid] == target { + return true + } else if array[mid] < target { + left = mid + 1 + } else if array[mid] > target { + right = mid - 1 + } + } + + return false +} From ec12f439df5d43a3de764b471a6cc4a13440f825 Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 19 Jun 2024 17:29:56 +0300 Subject: [PATCH 34/59] reworked --- repository/statistics/click_statistics.go | 1 + 1 file changed, 1 insertion(+) diff --git a/repository/statistics/click_statistics.go b/repository/statistics/click_statistics.go index 76c980c..40f2aa9 100644 --- a/repository/statistics/click_statistics.go +++ b/repository/statistics/click_statistics.go @@ -141,6 +141,7 @@ func (s *StatisticClick) GetPipelinesStatistics(ctx context.Context, quizID int6 if err != nil { return nil, err } + fmt.Println(funnel) var idS []int64 for queID := range funnel { idS = append(idS, queID) From 7b29c987a94dc68bcb07346a2d4b9c542a2a293d Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 19 Jun 2024 17:39:51 +0300 Subject: [PATCH 35/59] reworked --- dal/dal.go | 1 + go.mod | 2 ++ go.sum | 10 ++++++++++ 3 files changed, 13 insertions(+) diff --git a/dal/dal.go b/dal/dal.go index cf2ad7d..39a2c95 100644 --- a/dal/dal.go +++ b/dal/dal.go @@ -6,6 +6,7 @@ import ( _ "embed" "errors" "fmt" + _ "github.com/ClickHouse/clickhouse-go" _ "github.com/lib/pq" "github.com/minio/minio-go/v7" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/dal/sqlcgen" diff --git a/go.mod b/go.mod index b30afc3..41af95d 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,9 @@ require ( ) require ( + github.com/ClickHouse/clickhouse-go v1.5.4 // indirect github.com/andybalholm/brotli v1.0.5 // indirect + github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/google/go-cmp v0.6.0 // indirect diff --git a/go.sum b/go.sum index e82a594..5bcd421 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,18 @@ +github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0= +github.com/ClickHouse/clickhouse-go v1.5.4/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= +github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg= +github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/gofiber/fiber/v2 v2.52.0 h1:S+qXi7y+/Pgvqq4DrSmREGiFwtB7Bu6+QFLuIHYw/UE= github.com/gofiber/fiber/v2 v2.52.0/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ= github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw= @@ -20,6 +26,7 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= @@ -32,6 +39,7 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -41,6 +49,7 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.69 h1:l8AnsQFyY1xiwa/DaQskY4NXSLA2yrGsW5iD9nRPVS0= @@ -52,6 +61,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= From cf45d440fcd5fc65b94c1f42e537567f03426b5e Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 19 Jun 2024 17:51:34 +0300 Subject: [PATCH 36/59] reworked --- repository/statistics/click_statistics.go | 41 ++++++++--------------- 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/repository/statistics/click_statistics.go b/repository/statistics/click_statistics.go index 40f2aa9..52b9025 100644 --- a/repository/statistics/click_statistics.go +++ b/repository/statistics/click_statistics.go @@ -59,7 +59,7 @@ type Statistic struct { QuestionID int64 } -type PipeLineStatsResp [][]Statistic +type PipeLineStatsResp map[int64][]Statistic // пример: //"[0, 116783, 116810]" @@ -77,7 +77,7 @@ type PipeLineStatsResp [][]Statistic func (s *StatisticClick) getFunnel(ctx context.Context, quizID int64, from uint64, to uint64) (map[int64][]int64, error) { query := ` SELECT DISTINCT last_que, reversed - FROM ( SELECT groupArray(ctxquestionid) AS reversed, arraySlice(arrayReverse(groupArray(ctxquestionid)), 1, 1)[1] AS last_que + FROM ( SELECT groupUniqArray(ctxquestionid) AS reversed, arraySlice(arrayReverse(groupArray(ctxquestionid)), 1, 1)[1] AS last_que FROM statistics WHERE ctxquizid = ? AND event_time BETWEEN ? AND ? GROUP BY ctxsession ) AS sub; ` @@ -134,40 +134,27 @@ func (s *StatisticClick) getFunnel(ctx context.Context, quizID int64, from uint6 } func (s *StatisticClick) GetPipelinesStatistics(ctx context.Context, quizID int64, from uint64, to uint64) (PipeLineStatsResp, error) { - var pipelines PipeLineStatsResp + pipelines := make(PipeLineStatsResp) - // получили id вопросов воронок где массив состоит из 1 элемента funnel, err := s.getFunnel(ctx, quizID, from, to) if err != nil { return nil, err } - fmt.Println(funnel) - var idS []int64 - for queID := range funnel { - idS = append(idS, queID) - } - - if len(idS) == 0 { - return nil, nil - } - // тут считаем количество ответов на эти вопросы по уникальным сессиям - sesCount, err := s.countSession(ctx, quizID, from, to, idS) - if err != nil { - return nil, err - } - - for questionID := range funnel { - if sessionCount, ok := sesCount[questionID]; ok { - pipeline := []Statistic{ - { - QuestionID: questionID, + for lastQue, idS := range funnel { + sesCount, err := s.countSession(ctx, quizID, from, to, idS) + if err != nil { + return nil, err + } + for _, queID := range idS { + if sessionCount, ok := sesCount[queID]; ok { + pipeline := Statistic{ + QuestionID: queID, Count: sessionCount, - }, + } + pipelines[lastQue] = append(pipelines[lastQue], pipeline) } - pipelines = append(pipelines, pipeline) } } - return pipelines, nil } From c1e389efef031efbc445411c3c2ae312e99d854a Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 19 Jun 2024 18:06:59 +0300 Subject: [PATCH 37/59] add sort slice prepare for binary search --- repository/statistics/click_statistics.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/repository/statistics/click_statistics.go b/repository/statistics/click_statistics.go index 52b9025..e2b0dcf 100644 --- a/repository/statistics/click_statistics.go +++ b/repository/statistics/click_statistics.go @@ -117,6 +117,9 @@ func (s *StatisticClick) getFunnel(ctx context.Context, quizID int64, from uint6 for _, otherLastQue := range keys { if otherLastQue != lastQue { otherReversed := funnel[otherLastQue] + sort.Slice(otherReversed, func(i, j int) bool { + return otherReversed[i] < otherReversed[j] + }) index := utils.BinarySearch(lastQue, otherReversed) if index { found = true From 13260731828d7521cf84a23257f2d05f7b471502 Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 19 Jun 2024 20:27:59 +0300 Subject: [PATCH 38/59] update sqlc gen --- dal/db_query/queries.sql | 2 +- dal/sqlcgen/queries.sql.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dal/db_query/queries.sql b/dal/db_query/queries.sql index 87c7fa0..98d8247 100644 --- a/dal/db_query/queries.sql +++ b/dal/db_query/queries.sql @@ -974,7 +974,7 @@ SELECT a.quiz_id,a.id,a.result,a.question_id,a.content,a.session, COALESCE((SELECT a2.utm FROM answer a2 WHERE a2.start = true AND a2.session = a.session - LIMIT 1), '{}') AS utm + LIMIT 1), '{}'::jsonb) AS utm ,t.accesstoken,r.accountid,r.fieldsrule,r.tagstoadd,r.performerid,r.stepid,r.pipelineid,(SELECT u.name FROM usersAmo u WHERE u.AmoUserID = r.performerid AND u.deleted = false) AS performer_name,u.subdomain,u.accountid,u.driveurl FROM answer a INNER JOIN quiz q ON a.quiz_id = q.id diff --git a/dal/sqlcgen/queries.sql.go b/dal/sqlcgen/queries.sql.go index 27770ba..2315192 100644 --- a/dal/sqlcgen/queries.sql.go +++ b/dal/sqlcgen/queries.sql.go @@ -2736,7 +2736,7 @@ SELECT a.quiz_id,a.id,a.result,a.question_id,a.content,a.session, COALESCE((SELECT a2.utm FROM answer a2 WHERE a2.start = true AND a2.session = a.session - LIMIT 1), '{}') AS utm + LIMIT 1), '{}'::jsonb) AS utm ,t.accesstoken,r.accountid,r.fieldsrule,r.tagstoadd,r.performerid,r.stepid,r.pipelineid,(SELECT u.name FROM usersAmo u WHERE u.AmoUserID = r.performerid AND u.deleted = false) AS performer_name,u.subdomain,u.accountid,u.driveurl FROM answer a INNER JOIN quiz q ON a.quiz_id = q.id From e753438e8c63cd1a96414c816d54fc7c4638306d Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 19 Jun 2024 20:37:23 +0300 Subject: [PATCH 39/59] interface to type []byte --- repository/amo/amo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/amo/amo.go b/repository/amo/amo.go index e6c99e1..c00080d 100644 --- a/repository/amo/amo.go +++ b/repository/amo/amo.go @@ -1007,7 +1007,7 @@ func (r *AmoRepository) GettingAmoUsersTrueResults(ctx context.Context) ([]model } var utm model.UTMSavingMap - err = json.Unmarshal(row.Utm, &utm) + err = json.Unmarshal(row.Utm.([]byte), &utm) if err != nil { return nil, err } From 0116666117f63bcd8a57d2cb0481515241395678 Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 20 Jun 2024 18:37:25 +0300 Subject: [PATCH 40/59] add migrate file for save amo contact data --- dal/schema/000016_init.down.sql | 1 + dal/schema/000016_init.up.sql | 6 ++++++ model/amo.go | 8 ++++++++ sqlc.yaml | 2 ++ 4 files changed, 17 insertions(+) create mode 100644 dal/schema/000016_init.down.sql create mode 100644 dal/schema/000016_init.up.sql diff --git a/dal/schema/000016_init.down.sql b/dal/schema/000016_init.down.sql new file mode 100644 index 0000000..28b2333 --- /dev/null +++ b/dal/schema/000016_init.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS amoContact; \ No newline at end of file diff --git a/dal/schema/000016_init.up.sql b/dal/schema/000016_init.up.sql new file mode 100644 index 0000000..a4866d0 --- /dev/null +++ b/dal/schema/000016_init.up.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS amoContact ( + ID BIGSERIAL UNIQUE NOT NULL PRIMARY KEY, + AccountID INT NOT NULL, -- ID "компании" в амо + AmoID INT NOT NULL, -- ID контакта в амо + Field text NOT NULL DEFAULT '' -- значение чего то email? phone? etc +) \ No newline at end of file diff --git a/model/amo.go b/model/amo.go index bdcbcf1..5a0dae0 100644 --- a/model/amo.go +++ b/model/amo.go @@ -288,3 +288,11 @@ type AmoUsersTrueResults struct { QuizAccountID string DriveURL string } + +// возможно стоит добавить enum? тип ContactQuizConfig уже есть +type ContactAmo struct { + ID int64 + AccountID int32 // id аккаунта в амо к которому привязан контакт + AmoID int32 // id контакта в амо + Field string // значение поля +} diff --git a/sqlc.yaml b/sqlc.yaml index d367be4..07bb059 100644 --- a/sqlc.yaml +++ b/sqlc.yaml @@ -32,6 +32,8 @@ packages: - "./dal/schema/000013_init.down.sql" - "./dal/schema/000014_init.up.sql" - "./dal/schema/000014_init.down.sql" + - "./dal/schema/000016_init.up.sql" + - "./dal/schema/000016_init.down.sql" engine: "postgresql" emit_json_tags: true emit_db_tags: true From ced29eec4410da2e4b2e81755f6bc57e7ee93f36 Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 19 Jun 2024 20:44:42 +0300 Subject: [PATCH 41/59] update sqlc gen --- dal/sqlcgen/models.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dal/sqlcgen/models.go b/dal/sqlcgen/models.go index c6d928d..90a4ade 100644 --- a/dal/sqlcgen/models.go +++ b/dal/sqlcgen/models.go @@ -32,6 +32,13 @@ type Accountsamo struct { Driveurl string `db:"driveurl" json:"driveurl"` } +type Amocontact struct { + ID int64 `db:"id" json:"id"` + Accountid int32 `db:"accountid" json:"accountid"` + Amoid int32 `db:"amoid" json:"amoid"` + Field string `db:"field" json:"field"` +} + type Amocrmstatus struct { ID int64 `db:"id" json:"id"` Accountid int32 `db:"accountid" json:"accountid"` From dae8abec1830784856329f8f2ddc3e2172dcf810 Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 21 Jun 2024 12:36:21 +0300 Subject: [PATCH 42/59] add queries for work with contacts --- dal/db_query/queries.sql | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dal/db_query/queries.sql b/dal/db_query/queries.sql index 98d8247..2849355 100644 --- a/dal/db_query/queries.sql +++ b/dal/db_query/queries.sql @@ -1045,3 +1045,13 @@ WHERE AccountID = $1 AND Deleted = false; UPDATE privileges p SET amount = amount - 1 FROM account a WHERE p.account_id = a.id AND a.user_id = $1 AND p.privilegeID = $2 AND p.amount > 0 RETURNING p.id, p.privilegeID, p.account_id, p.privilege_name, p.amount, p.created_at; + +-- name: GetExistingContactAmo :one +SELECT ID, AccountID, AmoID, Field +FROM amoContact WHERE AccountID = $1 AND (Field = $2 OR Field = $3); + +-- name: InsertContactAmo :one +INSERT INTO amoContact (AccountID, AmoID, Field) VALUES ($1, $2, $3) RETURNING AmoID; + +-- name: UpdateAmoAccount :exec +UPDATE amoContact SET Field = $1 WHERE ID = $2; \ No newline at end of file From 1e97acc6a9bea55114ad30b019cb67483b6f7f09 Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 19 Jun 2024 20:47:28 +0300 Subject: [PATCH 43/59] update sqlc gen --- dal/db_query/queries.sql | 4 +-- dal/sqlcgen/queries.sql.go | 54 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/dal/db_query/queries.sql b/dal/db_query/queries.sql index 2849355..13dfcee 100644 --- a/dal/db_query/queries.sql +++ b/dal/db_query/queries.sql @@ -1053,5 +1053,5 @@ FROM amoContact WHERE AccountID = $1 AND (Field = $2 OR Field = $3); -- name: InsertContactAmo :one INSERT INTO amoContact (AccountID, AmoID, Field) VALUES ($1, $2, $3) RETURNING AmoID; --- name: UpdateAmoAccount :exec -UPDATE amoContact SET Field = $1 WHERE ID = $2; \ No newline at end of file +-- name: UpdateAmoContact :exec +UPDATE amoContact SET Field = $1 WHERE ID = $2; diff --git a/dal/sqlcgen/queries.sql.go b/dal/sqlcgen/queries.sql.go index 2315192..f02bc72 100644 --- a/dal/sqlcgen/queries.sql.go +++ b/dal/sqlcgen/queries.sql.go @@ -1449,6 +1449,29 @@ func (q *Queries) GetCurrentCompany(ctx context.Context, accountid string) (Acco return i, err } +const getExistingContactAmo = `-- name: GetExistingContactAmo :one +SELECT ID, AccountID, AmoID, Field +FROM amoContact WHERE AccountID = $1 AND (Field = $2 OR Field = $3) +` + +type GetExistingContactAmoParams struct { + Accountid int32 `db:"accountid" json:"accountid"` + Field string `db:"field" json:"field"` + Field_2 string `db:"field_2" json:"field_2"` +} + +func (q *Queries) GetExistingContactAmo(ctx context.Context, arg GetExistingContactAmoParams) (Amocontact, error) { + row := q.db.QueryRowContext(ctx, getExistingContactAmo, arg.Accountid, arg.Field, arg.Field_2) + var i Amocontact + err := row.Scan( + &i.ID, + &i.Accountid, + &i.Amoid, + &i.Field, + ) + return i, err +} + const getExpiredCountPrivilege = `-- name: GetExpiredCountPrivilege :many SELECT p.id, p.privilegeID, p.privilege_name, p.amount, p.created_at, a.user_id FROM privileges p @@ -2893,6 +2916,23 @@ func (q *Queries) InsertAnswers(ctx context.Context, arg InsertAnswersParams) (A return i, err } +const insertContactAmo = `-- name: InsertContactAmo :one +INSERT INTO amoContact (AccountID, AmoID, Field) VALUES ($1, $2, $3) RETURNING AmoID +` + +type InsertContactAmoParams struct { + Accountid int32 `db:"accountid" json:"accountid"` + Amoid int32 `db:"amoid" json:"amoid"` + Field string `db:"field" json:"field"` +} + +func (q *Queries) InsertContactAmo(ctx context.Context, arg InsertContactAmoParams) (int32, error) { + row := q.db.QueryRowContext(ctx, insertContactAmo, arg.Accountid, arg.Amoid, arg.Field) + var amoid int32 + err := row.Scan(&amoid) + return amoid, err +} + const insertPrivilege = `-- name: InsertPrivilege :exec INSERT INTO privileges (privilegeID, account_id, privilege_name, amount, created_at) VALUES ($1, $2, $3, $4, $5) ` @@ -3450,6 +3490,20 @@ func (q *Queries) UpdateAmoAccountUser(ctx context.Context, arg UpdateAmoAccount return err } +const updateAmoContact = `-- name: UpdateAmoContact :exec +UPDATE amoContact SET Field = $1 WHERE ID = $2 +` + +type UpdateAmoContactParams struct { + Field string `db:"field" json:"field"` + ID int64 `db:"id" json:"id"` +} + +func (q *Queries) UpdateAmoContact(ctx context.Context, arg UpdateAmoContactParams) error { + _, err := q.db.ExecContext(ctx, updateAmoContact, arg.Field, arg.ID) + return err +} + const updateFieldRules = `-- name: UpdateFieldRules :exec UPDATE rules SET FieldsRule = $1 WHERE AccountID = (SELECT AmoID FROM accountsAmo WHERE accountsAmo.AccountID = $2 AND accountsAmo.Deleted = false) AND QuizID = $3 AND Deleted = false From d2537b8ba40ef00faae1ffb47cb189c762d87a1b Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 20 Jun 2024 12:37:53 +0300 Subject: [PATCH 44/59] update mat view --- repository/statistics/click_statistics.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/repository/statistics/click_statistics.go b/repository/statistics/click_statistics.go index e2b0dcf..7bd3451 100644 --- a/repository/statistics/click_statistics.go +++ b/repository/statistics/click_statistics.go @@ -37,15 +37,10 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS mv_last_answers_events ENGINE = MergeTree() ORDER BY (ctxsession, event_time) POPULATE AS SELECT - event_time, ctxsession, ctxquizid, ctxquestionid, ctxidint, message, keyos, - ctxuserip, ctxuserport, keydomain, keypath, ctxquiz, ctxreferrer -FROM (SELECT - event_time, ctxsession, ctxquizid, ctxquestionid, ctxidint, message, keyos, - ctxuserip, ctxuserport, keydomain, keypath, ctxquiz, ctxreferrer, - row_number() OVER (PARTITION BY ctxsession ORDER BY event_time DESC) as row_num - FROM statistics - WHERE message IN ('InfoQuizOpen', 'InfoAnswer', 'InfoResult') AND event_level = 'info') AS sorted -WHERE row_num = 1; + event_time, ctxsession, ctxquizid, ctxquestionid, ctxidint, message,ctxquiz +FROM statistics +WHERE message IN ('InfoQuizOpen', 'InfoAnswer', 'InfoResult') AND event_level = 'info'; +; ` _, err := s.conn.ExecContext(ctx, query) if err != nil { @@ -78,7 +73,7 @@ func (s *StatisticClick) getFunnel(ctx context.Context, quizID int64, from uint6 query := ` SELECT DISTINCT last_que, reversed FROM ( SELECT groupUniqArray(ctxquestionid) AS reversed, arraySlice(arrayReverse(groupArray(ctxquestionid)), 1, 1)[1] AS last_que - FROM statistics WHERE ctxquizid = ? AND event_time BETWEEN ? AND ? GROUP BY ctxsession ) AS sub; + FROM mv_last_answers_events WHERE ctxquizid = ? AND event_time BETWEEN ? AND ? GROUP BY ctxsession ) AS sub; ` rows, err := s.conn.QueryContext(ctx, query, quizID, from, to) From 3755aa4ac98585b76016d74bafd9e97a20377acd Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 20 Jun 2024 15:24:25 +0300 Subject: [PATCH 45/59] - --- repository/statistics/click_statistics.go | 1 - 1 file changed, 1 deletion(-) diff --git a/repository/statistics/click_statistics.go b/repository/statistics/click_statistics.go index 7bd3451..91061c2 100644 --- a/repository/statistics/click_statistics.go +++ b/repository/statistics/click_statistics.go @@ -40,7 +40,6 @@ SELECT event_time, ctxsession, ctxquizid, ctxquestionid, ctxidint, message,ctxquiz FROM statistics WHERE message IN ('InfoQuizOpen', 'InfoAnswer', 'InfoResult') AND event_level = 'info'; -; ` _, err := s.conn.ExecContext(ctx, query) if err != nil { From 629c956e37caa812cb93e3c5ed4779f349b39686 Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 21 Jun 2024 12:49:36 +0300 Subject: [PATCH 46/59] update query --- dal/db_query/queries.sql | 2 +- repository/amo/amo.go | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/dal/db_query/queries.sql b/dal/db_query/queries.sql index 13dfcee..1a863bb 100644 --- a/dal/db_query/queries.sql +++ b/dal/db_query/queries.sql @@ -1048,7 +1048,7 @@ RETURNING p.id, p.privilegeID, p.account_id, p.privilege_name, p.amount, p.creat -- name: GetExistingContactAmo :one SELECT ID, AccountID, AmoID, Field -FROM amoContact WHERE AccountID = $1 AND (Field = $2 OR Field = $3); +FROM amoContact WHERE AccountID = $1 AND Field = $2; -- name: InsertContactAmo :one INSERT INTO amoContact (AccountID, AmoID, Field) VALUES ($1, $2, $3) RETURNING AmoID; diff --git a/repository/amo/amo.go b/repository/amo/amo.go index c00080d..998433e 100644 --- a/repository/amo/amo.go +++ b/repository/amo/amo.go @@ -6,6 +6,7 @@ import ( "encoding/json" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/dal/sqlcgen" "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/model" + "penahub.gitlab.yandexcloud.net/backend/quiz/common.git/pj_errors" "time" ) @@ -1080,3 +1081,26 @@ func (r *AmoRepository) UpdatingDealAmoStatus(ctx context.Context, deps SaveDeal return nil } + +// методы для contact в амо + +func (r *AmoRepository) GetExistingContactAmo(ctx context.Context, accountID int32, email, phone string) (model.ContactAmo, error) { + row, err := r.queries.GetExistingContactAmo(ctx, sqlcgen.GetExistingContactAmoParams{ + Accountid: accountID, + Field: email, + Field_2: phone, + }) + + if err != nil { + if err == sql.ErrNoRows { + return model.ContactAmo{}, pj_errors.ErrNotFound + } + return model.ContactAmo{}, err + } + + return model.ContactAmo{ + ID: row.ID, + AccountID: row.Accountid, + AmoID: row.Amoid, + }, nil +} From bf2e6c47df562a4249dd4217a66f3dc96d72abf1 Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 21 Jun 2024 12:51:51 +0300 Subject: [PATCH 47/59] update sqlc gen --- dal/sqlcgen/queries.sql.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dal/sqlcgen/queries.sql.go b/dal/sqlcgen/queries.sql.go index f02bc72..7807b85 100644 --- a/dal/sqlcgen/queries.sql.go +++ b/dal/sqlcgen/queries.sql.go @@ -1451,17 +1451,16 @@ func (q *Queries) GetCurrentCompany(ctx context.Context, accountid string) (Acco const getExistingContactAmo = `-- name: GetExistingContactAmo :one SELECT ID, AccountID, AmoID, Field -FROM amoContact WHERE AccountID = $1 AND (Field = $2 OR Field = $3) +FROM amoContact WHERE AccountID = $1 AND Field = $2 ` type GetExistingContactAmoParams struct { Accountid int32 `db:"accountid" json:"accountid"` Field string `db:"field" json:"field"` - Field_2 string `db:"field_2" json:"field_2"` } func (q *Queries) GetExistingContactAmo(ctx context.Context, arg GetExistingContactAmoParams) (Amocontact, error) { - row := q.db.QueryRowContext(ctx, getExistingContactAmo, arg.Accountid, arg.Field, arg.Field_2) + row := q.db.QueryRowContext(ctx, getExistingContactAmo, arg.Accountid, arg.Field) var i Amocontact err := row.Scan( &i.ID, From 0b2b219d97641e024f5d4af4ed69c86397449090 Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 21 Jun 2024 13:20:47 +0300 Subject: [PATCH 48/59] update query --- dal/db_query/queries.sql | 8 +++++--- repository/amo/amo.go | 32 +++++++++++++++++++++++++++++--- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/dal/db_query/queries.sql b/dal/db_query/queries.sql index 1a863bb..e82d021 100644 --- a/dal/db_query/queries.sql +++ b/dal/db_query/queries.sql @@ -1046,9 +1046,11 @@ UPDATE privileges p SET amount = amount - 1 FROM account a WHERE p.account_id = a.id AND a.user_id = $1 AND p.privilegeID = $2 AND p.amount > 0 RETURNING p.id, p.privilegeID, p.account_id, p.privilege_name, p.amount, p.created_at; --- name: GetExistingContactAmo :one -SELECT ID, AccountID, AmoID, Field -FROM amoContact WHERE AccountID = $1 AND Field = $2; +-- name: GetExistingContactAmo :many +WITH getAmoID AS ( + SELECT AmoID FROM amoContact WHERE AccountID = $1 AND Field IN ($2) +) SELECT DISTINCT * FROM amoContact +WHERE AccountID = $1 AND AmoID IN (SELECT AmoID FROM getAmoID); -- name: InsertContactAmo :one INSERT INTO amoContact (AccountID, AmoID, Field) VALUES ($1, $2, $3) RETURNING AmoID; diff --git a/repository/amo/amo.go b/repository/amo/amo.go index 998433e..7187df9 100644 --- a/repository/amo/amo.go +++ b/repository/amo/amo.go @@ -1084,11 +1084,10 @@ func (r *AmoRepository) UpdatingDealAmoStatus(ctx context.Context, deps SaveDeal // методы для contact в амо -func (r *AmoRepository) GetExistingContactAmo(ctx context.Context, accountID int32, email, phone string) (model.ContactAmo, error) { +func (r *AmoRepository) GetExistingContactAmo(ctx context.Context, accountID int32, field string) (model.ContactAmo, error) { row, err := r.queries.GetExistingContactAmo(ctx, sqlcgen.GetExistingContactAmoParams{ Accountid: accountID, - Field: email, - Field_2: phone, + Field: field, }) if err != nil { @@ -1104,3 +1103,30 @@ func (r *AmoRepository) GetExistingContactAmo(ctx context.Context, accountID int AmoID: row.Amoid, }, nil } + +func (r *AmoRepository) InsertContactAmo(ctx context.Context, val model.ContactAmo) (int32, error) { + amoID, err := r.queries.InsertContactAmo(ctx, sqlcgen.InsertContactAmoParams{ + Accountid: val.AccountID, + Amoid: val.AmoID, + Field: val.Field, + }) + + if err != nil { + return 0, err + } + + return amoID, err +} + +func (r *AmoRepository) UpdateAmoContact(ctx context.Context, id int64, field string) error { + err := r.queries.UpdateAmoContact(ctx, sqlcgen.UpdateAmoContactParams{ + Field: field, + ID: id, + }) + + if err != nil { + return err + } + + return nil +} From 80771c049304f59b072777d41c33ca201cdeba8c Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 21 Jun 2024 13:24:19 +0300 Subject: [PATCH 49/59] update sqlc gen --- dal/db_query/queries.sql | 4 ++-- dal/sqlcgen/queries.sql.go | 44 +++++++++++++++++++++++++++----------- 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/dal/db_query/queries.sql b/dal/db_query/queries.sql index e82d021..504438b 100644 --- a/dal/db_query/queries.sql +++ b/dal/db_query/queries.sql @@ -1048,9 +1048,9 @@ RETURNING p.id, p.privilegeID, p.account_id, p.privilege_name, p.amount, p.creat -- name: GetExistingContactAmo :many WITH getAmoID AS ( - SELECT AmoID FROM amoContact WHERE AccountID = $1 AND Field IN ($2) + SELECT AmoID FROM amoContact WHERE amoContact.AccountID = $1 AND amoContact.Field IN ($2) ) SELECT DISTINCT * FROM amoContact -WHERE AccountID = $1 AND AmoID IN (SELECT AmoID FROM getAmoID); +WHERE amoContact.AccountID = $1 AND amoContact.AmoID IN (SELECT AmoID FROM getAmoID); -- name: InsertContactAmo :one INSERT INTO amoContact (AccountID, AmoID, Field) VALUES ($1, $2, $3) RETURNING AmoID; diff --git a/dal/sqlcgen/queries.sql.go b/dal/sqlcgen/queries.sql.go index 7807b85..3f6b18f 100644 --- a/dal/sqlcgen/queries.sql.go +++ b/dal/sqlcgen/queries.sql.go @@ -1449,9 +1449,11 @@ func (q *Queries) GetCurrentCompany(ctx context.Context, accountid string) (Acco return i, err } -const getExistingContactAmo = `-- name: GetExistingContactAmo :one -SELECT ID, AccountID, AmoID, Field -FROM amoContact WHERE AccountID = $1 AND Field = $2 +const getExistingContactAmo = `-- name: GetExistingContactAmo :many +WITH getAmoID AS ( + SELECT AmoID FROM amoContact WHERE amoContact.AccountID = $1 AND amoContact.Field IN ($2) +) SELECT DISTINCT id, accountid, amoid, field FROM amoContact +WHERE amoContact.AccountID = $1 AND amoContact.AmoID IN (SELECT AmoID FROM getAmoID) ` type GetExistingContactAmoParams struct { @@ -1459,16 +1461,32 @@ type GetExistingContactAmoParams struct { Field string `db:"field" json:"field"` } -func (q *Queries) GetExistingContactAmo(ctx context.Context, arg GetExistingContactAmoParams) (Amocontact, error) { - row := q.db.QueryRowContext(ctx, getExistingContactAmo, arg.Accountid, arg.Field) - var i Amocontact - err := row.Scan( - &i.ID, - &i.Accountid, - &i.Amoid, - &i.Field, - ) - return i, err +func (q *Queries) GetExistingContactAmo(ctx context.Context, arg GetExistingContactAmoParams) ([]Amocontact, error) { + rows, err := q.db.QueryContext(ctx, getExistingContactAmo, arg.Accountid, arg.Field) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Amocontact + for rows.Next() { + var i Amocontact + if err := rows.Scan( + &i.ID, + &i.Accountid, + &i.Amoid, + &i.Field, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil } const getExpiredCountPrivilege = `-- name: GetExpiredCountPrivilege :many From f9eec59ce1529d0c5193946fab4dfd4726a79a5e Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 21 Jun 2024 13:26:42 +0300 Subject: [PATCH 50/59] update sqlc gen --- dal/db_query/queries.sql | 2 +- dal/sqlcgen/queries.sql.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dal/db_query/queries.sql b/dal/db_query/queries.sql index 504438b..edc8588 100644 --- a/dal/db_query/queries.sql +++ b/dal/db_query/queries.sql @@ -1048,7 +1048,7 @@ RETURNING p.id, p.privilegeID, p.account_id, p.privilege_name, p.amount, p.creat -- name: GetExistingContactAmo :many WITH getAmoID AS ( - SELECT AmoID FROM amoContact WHERE amoContact.AccountID = $1 AND amoContact.Field IN ($2) + SELECT AmoID FROM amoContact WHERE amoContact.AccountID = $1 AND amoContact.Field IN ($2::text[]) ) SELECT DISTINCT * FROM amoContact WHERE amoContact.AccountID = $1 AND amoContact.AmoID IN (SELECT AmoID FROM getAmoID); diff --git a/dal/sqlcgen/queries.sql.go b/dal/sqlcgen/queries.sql.go index 3f6b18f..34eefd4 100644 --- a/dal/sqlcgen/queries.sql.go +++ b/dal/sqlcgen/queries.sql.go @@ -1451,18 +1451,18 @@ func (q *Queries) GetCurrentCompany(ctx context.Context, accountid string) (Acco const getExistingContactAmo = `-- name: GetExistingContactAmo :many WITH getAmoID AS ( - SELECT AmoID FROM amoContact WHERE amoContact.AccountID = $1 AND amoContact.Field IN ($2) + SELECT AmoID FROM amoContact WHERE amoContact.AccountID = $1 AND amoContact.Field IN ($2::text[]) ) SELECT DISTINCT id, accountid, amoid, field FROM amoContact WHERE amoContact.AccountID = $1 AND amoContact.AmoID IN (SELECT AmoID FROM getAmoID) ` type GetExistingContactAmoParams struct { - Accountid int32 `db:"accountid" json:"accountid"` - Field string `db:"field" json:"field"` + Accountid int32 `db:"accountid" json:"accountid"` + Column2 []string `db:"column_2" json:"column_2"` } func (q *Queries) GetExistingContactAmo(ctx context.Context, arg GetExistingContactAmoParams) ([]Amocontact, error) { - rows, err := q.db.QueryContext(ctx, getExistingContactAmo, arg.Accountid, arg.Field) + rows, err := q.db.QueryContext(ctx, getExistingContactAmo, arg.Accountid, pq.Array(arg.Column2)) if err != nil { return nil, err } From 70c87cef1b4a74ecdfcef88e496df18b406672e8 Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 21 Jun 2024 13:31:15 +0300 Subject: [PATCH 51/59] update query --- dal/db_query/queries.sql | 2 +- repository/amo/amo.go | 27 +++++++++++++++++---------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/dal/db_query/queries.sql b/dal/db_query/queries.sql index edc8588..85bcd9f 100644 --- a/dal/db_query/queries.sql +++ b/dal/db_query/queries.sql @@ -1049,7 +1049,7 @@ RETURNING p.id, p.privilegeID, p.account_id, p.privilege_name, p.amount, p.creat -- name: GetExistingContactAmo :many WITH getAmoID AS ( SELECT AmoID FROM amoContact WHERE amoContact.AccountID = $1 AND amoContact.Field IN ($2::text[]) -) SELECT DISTINCT * FROM amoContact +) SELECT * FROM amoContact WHERE amoContact.AccountID = $1 AND amoContact.AmoID IN (SELECT AmoID FROM getAmoID); -- name: InsertContactAmo :one diff --git a/repository/amo/amo.go b/repository/amo/amo.go index 7187df9..0a9b15a 100644 --- a/repository/amo/amo.go +++ b/repository/amo/amo.go @@ -1084,24 +1084,31 @@ func (r *AmoRepository) UpdatingDealAmoStatus(ctx context.Context, deps SaveDeal // методы для contact в амо -func (r *AmoRepository) GetExistingContactAmo(ctx context.Context, accountID int32, field string) (model.ContactAmo, error) { - row, err := r.queries.GetExistingContactAmo(ctx, sqlcgen.GetExistingContactAmoParams{ +func (r *AmoRepository) GetExistingContactAmo(ctx context.Context, accountID int32, fields []string) ([]model.ContactAmo, error) { + rows, err := r.queries.GetExistingContactAmo(ctx, sqlcgen.GetExistingContactAmoParams{ Accountid: accountID, - Field: field, + Column2: fields, }) if err != nil { if err == sql.ErrNoRows { - return model.ContactAmo{}, pj_errors.ErrNotFound + return nil, pj_errors.ErrNotFound } - return model.ContactAmo{}, err + return nil, err } - return model.ContactAmo{ - ID: row.ID, - AccountID: row.Accountid, - AmoID: row.Amoid, - }, nil + var result []model.ContactAmo + + for _, row := range rows { + result = append(result, model.ContactAmo{ + ID: row.ID, + AmoID: row.Amoid, + AccountID: row.Accountid, + Field: row.Field, + }) + } + + return result, nil } func (r *AmoRepository) InsertContactAmo(ctx context.Context, val model.ContactAmo) (int32, error) { From 42e95667c132e38848edbe732b4817d411bfd919 Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 21 Jun 2024 13:32:11 +0300 Subject: [PATCH 52/59] update sqlc gen --- dal/sqlcgen/queries.sql.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dal/sqlcgen/queries.sql.go b/dal/sqlcgen/queries.sql.go index 34eefd4..6b6d223 100644 --- a/dal/sqlcgen/queries.sql.go +++ b/dal/sqlcgen/queries.sql.go @@ -1452,7 +1452,7 @@ func (q *Queries) GetCurrentCompany(ctx context.Context, accountid string) (Acco const getExistingContactAmo = `-- name: GetExistingContactAmo :many WITH getAmoID AS ( SELECT AmoID FROM amoContact WHERE amoContact.AccountID = $1 AND amoContact.Field IN ($2::text[]) -) SELECT DISTINCT id, accountid, amoid, field FROM amoContact +) SELECT id, accountid, amoid, field FROM amoContact WHERE amoContact.AccountID = $1 AND amoContact.AmoID IN (SELECT AmoID FROM getAmoID) ` From 9616b086fa4ac86fa165664e59680f3cb1bcf867 Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 21 Jun 2024 13:35:13 +0300 Subject: [PATCH 53/59] update amo reps --- repository/amo/amo.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/repository/amo/amo.go b/repository/amo/amo.go index 0a9b15a..cc8a77d 100644 --- a/repository/amo/amo.go +++ b/repository/amo/amo.go @@ -1084,7 +1084,7 @@ func (r *AmoRepository) UpdatingDealAmoStatus(ctx context.Context, deps SaveDeal // методы для contact в амо -func (r *AmoRepository) GetExistingContactAmo(ctx context.Context, accountID int32, fields []string) ([]model.ContactAmo, error) { +func (r *AmoRepository) GetExistingContactAmo(ctx context.Context, accountID int32, fields []string) (map[string]model.ContactAmo, error) { rows, err := r.queries.GetExistingContactAmo(ctx, sqlcgen.GetExistingContactAmoParams{ Accountid: accountID, Column2: fields, @@ -1097,15 +1097,15 @@ func (r *AmoRepository) GetExistingContactAmo(ctx context.Context, accountID int return nil, err } - var result []model.ContactAmo + result := make(map[string]model.ContactAmo) for _, row := range rows { - result = append(result, model.ContactAmo{ + result[row.Field] = model.ContactAmo{ ID: row.ID, AmoID: row.Amoid, AccountID: row.Accountid, Field: row.Field, - }) + } } return result, nil From 4b70022ad6757ddd17d07f66533b6399b9b30e3e Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 21 Jun 2024 18:01:28 +0300 Subject: [PATCH 54/59] - --- repository/amo/amo.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/repository/amo/amo.go b/repository/amo/amo.go index cc8a77d..6aa8211 100644 --- a/repository/amo/amo.go +++ b/repository/amo/amo.go @@ -1084,7 +1084,7 @@ func (r *AmoRepository) UpdatingDealAmoStatus(ctx context.Context, deps SaveDeal // методы для contact в амо -func (r *AmoRepository) GetExistingContactAmo(ctx context.Context, accountID int32, fields []string) (map[string]model.ContactAmo, error) { +func (r *AmoRepository) GetExistingContactAmo(ctx context.Context, accountID int32, fields []string) (map[int32][]model.ContactAmo, error) { rows, err := r.queries.GetExistingContactAmo(ctx, sqlcgen.GetExistingContactAmoParams{ Accountid: accountID, Column2: fields, @@ -1097,15 +1097,15 @@ func (r *AmoRepository) GetExistingContactAmo(ctx context.Context, accountID int return nil, err } - result := make(map[string]model.ContactAmo) + result := make(map[int32][]model.ContactAmo) for _, row := range rows { - result[row.Field] = model.ContactAmo{ + result[row.Amoid] = append(result[row.Amoid], model.ContactAmo{ ID: row.ID, AmoID: row.Amoid, AccountID: row.Accountid, Field: row.Field, - } + }) } return result, nil From bd9e81bf78b98c53ab91bcda69194ac67f5ae92b Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 21 Jun 2024 20:24:06 +0300 Subject: [PATCH 55/59] update --- dal/db_query/queries.sql | 2 +- repository/amo/amo.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dal/db_query/queries.sql b/dal/db_query/queries.sql index 85bcd9f..bc95763 100644 --- a/dal/db_query/queries.sql +++ b/dal/db_query/queries.sql @@ -1056,4 +1056,4 @@ WHERE amoContact.AccountID = $1 AND amoContact.AmoID IN (SELECT AmoID FROM getAm INSERT INTO amoContact (AccountID, AmoID, Field) VALUES ($1, $2, $3) RETURNING AmoID; -- name: UpdateAmoContact :exec -UPDATE amoContact SET Field = $1 WHERE ID = $2; +UPDATE amoContact SET Field = $1,AmoID=$3 WHERE ID = $2; diff --git a/repository/amo/amo.go b/repository/amo/amo.go index 6aa8211..010f828 100644 --- a/repository/amo/amo.go +++ b/repository/amo/amo.go @@ -1125,7 +1125,7 @@ func (r *AmoRepository) InsertContactAmo(ctx context.Context, val model.ContactA return amoID, err } -func (r *AmoRepository) UpdateAmoContact(ctx context.Context, id int64, field string) error { +func (r *AmoRepository) UpdateAmoContact(ctx context.Context, id int64, field string, newAmoID int32) error { err := r.queries.UpdateAmoContact(ctx, sqlcgen.UpdateAmoContactParams{ Field: field, ID: id, From 56aa1da5d2a23c6f35ecd195ab11dc2788af98ec Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 21 Jun 2024 19:51:55 +0300 Subject: [PATCH 56/59] update sqlc gen --- dal/sqlcgen/queries.sql.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dal/sqlcgen/queries.sql.go b/dal/sqlcgen/queries.sql.go index 6b6d223..3fbd37c 100644 --- a/dal/sqlcgen/queries.sql.go +++ b/dal/sqlcgen/queries.sql.go @@ -3508,16 +3508,17 @@ func (q *Queries) UpdateAmoAccountUser(ctx context.Context, arg UpdateAmoAccount } const updateAmoContact = `-- name: UpdateAmoContact :exec -UPDATE amoContact SET Field = $1 WHERE ID = $2 +UPDATE amoContact SET Field = $1,AmoID=$3 WHERE ID = $2 ` type UpdateAmoContactParams struct { Field string `db:"field" json:"field"` ID int64 `db:"id" json:"id"` + Amoid int32 `db:"amoid" json:"amoid"` } func (q *Queries) UpdateAmoContact(ctx context.Context, arg UpdateAmoContactParams) error { - _, err := q.db.ExecContext(ctx, updateAmoContact, arg.Field, arg.ID) + _, err := q.db.ExecContext(ctx, updateAmoContact, arg.Field, arg.ID, arg.Amoid) return err } From 16c15074c4961dc5669b3e650347af55eae74c50 Mon Sep 17 00:00:00 2001 From: Pavel Date: Sun, 23 Jun 2024 12:57:35 +0300 Subject: [PATCH 57/59] fix --- dal/db_query/queries.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dal/db_query/queries.sql b/dal/db_query/queries.sql index bc95763..f913b47 100644 --- a/dal/db_query/queries.sql +++ b/dal/db_query/queries.sql @@ -1048,7 +1048,7 @@ RETURNING p.id, p.privilegeID, p.account_id, p.privilege_name, p.amount, p.creat -- name: GetExistingContactAmo :many WITH getAmoID AS ( - SELECT AmoID FROM amoContact WHERE amoContact.AccountID = $1 AND amoContact.Field IN ($2::text[]) + SELECT AmoID FROM amoContact WHERE amoContact.AccountID = $1 AND amoContact.Field = ANY($2::text[]) ) SELECT * FROM amoContact WHERE amoContact.AccountID = $1 AND amoContact.AmoID IN (SELECT AmoID FROM getAmoID); From e374d50e42e671b71782f094b90e39cf796a091f Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 21 Jun 2024 19:53:14 +0300 Subject: [PATCH 58/59] update sqlc gen --- dal/sqlcgen/queries.sql.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dal/sqlcgen/queries.sql.go b/dal/sqlcgen/queries.sql.go index 3fbd37c..ef3ca61 100644 --- a/dal/sqlcgen/queries.sql.go +++ b/dal/sqlcgen/queries.sql.go @@ -1451,7 +1451,7 @@ func (q *Queries) GetCurrentCompany(ctx context.Context, accountid string) (Acco const getExistingContactAmo = `-- name: GetExistingContactAmo :many WITH getAmoID AS ( - SELECT AmoID FROM amoContact WHERE amoContact.AccountID = $1 AND amoContact.Field IN ($2::text[]) + SELECT AmoID FROM amoContact WHERE amoContact.AccountID = $1 AND amoContact.Field = ANY($2::text[]) ) SELECT id, accountid, amoid, field FROM amoContact WHERE amoContact.AccountID = $1 AND amoContact.AmoID IN (SELECT AmoID FROM getAmoID) ` From 5cbeb2b88f0a694bd409017dff5bd28867144fae Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 21 Jun 2024 20:27:46 +0300 Subject: [PATCH 59/59] update --- repository/amo/amo.go | 1 + 1 file changed, 1 insertion(+) diff --git a/repository/amo/amo.go b/repository/amo/amo.go index 010f828..63e2e3d 100644 --- a/repository/amo/amo.go +++ b/repository/amo/amo.go @@ -1129,6 +1129,7 @@ func (r *AmoRepository) UpdateAmoContact(ctx context.Context, id int64, field st err := r.queries.UpdateAmoContact(ctx, sqlcgen.UpdateAmoContactParams{ Field: field, ID: id, + Amoid: newAmoID, }) if err != nil {