PenaSide/trashlog
3
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

upd with dev into workWithConfig

Browse Source
This commit is contained in:
Pasha 2024-12-06 16:33:36 +03:00
commit f1c60512a4
23 changed files with 210 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
.gitea/workflows/deploy.yml
View File

@ -10,7 +10,7 @@ on:
jobs:
CreateImage:
runs-on: [apimetrics]
uses: http://gitea.pena:3000/PenaDevops/actions.git/.gitea/workflows/build-image.yml@b38d48f5481ddd5a135a6c47ad7169db75d73685
uses: http://gitea.pena/PenaDevops/actions.git/.gitea/workflows/build-image.yml@v1.0.1
with:
runner: apimetrics
secrets:
@ -19,17 +19,6 @@ jobs:
DeployService:
runs-on: [apimetrics]
needs: CreateImage
container:
image: gitea.pena:3000/penadevops/container-images/node-compose:main
env:
CI_REGISTRY_IMAGE: ${{ needs.Build-Image.outputs.image }}
volumes:
- /run/user/1000/podman/podman.sock:/run/user/1000/podman/podman.sock
steps:
- name: Check out repository code
uses: http://gitea.pena:3000/PenaDevops/actions.git/checkout@v1
- run: compose -f deployments/${{ gitea.ref_name }}/docker-compose.yaml up -d
#uses: http://gitea.pena:3000/PenaDevops/actions.git/.gitea/workflows/deploy.yml@c7a3d48afc455f722bc354965080f50e2d564a95
#with:
#runner: apimetrics
uses: http://gitea.pena/PenaDevops/actions.git/.gitea/workflows/deploy.yml@v1.0.1
with:
runner: apimetrics

14
.gitea/workflows/lint.yml Normal file
View File

@ -0,0 +1,14 @@
name: Lint
run-name: ${{ gitea.actor }} produce linting
on:
push:
branches:
- 'dev'
jobs:
Lint:
runs-on: [hubstaging]
uses: http://gitea.pena/PenaDevops/actions.git/.gitea/workflows/lint.yml@v1.1.2
with:
runner: hubstaging

24
.gitlab-ci.yml
View File

@ -1,24 +0,0 @@
include:
- project: "devops/pena-continuous-integration"
file: "/templates/docker/build-template.gitlab-ci.yml"
- project: "devops/pena-continuous-integration"
file: "/templates/docker/deploy-template.gitlab-ci.yml"
- project: "devops/pena-continuous-integration"
file: "/templates/docker/golint.gitlab-ci.yml"
stages:
- lint
- build
- deploy
lint:
extends: .golint_template
build-app:
extends: .build_template
deploy-to-staging:
extends: .deploy_template
tags:
- trashlog
rules:
- if: "$CI_COMMIT_BRANCH == $STAGING_BRANCH"

4
Taskfile.dist.yml
View File

@ -3,8 +3,8 @@ version: "3"
tasks:
update-linter:
cmds:
- go get -u penahub.gitlab.yandexcloud.net/devops/linters/golang.git
- go get -u gitea.pena/PenaSide/linters-golang
lint:
cmds:
- task: update-linter
- cmd: golangci-lint run -v -c $(go list -f '{{"{{"}}.Dir{{"}}"}}' -m penahub.gitlab.yandexcloud.net/devops/linters/golang.git)/.golangci.yml
- cmd: golangci-lint run -v -c $(go list -f '{{"{{"}}.Dir{{"}}"}}' -m gitea.pena/PenaSide/linters-golang)/.golangci.yml

19
app/app.go
View File

@ -4,21 +4,20 @@ import (
"context"
"errors"
"fmt"
"net"
"penahub.gitlab.yandexcloud.net/external/trashlog/clients"
"penahub.gitlab.yandexcloud.net/external/trashlog/controllers/http/health_check"
"penahub.gitlab.yandexcloud.net/external/trashlog/server"
"gitea.pena/PenaSide/trashlog/clients"
"gitea.pena/PenaSide/trashlog/controllers/http/health_check"
"gitea.pena/PenaSide/trashlog/dal/clickhouse"
trashlogProto "gitea.pena/PenaSide/trashlog/proto/generated"
"gitea.pena/PenaSide/trashlog/server"
"gitea.pena/PenaSide/trashlog/sink"
"gitea.pena/PenaSide/trashlog/version"
"gitea.pena/PenaSide/trashlog/wrappers/zaptg"
"github.com/skeris/appInit"
"github.com/themakers/hlog"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"google.golang.org/grpc"
"penahub.gitlab.yandexcloud.net/external/trashlog/dal/clickhouse"
trashlogProto "penahub.gitlab.yandexcloud.net/external/trashlog/proto/generated"
"penahub.gitlab.yandexcloud.net/external/trashlog/sink"
"penahub.gitlab.yandexcloud.net/external/trashlog/version"
"penahub.gitlab.yandexcloud.net/external/trashlog/wrappers/zaptg"
"net"
)
type App struct {

2
app/validate.go
View File

@ -5,12 +5,12 @@ import (
"database/sql"
"errors"
"fmt"
"gitea.pena/PenaSide/trashlog/dal/bbolt"
_ "github.com/ClickHouse/clickhouse-go"
bolt "go.etcd.io/bbolt"
tb "gopkg.in/tucnak/telebot.v2"
"os"
"path/filepath"
"penahub.gitlab.yandexcloud.net/external/trashlog/dal/bbolt"
"strconv"
"strings"
"time"

2
clients/trashlog_rpc.go
View File

@ -3,9 +3,9 @@ package clients
import (
"context"
"fmt"
trashlogProto "gitea.pena/PenaSide/trashlog/proto/generated"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
trashlogProto "penahub.gitlab.yandexcloud.net/external/trashlog/proto/generated"
)
type TrashLogClient struct {

2
cmd/main.go
View File

@ -1,8 +1,8 @@
package main
import (
"gitea.pena/PenaSide/trashlog/app"
"log"
"penahub.gitlab.yandexcloud.net/external/trashlog/app"
)
func initApp(cfg *app.Options) {

4
controllers/http/health_check/health_check.go
View File

@ -1,10 +1,10 @@
package health_check
import (
"gitea.pena/PenaSide/trashlog/clients"
"gitea.pena/PenaSide/trashlog/dal/clickhouse"
"github.com/gofiber/fiber/v2"
"github.com/themakers/hlog"
"penahub.gitlab.yandexcloud.net/external/trashlog/clients"
"penahub.gitlab.yandexcloud.net/external/trashlog/dal/clickhouse"
)
type HealthCheck struct {

4
dal/bbolt/bbolt_test.go
View File

@ -7,8 +7,8 @@ import (
bolt "go.etcd.io/bbolt"
"go.uber.org/zap"
"os"
"penahub.gitlab.yandexcloud.net/external/trashlog/dal/bbolt"
"penahub.gitlab.yandexcloud.net/external/trashlog/model"
"gitea.pena/PenaSide/trashlog/dal/bbolt"
"gitea.pena/PenaSide/trashlog/model"
"testing"
"time"
)

2
dal/bbolt/boltdb.go
View File

@ -6,7 +6,7 @@ import (
"github.com/rs/xid"
bolt "go.etcd.io/bbolt"
"os"
"penahub.gitlab.yandexcloud.net/external/trashlog/model"
"gitea.pena/PenaSide/trashlog/model"
"time"
)

4
dal/clickhouse/clickhouse.go
View File

@ -6,8 +6,8 @@ import (
"fmt"
"github.com/ClickHouse/clickhouse-go"
"github.com/themakers/hlog"
"penahub.gitlab.yandexcloud.net/external/trashlog/dal/bbolt"
"penahub.gitlab.yandexcloud.net/external/trashlog/model"
"gitea.pena/PenaSide/trashlog/dal/bbolt"
"gitea.pena/PenaSide/trashlog/model"
"strings"
"sync"
"time"

2
dal/clickhouse/clickhouse_test.go
View File

@ -9,7 +9,7 @@ import (
"github.com/themakers/bdd"
"github.com/themakers/hlog"
"go.uber.org/zap"
"penahub.gitlab.yandexcloud.net/external/trashlog/model"
"gitea.pena/PenaSide/trashlog/model"
"strings"
"testing"
"time"

4
go.mod
View File

@ -1,8 +1,9 @@
module penahub.gitlab.yandexcloud.net/external/trashlog
module gitea.pena/PenaSide/trashlog
go 1.22.0
require (
gitea.pena/PenaSide/linters-golang v0.0.0-20241119212350-2759fa93724a
github.com/ClickHouse/clickhouse-go v1.5.4
github.com/gofiber/fiber/v2 v2.52.5
github.com/onsi/ginkgo v1.16.0
@ -18,7 +19,6 @@ require (
google.golang.org/grpc v1.47.0
google.golang.org/protobuf v1.28.0
gopkg.in/tucnak/telebot.v2 v2.5.0
penahub.gitlab.yandexcloud.net/devops/linters/golang.git v0.0.0-20240827173819-1561c532e463
)
require (

4
go.sum
View File

@ -1,5 +1,7 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
gitea.pena/PenaSide/linters-golang v0.0.0-20241119212350-2759fa93724a h1:UySqMgaOKNsR42Y6GQXoM2wn/waYNc9cakMUSvbEEAg=
gitea.pena/PenaSide/linters-golang v0.0.0-20241119212350-2759fa93724a/go.mod h1:gdd+vOT6up9STkEbxa2qESLIMZFjCmRbkcheFQCVgZU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0=
github.com/ClickHouse/clickhouse-go v1.5.4/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=
@ -281,5 +283,3 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
penahub.gitlab.yandexcloud.net/devops/linters/golang.git v0.0.0-20240827173819-1561c532e463 h1:Pz86cnhDTWNd+4nICjGtbNnIRjIIGnyOxn5iRx0qVKY=
penahub.gitlab.yandexcloud.net/devops/linters/golang.git v0.0.0-20240827173819-1561c532e463/go.mod h1:i7M72RIpkSjcQtHID6KKj9RT/EYZ1rxS6tIPKWa/BSY=

4
main.go
View File

@ -2,8 +2,8 @@ package main
import (
"github.com/skeris/appInit"
"penahub.gitlab.yandexcloud.net/external/trashlog/app"
_ "penahub.gitlab.yandexcloud.net/devops/linters/golang.git/pkg/dummy"
"gitea.pena/PenaSide/trashlog/app"
_ "gitea.pena/PenaSide/linters-golang/pkg/dummy"
)
func main() {

6
sink/sink.go
View File

@ -5,9 +5,9 @@ import (
"fmt"
"github.com/themakers/hlog"
"penahub.gitlab.yandexcloud.net/external/trashlog/dal/clickhouse"
"penahub.gitlab.yandexcloud.net/external/trashlog/model"
trashlogProto "penahub.gitlab.yandexcloud.net/external/trashlog/proto/generated"
"gitea.pena/PenaSide/trashlog/dal/clickhouse"
"gitea.pena/PenaSide/trashlog/model"
trashlogProto "gitea.pena/PenaSide/trashlog/proto/generated"
)
type Sink struct {

2
sink/sink_test.go
View File

@ -6,7 +6,7 @@ import (
. "github.com/onsi/ginkgo"
"go.uber.org/zap"
"google.golang.org/grpc"
pb "penahub.gitlab.yandexcloud.net/external/trashlog/proto/generated"
pb "gitea.pena/PenaSide/trashlog/proto/generated"
)
var _ = Describe("Sink", func() {

2
tlanaliz/main.go
View File

@ -12,7 +12,7 @@ import (
"google.golang.org/grpc/credentials/insecure"
"io"
"os"
pb "penahub.gitlab.yandexcloud.net/external/trashlog/proto/generated"
pb "gitea.pena/PenaSide/trashlog/proto/generated"
"strings"
)

2
wrappers/zaptg/zaptg.go
View File

@ -7,7 +7,7 @@ import (
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
tb "gopkg.in/tucnak/telebot.v2"
pb "penahub.gitlab.yandexcloud.net/external/trashlog/proto/generated"
pb "gitea.pena/PenaSide/trashlog/proto/generated"
"strings"
)

2
wrappers/zaptg/zaptg_test.go
View File

@ -7,7 +7,7 @@ import (
"github.com/themakers/hlog"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"penahub.gitlab.yandexcloud.net/external/trashlog/wrappers/zaptg"
"gitea.pena/PenaSide/trashlog/wrappers/zaptg"
)
var _ = Describe("Zaptg", func() {

18
wrappers/zaptrashlog/zaptrashlog.go
View File

@ -3,15 +3,15 @@ package zaptrashlog
import (
"context"
"fmt"
"gitea.pena/PenaSide/trashlog/dal/bbolt"
"gitea.pena/PenaSide/trashlog/model"
pb "gitea.pena/PenaSide/trashlog/proto/generated"
"gitea.pena/PenaSide/trashlog/sink"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"google.golang.org/grpc"
"google.golang.org/grpc/backoff"
"google.golang.org/grpc/credentials/insecure"
"penahub.gitlab.yandexcloud.net/external/trashlog/dal/bbolt"
"penahub.gitlab.yandexcloud.net/external/trashlog/model"
pb "penahub.gitlab.yandexcloud.net/external/trashlog/proto/generated"
"penahub.gitlab.yandexcloud.net/external/trashlog/sink"
"strings"
"sync"
"time"
@ -31,7 +31,7 @@ type TrashLogCore struct {
keyFields map[string]*pb.Value
recoverStore *bbolt.DAL
url string
sendMutex sync.Mutex
SendMutex *sync.Mutex
failCh chan bool
emitCh chan entryWithFields
connected bool
@ -71,7 +71,7 @@ func NewCore(
url: url,
ctxFields: make(map[string]*pb.Value),
keyFields: make(map[string]*pb.Value),
sendMutex: sync.Mutex{},
SendMutex: &sync.Mutex{},
failCh: make(chan bool),
emitCh: make(chan entryWithFields, 100),
connected: false,
@ -212,7 +212,7 @@ func (c *TrashLogCore) With(fields []zapcore.Field) zapcore.Core {
stream: c.stream,
recoverStore: c.recoverStore,
url: c.url,
sendMutex: c.sendMutex,
SendMutex: c.SendMutex,
failCh: c.failCh,
emitCh: c.emitCh,
}
@ -251,8 +251,8 @@ func (c *TrashLogCore) Write(
fields []zapcore.Field,
) error {
fmt.Println("Write()")
c.sendMutex.Lock()
defer c.sendMutex.Unlock()
c.SendMutex.Lock()
defer c.SendMutex.Unlock()
c.saveFields(entry, fields)
fmt.Println("Write()end")

154
wrappers/zaptrashlog/zaptrashlog_test.go
View File

@ -4,14 +4,16 @@ import (
"context"
"database/sql"
"fmt"
"gitea.pena/PenaSide/trashlog/app"
"github.com/skeris/appInit"
bolt "go.etcd.io/bbolt"
"penahub.gitlab.yandexcloud.net/external/trashlog/app"
"sync"
"testing"
"gitea.pena/PenaSide/trashlog/wrappers/zaptrashlog"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/themakers/hlog"
"penahub.gitlab.yandexcloud.net/external/trashlog/wrappers/zaptrashlog"
//bolt "go.etcd.io/bbolt"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
@ -29,10 +31,10 @@ var _ = Describe("Zaptrashlog", func() {
ctx := context.Background()
fmt.Println("MMMMM")
a, err := app.New(ctx, app.Options{
Development: true,
AppName: "trashlog-test",
Addr: ":7112",
ClickhouseCred: "tcp://127.0.0.1:9000?debug=true",
Development: true,
AppName: "trashlog-test",
GrpcURL: ":7112",
ClickhouseURL: "tcp://127.0.0.1:9000?debug=true",
}, appInit.Version{
Release: "test",
Commit: "",
@ -114,3 +116,143 @@ var _ = Describe("Zaptrashlog", func() {
type InfoTest struct {
Test string
}
type InfoLogger1 struct {
Name string
}
type InfoLogger2 struct {
Name string
}
// тест по сути ничего не проверяет просто два логгера которые емитят, ожидалась паника
// так как мьютекс не был указан в with, но паники не было
func TestMutex(t *testing.T) {
ctx1 := context.Background()
ctx2 := context.Background()
logger1, err := zap.NewProduction()
if err != nil {
panic(err)
}
logger2, err := zap.NewProduction()
if err != nil {
panic(err)
}
clickHouseLogger1, err := zaptrashlog.NewCore(ctx1, zap.InfoLevel, "localhost:7113", "1", "1", time.Now().Unix())
if err != nil {
panic(err)
}
clickHouseLogger2, err := zaptrashlog.NewCore(ctx2, zap.InfoLevel, "localhost:7113", "2", "2", time.Now().Unix())
if err != nil {
panic(err)
}
loggerForHlog1 := logger1.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core {
return zapcore.NewTee(core, clickHouseLogger1)
}))
loggerHlog1 := hlog.New(loggerForHlog1).Module("1")
loggerForHlog2 := logger2.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core {
return zapcore.NewTee(core, clickHouseLogger2)
}))
loggerHlog2 := hlog.New(loggerForHlog2).Module("2")
loggerHlog1.With(InfoTest{})
loggerHlog2.With(InfoTest{})
go func() {
count1 := 1
for {
level := fmt.Sprintf("loggerHlog1-Number%d", count1)
loggerHlog1.Emit(InfoTest{Test: level})
count1++
time.Sleep(time.Second * 1)
}
}()
go func() {
count2 := 1
for {
level := fmt.Sprintf("loggerHlog2-Number%d", count2)
loggerHlog2.Emit(InfoTest{Test: level})
count2++
time.Sleep(time.Second * 1)
}
}()
loggerHlog1.With(InfoLogger1{})
loggerHlog2.With(InfoLogger2{})
go func() {
count1 := 1
for {
level := fmt.Sprintf("InfoLogger1-Number%d", count1)
loggerHlog1.Emit(InfoLogger1{Name: level})
count1++
time.Sleep(time.Second * 1)
}
}()
go func() {
count2 := 1
for {
level := fmt.Sprintf("InfoLogger2-Number%d", count2)
loggerHlog2.Emit(InfoLogger2{Name: level})
count2++
time.Sleep(time.Second * 1)
}
}()
time.Sleep(time.Second * 10)
}
//тест чекает на пересоздание мьютекса в методе write, суть проблемы в том что это может порождать несколько мьютексов
//в тесте должны получить то что один мьютекс используется даже после пересоздания его в методе with
// алгоритм таков: создается в ядре -> потом создается экземпляр от zap и hlog -> проверка на то что senfmutex один,
// проверка средствами trylock, после emit вызывается write в котором лочим мьютекс и не разлочиваем, далее чекаем на то что
// основной логер от ядра который родитель этого мьютекса тоже имеет заблокированный мьютекс, это доказывает то что
//мьютекс один даже после with метода
func TestSendMutex(t *testing.T) {
l, err := zap.NewDevelopment()
if err != nil {
panic(err)
}
logger1, err := zaptrashlog.NewCore(context.Background(), zapcore.InfoLevel, "localhost:7113", "1", "commit123", time.Now().Unix())
if err != nil {
panic(err)
}
loggerForHlog1 := l.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core {
return zapcore.NewTee(core, logger1)
}))
loggerHlog1 := hlog.New(loggerForHlog1).Module("loggerForHlog1")
llooggerZap := loggerHlog1.With(InfoLogger1{})
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
llooggerZap.Emit(InfoTest{Test: "test"}) // мьютекс лочим
}()
time.Sleep(1 * time.Second)
// TryLock в логгере корневом что он заблокирован
resultLock := logger1.SendMutex.TryLock()
if resultLock {
fmt.Println(resultLock)
t.Error("Должен быть заблокирован но это не так")
} else {
fmt.Println(resultLock)
t.Log("sendMutex заблокирован ок")
}
wg.Wait()
}