package sink import ( "context" "github.com/themakers/hlog" "penahub.gitlab.yandexcloud.net/external/trashlog.git/dal/clickhouse" "penahub.gitlab.yandexcloud.net/external/trashlog.git/model" trashlogProto "penahub.gitlab.yandexcloud.net/external/trashlog.git/proto/generated" ) type Sink struct { logger hlog.Logger store *clickhouse.DAL } type InfoMobileRec struct { Level string TS uint64 Message string Module []string Stacktrace string KeyFields map[string]interface{} CtxFields map[string]interface{} SvcBuildTime uint64 SvcVersion string SvcCommit string SvcFile string SvcLine uint64 } func (s *Sink) TgLog(_ context.Context, record *trashlogProto.Record) (*trashlogProto.Dummy, error) { data := Dto2daoRecord(record) s.logger.Emit(InfoMobileRec{ Level: data.Level, TS: data.TS, Message: data.Message, Module: data.Module, Stacktrace: data.Stacktrace, KeyFields: data.KeyFields, CtxFields: data.CtxFields, SvcBuildTime: data.SvcBuildTime, SvcVersion: data.SvcVersion, SvcCommit: data.SvcCommit, SvcFile: data.SvcFile, SvcLine: data.SvcLine, }) return &trashlogProto.Dummy{}, nil } func New(log hlog.Logger, store *clickhouse.DAL) *Sink { return &Sink{ logger: log.Module("sink"), store: store, } } func (s *Sink) GetFields( _ context.Context, _ *trashlogProto.Dummy, ) (*trashlogProto.Fields2Add, error) { var result trashlogProto.Fields2Add result.Fields = make(map[string]string) result.Fields = s.store.Schema return &result, nil } func (s *Sink) Modify( ctx context.Context, fields *trashlogProto.Fields2Add, ) (*trashlogProto.NotModified, error) { if err := s.store.AddColumn(ctx, fields.Fields); err != nil { return &trashlogProto.NotModified{Columns: fields.Fields}, err } return &trashlogProto.NotModified{}, nil } func (s *Sink) Valve(stream trashlogProto.Trashlog_ValveServer) error { ctxChan := stream.Context() dataChan := make(chan *trashlogProto.Record) go func() { for { record, err := stream.Recv() if err != nil { return } dataChan <- record } }() for { select { case <-ctxChan.Done(): if err := stream.SendAndClose(&trashlogProto.Dummy{}); err != nil { return err } return nil case record := <-dataChan: s.store.PutRecord(Dto2daoRecord(record)) } } } func Dto2daoRecord(in *trashlogProto.Record) model.Record { keyFields := map[string]interface{}{} for key, value := range in.KeyFields { keyFields[key] = Dto2daoValue(value) } ctxFields := map[string]interface{}{} for key, value := range in.CtxFields { ctxFields[key] = Dto2daoValue(value) } return model.Record{ Level: in.Level, TS: in.TS, Message: in.Message, Module: in.Module, Stacktrace: in.Stacktrace, SvcBuildTime: in.SvcFields.BuildTime, SvcVersion: in.SvcFields.Version, SvcCommit: in.SvcFields.Commit, SvcFile: in.SvcFields.File, SvcLine: in.SvcFields.Line, KeyFields: keyFields, CtxFields: ctxFields, } } func Dto2daoValue(in *trashlogProto.Value) interface{} { switch in.Value.(type) { case *trashlogProto.Value_Str: return in.GetStr() case *trashlogProto.Value_Double: return in.GetDouble() case *trashlogProto.Value_Num: return in.GetNum() case *trashlogProto.Value_Flag: return in.GetFlag() } return nil }