reworked export to excel

This commit is contained in:
Pavel 2024-03-26 20:48:49 +03:00
parent f044fb553f
commit cea417f85d
4 changed files with 217 additions and 25 deletions

@ -51,10 +51,10 @@ type Options struct {
NumberPort string `env:"PORT" default:"1488"`
CrtFile string `env:"CRT" default:"server.crt"`
KeyFile string `env:"KEY" default:"server.key"`
PostgresCredentials string `env:"PG_CRED" default:"host=localhost port=5432 user=squiz password=Redalert2 dbname=squiz sslmode=disable"`
PostgresCredentials string `env:"PG_CRED" default:"host=localhost port=35432 user=squiz password=Redalert2 dbname=squiz sslmode=disable"`
HubAdminUrl string `env:"HUB_ADMIN_URL" default:"http://localhost:8001/"`
ServiceName string `env:"SERVICE_NAME" default:"squiz"`
AuthServiceURL string `env:"AUTH_URL"`
AuthServiceURL string `env:"AUTH_URL" default:"http://localhost:8000/"`
RedirectURL string `env:"REDIRECT_URL" default:"https://squiz.pena.digital"`
PubKey string `env:"PUBLIC_KEY"`
PrivKey string `env:"PRIVATE_KEY"`

13
go.mod

@ -10,9 +10,9 @@ require (
github.com/pioz/faker v1.7.3
github.com/skeris/appInit v1.0.2
github.com/stretchr/testify v1.8.4
github.com/tealeg/xlsx v1.0.5
github.com/themakers/bdd v0.0.0-20210316111417-6b1dfe326f33
github.com/themakers/hlog v0.0.0-20191205140925-235e0e4baddf
github.com/xuri/excelize/v2 v2.8.1
go.uber.org/zap v1.26.0
penahub.gitlab.yandexcloud.net/backend/penahub_common v0.0.0-20240202120244-c4ef330cfe5d
penahub.gitlab.yandexcloud.net/backend/quiz/common.git v0.0.0-20240325084116-830f54870853
@ -35,16 +35,23 @@ require (
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/richardlehane/mscfb v1.0.4 // indirect
github.com/richardlehane/msoleps v1.0.3 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/rs/xid v1.5.0 // indirect
github.com/tealeg/xlsx v1.0.5 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasthttp v1.51.0 // indirect
github.com/valyala/tcplisten v1.0.0 // indirect
github.com/xuri/efp v0.0.0-20231025114914-d1ff6096ae53 // indirect
github.com/xuri/nfp v0.0.0-20230919160717-d98342af3f05 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.10.0 // indirect
golang.org/x/net v0.18.0 // indirect
golang.org/x/sys v0.15.0 // indirect
golang.org/x/crypto v0.19.0 // indirect
golang.org/x/net v0.21.0 // indirect
golang.org/x/sys v0.17.0 // indirect
golang.org/x/text v0.14.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect
google.golang.org/grpc v1.61.1 // indirect

25
go.sum

@ -74,6 +74,8 @@ github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZ
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
@ -93,6 +95,11 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM=
github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk=
github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
github.com/richardlehane/msoleps v1.0.3 h1:aznSZzrwYRl3rLKRT3gUk9am7T/mLNSnJINvN0AQoVM=
github.com/richardlehane/msoleps v1.0.3/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@ -118,6 +125,12 @@ github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1S
github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g=
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
github.com/xuri/efp v0.0.0-20231025114914-d1ff6096ae53 h1:Chd9DkqERQQuHpXjR/HSV1jLZA6uaoiwwH3vSuF3IW0=
github.com/xuri/efp v0.0.0-20231025114914-d1ff6096ae53/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI=
github.com/xuri/excelize/v2 v2.8.1 h1:pZLMEwK8ep+CLIUWpWmvW8IWE/yxqG0I1xcN6cVMGuQ=
github.com/xuri/excelize/v2 v2.8.1/go.mod h1:oli1E4C3Pa5RXg1TBXn4ENCXDV5JUMlBluUhG7c+CEE=
github.com/xuri/nfp v0.0.0-20230919160717-d98342af3f05 h1:qhbILQo1K3mphbwKh1vNm4oGezE1eF9fQWmNiIpSfI4=
github.com/xuri/nfp v0.0.0-20230919160717-d98342af3f05/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ=
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
@ -132,6 +145,10 @@ go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4=
golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU=
@ -139,8 +156,8 @@ golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=
golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -149,8 +166,8 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=

@ -1,15 +1,35 @@
package tools
import (
"github.com/tealeg/xlsx"
"github.com/xuri/excelize/v2"
_ "image/gif"
_ "image/jpeg"
_ "image/png"
"io"
"io/ioutil"
"net/http"
"net/url"
"path/filepath"
"penahub.gitlab.yandexcloud.net/backend/quiz/common.git/model"
"regexp"
"sort"
"strconv"
"strings"
)
const (
bucketImages = "squizimages"
bucketFonts = "squizfonts"
bucketScripts = "squizscript"
bucketStyle = "squizstyle"
bucketAnswers = "squizanswer"
)
func WriteDataToExcel(buffer io.Writer, questions []model.Question, answers []model.Answer) error {
file := xlsx.NewFile()
sheet, err := file.AddSheet("Results")
file := excelize.NewFile()
sheet := "Sheet1"
_, err := file.NewSheet(sheet)
if err != nil {
return err
}
@ -34,10 +54,11 @@ func WriteDataToExcel(buffer io.Writer, questions []model.Question, answers []mo
headers = append(headers, "Результат")
// добавляем заголовки в первую строку
row := sheet.AddRow()
for _, header := range headers {
cell := row.AddCell()
cell.Value = header
for col, header := range headers {
cell := ToAlphaString(col+1) + "1"
if err := file.SetCellValue(sheet, cell, header); err != nil {
return err
}
}
// мапа для хранения обычных ответов респондентов
@ -49,18 +70,20 @@ func WriteDataToExcel(buffer io.Writer, questions []model.Question, answers []mo
// заполняем мапу ответами и данными респондентов
for _, answer := range answers {
if answer.Result {
// если это результат то данные респондента берутся из контента ответа по сессии
results[answer.Session] = answer
} else {
// если это обычный ответ то добавляем его в соответствующий список ответов респондента
standart[answer.Session] = append(standart[answer.Session], answer)
}
}
// записываем данные в файл
row := 2
for session, _ := range results {
response := standart[session]
row := sheet.AddRow()
row.AddCell().Value = results[session].Content // данные респондента
if err := file.SetCellValue(sheet, "A"+strconv.Itoa(row), results[session].Content); err != nil {
return err
}
count := 2
for _, q := range questions {
if !q.Deleted && q.Type != model.TypeResult {
sort.Slice(response, func(i, j int) bool {
@ -68,18 +91,68 @@ func WriteDataToExcel(buffer io.Writer, questions []model.Question, answers []mo
})
index := binarySearch(response, q.Id)
if index != -1 {
row.AddCell().Value = response[index].Content
cell := ToAlphaString(count) + strconv.Itoa(row)
typeMap := FileSearch(response[index].Content)
for _, tipe := range typeMap {
if tipe != "Text" && q.Type == model.TypeImages || q.Type == model.TypeVarImages {
urle := ExtractImageURL(response[index].Content)
urlData := strings.Split(urle, " ")
for _, k := range urlData {
u, err := url.Parse(k)
if err == nil && u.Scheme != "" && u.Host != "" {
picture, err := downloadImage(k)
if err != nil {
return err
}
file.SetColWidth(sheet, ToAlphaString(count), ToAlphaString(count), 50)
file.SetRowHeight(sheet, row, 150)
if err := file.AddPictureFromBytes(sheet, cell, picture); err != nil {
return err
}
} else {
if err := file.SetCellValue(sheet, cell, k); err != nil {
return err
}
}
}
} else if tipe != "Text" && q.Type == model.TypeFile {
urle := ExtractImageURL(response[index].Content)
display, tooltip := urle, urle
if err := file.SetCellValue(sheet, cell, response[index].Content); err != nil {
return err
}
if err := file.SetCellHyperLink(sheet, cell, urle, "External", excelize.HyperlinkOpts{
Display: &display,
Tooltip: &tooltip,
}); err != nil {
return err
}
} else if q.Type != model.TypeFile {
if err := file.SetCellValue(sheet, cell, response[index].Content); err != nil {
return err
}
}
}
} else {
row.AddCell().Value = "-"
cell := ToAlphaString(count) + strconv.Itoa(row)
if err := file.SetCellValue(sheet, cell, "-"); err != nil {
return err
}
}
count++
}
}
row.AddCell().Value = mapQueRes[results[session].QuestionId]
cell := ToAlphaString(len(headers)) + strconv.Itoa(row)
if err := file.SetCellValue(sheet, cell, mapQueRes[results[session].QuestionId]); err != nil {
return err
}
row++
}
// cохраняем данные в буфер
err = file.Write(buffer)
if err != nil {
if err := file.Write(buffer); err != nil {
return err
}
@ -101,3 +174,98 @@ func binarySearch(answers []model.Answer, questionID uint64) int {
}
return -1
}
func FileSearch(content string) map[string]string {
types := make(map[string]string)
words := strings.Fields(content)
for _, word := range words {
if strings.Contains(word, bucketImages) {
types[word] = FileType(word)
} else if strings.Contains(word, bucketFonts) {
types[word] = FileType(word)
} else if strings.Contains(word, bucketScripts) {
types[word] = FileType(word)
} else if strings.Contains(word, bucketStyle) {
types[word] = FileType(word)
} else if strings.Contains(word, bucketAnswers) {
types[word] = FileType(word)
} else {
types[word] = "Text"
}
}
return types
}
func FileType(filename string) string {
parts := strings.Split(filename, ".")
extension := parts[len(parts)-1]
switch extension {
case "png", "jpg", "jpeg", "gif", "bmp", "svg", "webp", "tiff", "ico":
return "Image"
default:
return "File"
}
}
func downloadImage(url string) (*excelize.Picture, error) {
resp, err := http.Get(url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
imgData, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
ext := filepath.Ext(url)
if ext == "" {
contentType := resp.Header.Get("Content-Type")
switch {
case strings.HasPrefix(contentType, "image/jpeg"):
ext = ".jpg"
case strings.HasPrefix(contentType, "image/png"):
ext = ".png"
default:
ext = ".png"
}
}
pic := &excelize.Picture{
Extension: ext,
File: imgData,
Format: &excelize.GraphicOptions{
AutoFit: true,
Positioning: "oneCell",
},
}
return pic, nil
}
func ToAlphaString(col int) string {
var result string
for col > 0 {
col--
result = string(rune('A'+col%26)) + result
col /= 26
}
return result
}
func ExtractImageURL(htmlContent string) string {
re := regexp.MustCompile(`(?:<img[^>]*src="([^"]+)"[^>]*>)|(?:<td[^>]*>.*?<img[^>]*src="([^"]+)"[^>]*>.*?</td>)|(?:<tr[^>]*>.*?<td[^>]*>.*?<img[^>]*src="([^"]+)"[^>]*>.*?</td>.*?</tr>)|(?:<a[^>]*\s+download[^>]*>([^<]+)<\/a>)`)
matches := re.FindAllStringSubmatch(htmlContent, -1)
for _, match := range matches {
for i := 1; i < len(match); i++ {
if match[i] != "" {
return strings.TrimSpace(match[i])
}
}
}
return htmlContent
}