csv_exporter

gin/request/request.go

+package request
+
+import (
+	"math"
+
+	"gitlab.wanzhuangkj.com/tush/xpkg/pkg/errcode"
+	"gitlab.wanzhuangkj.com/tush/xpkg/pkg/xerrors/xerror"
+	"gitlab.wanzhuangkj.com/tush/xpkg/utils/xsf"
+)
+
+const (
+	LastIDLimit = 1000
+)
+
+var (
+	LastIDMax = xsf.ParseInt64(math.MaxInt64)
+)
+
+type Params struct {
+	PageIndex int      `json:"page"  comment:"page" encomment:"page" form:"page" validate:"required" example:"1"`                  // page number, starting from page 1
+	PageSize  int      `json:"pageSize"  comment:"pageSize" encomment:"pageSize" form:"pageSize" validate:"required" example:"10"` // lines per page
+	Sort      string   `json:"sort,omitempty" form:"sort,omitempty" validate:"required" example:"id"`                              // sorted fields, multi-column sorting separated by commas
+	Columns   []Column `json:"columns,omitempty" form:"columns,omitempty"`                                                         // query conditions
+}
+
+type Column struct {
+	Name  string      `json:"name"  comment:"name" encomment:"name" form:"name"  validate:"required" example:"id"`     // column name
+	Exp   string      `json:"exp"  comment:"exp" encomment:"exp" form:"exp"  validate:"required" example:"="`          // expressions, which default to = when the value is null, have =, !=, >, >=, <, <=, like
+	Value interface{} `json:"value"  comment:"value" encomment:"value" form:"value" validate:"required" example:"102"` // column value
+	Logic string      `json:"logic"  comment:"logic" encomment:"logic" form:"logic" example:"and"`                     // logical type, default value is "and", support &, and, ||, or
+}
+
+type Conditions struct {
+	Columns []Column `json:"columns"  comment:"columns" encomment:"columns" form:"columns"` // columns info
+}
+
+type BaseCreateReply struct {
+	ID xsf.ID `json:"id"  comment:"id" encomment:"id" form:"id" example:"1924346047086211072"` // id
+}
+
+type BaseIDReq struct {
+	ID xsf.ID `json:"id"  comment:"id" encomment:"id" form:"id" swaggertype:"string" validate:"required" example:"1"` // id
+}
+
+func (x *BaseIDReq) Valid() error {
+	if x.ID <= 0 {
+		return xerror.NewC(errcode.InvalidParams.Code(), "参数[id]不合法")
+	}
+	return nil
+}
+
+type BaseIDsReq struct {
+	IDs []xsf.ID `json:"ids"  comment:"ids" encomment:"ids" swaggertype:"array,string" form:"ids" validate:"required" example:"1" binding:"min=1"` // id list
+}
+
+func (x BaseIDsReq) Valid() error {
+	if len(x.IDs) == 0 {
+		return xerror.NewC(errcode.InvalidParams.Code(), "参数[ids]不能为空")
+	}
+	return nil
+}
+
+type GetByLastIDReq struct {
+	LastID xsf.ID `json:"lastID" form:"lastID" validate:"required" example:"10086"` // 最后一个id
+	Limit  int    `json:"limit" form:"limit" validate:"required" example:"1000"`    // 限制数量
+	Sort   string `json:"sort" form:"sort" example:"id asc"`                        // 排序
+}
+
+func (x *GetByLastIDReq) Valid() error {
+	return nil
+}
+
+func (r *GetByLastIDReq) GetLastID() xsf.ID {
+	if r.LastID <= 0 {
+		r.LastID = LastIDMax
+	}
+	return r.LastID
+}
+
+func (r *GetByLastIDReq) GetLimit() int {
+	if r.Limit <= 0 {
+		r.Limit = LastIDLimit
+	}
+	return r.Limit
+}
+
+func (r *GetByLastIDReq) GetSort() string {
+	if r.Sort == "" {
+		r.Sort = "id asc"
+	}
+	return r.Sort
+}

gin/response/response.go

@@ -20,6 +20,19 @@ import (
 
 const CustomErrorCode = 0
 
+type BasePageReply struct {
+	BaseListReply
+	Total int64 `json:"total"  comment:"total" encomment:"total" form:"total" example:"2000"` //
+}
+
+type BaseListReply struct {
+	List []any `json:"list"  comment:"list" encomment:"list" form:"list"` //
+}
+
+type BaseReply struct {
+	Result
+}
+
 // Result output data format
 type Result struct {
 	Code      int         `json:"code" form:"code" example:"1"`

go.mod

@@ -59,6 +59,7 @@ require (
 	github.com/dlclark/regexp2 v1.11.5
 	github.com/duke-git/lancet/v2 v2.3.4
 	github.com/go-redsync/redsync/v4 v4.13.0
+	github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1
 	github.com/hashicorp/consul/api v1.12.0
 	github.com/mojocn/base64Captcha v1.3.8
 	github.com/nacos-group/nacos-sdk-go v1.1.6

go.sum

@@ -207,6 +207,8 @@ github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB
 github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
 github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 h1:FWNFq4fM1wPfcK40yHE5UO3RUdSNPaBC+j3PokzA6OQ=
+github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI=
 github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=

pkg/gocrypto/aes.go

@@ -8,7 +8,7 @@ import (
 	"encoding/hex"
 	"errors"
 
-	"gitlab.wanzhuangkj.com/tush/xpkg/gocrypto/wcipher"
+	"gitlab.wanzhuangkj.com/tush/xpkg/pkg/gocrypto/wcipher"
 )
 
 // AesEncrypt aes encryption, returns ciphertext is not transcoded

pkg/gocrypto/des.go

@@ -6,7 +6,7 @@ package gocrypto
 import (
 	"encoding/hex"
 
-	"gitlab.wanzhuangkj.com/tush/xpkg/gocrypto/wcipher"
+	"gitlab.wanzhuangkj.com/tush/xpkg/pkg/gocrypto/wcipher"
 )
 
 // DesEncrypt des encryption, the returned ciphertext is not transcoded

utils/csv_exporter/csv_exporter.go

+package csvexporter
+
+import (
+	"bytes"
+
+	"github.com/gocarina/gocsv"
+	"github.com/jinzhu/copier"
+	"gitlab.wanzhuangkj.com/tush/xpkg/config"
+	"gitlab.wanzhuangkj.com/tush/xpkg/pkg/oss"
+	"gitlab.wanzhuangkj.com/tush/xpkg/xcommon/odao"
+)
+
+type CSVExporter[T any] struct {
+	pageSize int
+	page     PageFn[T]
+	total    TotalFn
+	fileURL  FileURLFn
+}
+
+type PageFn[T any] func(offset int) ([]*T, error)
+
+type TotalFn func() (int64, error)
+
+type FileURLFn func() string
+
+type IPageReq interface {
+	odao.Query
+}
+
+func New[T any](pageSize int, total TotalFn, page PageFn[T], fileURL FileURLFn) *CSVExporter[T] {
+	return &CSVExporter[T]{
+		pageSize: pageSize,
+		total:    total,
+		page:     page,
+		fileURL:  fileURL,
+	}
+}
+
+func (x *CSVExporter[T]) Export() (string, error) {
+	total, err := x.total()
+	if err != nil {
+		return "", err
+	}
+	var rs []*T
+	if total > 0 {
+		rs, err = x.nestedLoop(total)
+		if err != nil {
+			return "", err
+		}
+	}
+	csvBytes, err := gocsv.MarshalBytes(&rs)
+	if err != nil {
+		return "", err
+	}
+	return x.upload(x.fileURL(), bytes.NewBuffer(csvBytes))
+}
+
+func (x *CSVExporter[T]) nestedLoop(total int64) ([]*T, error) {
+	ret := make([]*T, 0, total)
+	pageIndex := 1
+	offset := 0
+	for {
+		offset = (pageIndex - 1) * x.pageSize
+		rs, err := x.page(offset)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(rs) == 0 {
+			break
+		}
+		ret = append(ret, rs...)
+		pageIndex++
+	}
+	return ret, nil
+}
+
+func (x *CSVExporter[T]) upload(fileURL string, buffer *bytes.Buffer) (fileUrl string, err error) {
+	ossCfg := oss.AliOssConfig{}
+	config.Read(func(c *config.Config) {
+		copier.Copy(&ossCfg, c.Oss)
+	})
+	if fileUrl, _, err = oss.OSS.UploadFile(buffer, fileURL, &ossCfg); err != nil {
+		return "", err
+	}
+	return fileUrl, nil
+}