增加gormutils用于构建复杂where条件，适应多表联查场景

cmd/pagination/cmd.go

@@ -84,7 +84,7 @@ func main1() {
 		SiteID:       1320976731124793344,
 		Name:         "优盟店",
 	}
-	dsn := "qitu_test:By1rGembg6@tcp(rm-bp15831972934dz2c.mysql.rds.aliyuncs.com:3306)/qitu_athena_test?charset=utf8mb4&parseTime=True&loc=Local"
+	dsn := "root:123456@tcp(localhost:30006)/qitu_athena?charset=utf8mb4&parseTime=True&loc=Local"
 	// 方法2：全局DryRun配置
 	dryRunDB, _ := gorm.Open(mysql.Open(dsn), &gorm.Config{Logger: newLogger})
 	db = dryRunDB

pkg/errcode/http_error.go

@@ -88,6 +88,10 @@ func (e *Error) ToXerror() (int, string) {
 	return e.code, e.msg
 }
 
+func (e *Error) ToError() (int, string) {
+	return e.code, e.msg
+}
+
 // NeedHTTPCode need to convert to standard http code
 func (e *Error) NeedHTTPCode() bool {
 	return e.needHTTPCode

pkg/errcode/http_system_code.go

@@ -32,8 +32,8 @@ var (
 	TooEarly            = NewError(100025, "Too Early")
 
 	// 已下错误不会返给前端
-	DBError    = NewError(160001, "DB Error")
-	CacheError = NewError(160002, "Cache Error")
-	NetError   = NewError(160003, "Net Error")
-	FileError  = NewError(160004, "File Error")
+	DBError    = NewError(110001, "DB Error")
+	CacheError = NewError(110002, "Cache Error")
+	NetError   = NewError(110003, "Net Error")
+	FileError  = NewError(110004, "File Error")
 )

xcommon/odao/odao.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
-	"strings"
 
 	"gitlab.wanzhuangkj.com/tush/xpkg/database"
 	"gitlab.wanzhuangkj.com/tush/xpkg/pkg/goredis"
@@ -317,42 +316,6 @@ func (x *ODao[T]) Page(ctx context.Context, where IPageParams) (rs []*T, total i
 	return rs, total, nil
 }
 
-func (x *ODao[T]) JPage(ctx context.Context, where IPageParams, result any, resultType any) (total int64, err error) {
-	condition := x.DB().WithContext(ctx).Model(new(T)).Scopes(x.MakeCondition(where))
-	if where.Unscoped() {
-		condition = condition.Unscoped()
-	}
-	if err = condition.Limit(-1).Offset(-1).Count(&total).Error; err != nil {
-		return total, xerror.NewDBError(err.Error())
-	}
-	if total == 0 {
-		return total, nil
-	}
-	fields := GetGormTags(resultType)
-	if err = condition.Select(strings.Join(fields, ",")).Limit(where.GetLimit()).Offset(where.GetOffset()).Scan(result).Error; err != nil {
-		return total, xerror.NewDBError(err.Error())
-	}
-	return total, nil
-}
-
-func (x *ODao[T]) JList(ctx context.Context, where IListParams, result any, resultType any) (total int64, err error) {
-	condition := x.DB().WithContext(ctx).Model(new(T)).Scopes(x.MakeCondition(where))
-	if where.Unscoped() {
-		condition = condition.Unscoped()
-	}
-	if err = condition.Limit(-1).Offset(-1).Count(&total).Error; err != nil {
-		return total, xerror.NewDBError(err.Error())
-	}
-	if total == 0 {
-		return total, nil
-	}
-	fields := GetGormTags(resultType)
-	if err = condition.Select(strings.Join(fields, ",")).Scan(result).Error; err != nil {
-		return total, xerror.NewDBError(err.Error())
-	}
-	return total, nil
-}
-
 func (x *ODao[T]) GetPageByColumns(ctx context.Context, params *query.Params) (rs []*T, total int64, err error) {
 	queryStr, args, err := params.ConvertToGormConditions()
 	if err != nil {

xcommon/odao/query.go

@@ -41,6 +41,7 @@ type Query interface {
 const (
 	// FromQueryTag tag标记
 	FromQueryTag  = "query"
+	FromQueryTag2 = "gq"
 	// Mysql 数据库标识
 	Mysql = "mysql"
 	// Postgres 数据库标识
@@ -89,15 +90,21 @@ func ResolveSearchQuery(driver string, q any, condition Condition, pTName string
 		fieldName := field.Name
 		fieldValue := qValue.Field(i)
 		kind := field.Type.Kind()
+
+		tag, ok = field.Tag.Lookup(FromQueryTag2)
+		if !ok {
 			tag, ok = field.Tag.Lookup(FromQueryTag)
 			if !ok {
 				//递归调用
 				ResolveSearchQuery(driver, fieldValue.Interface(), condition, tname)
 				continue
 			}
+		}
+
 		if tag == "-" {
 			continue
 		}
+
 		t = parseTag(tag)
 		if t.Join == "" {
 			if !fieldValue.IsValid() {
@@ -224,10 +231,7 @@ func pgSql(driver string, t *resolveSearchTag, condition Condition, qValue refle
 		// join := condition.SetJoinOn(t.Type, fmt.Sprintf(
 		// 	"left join %s on `%s`.`%s` = `%s`.`%s`", t.Join, t.Join, t.On[0], t.Table, t.On[1],
 		// ))
-		join := condition.SetJoinOn(t.Type, fmt.Sprintf(
-			"%s",
-			t.Join,
-		))
+		join := condition.SetJoinOn(t.Type, fmt.Sprintf("%s", t.Join))
 		ResolveSearchQuery(driver, qValue.Field(i).Interface(), join, tname)
 		return
 	default:

xcron/service/cron_job.go

@@ -2,7 +2,6 @@ package service
 
 import (
 	"context"
-	"errors"
 	"fmt"
 
 	"gitlab.wanzhuangkj.com/tush/xpkg/xcommon"
@@ -61,7 +60,7 @@ func (x *cronJobService) DeleteByID(ctx context.Context, id xsf.ID) error {
 		return err
 	}
 	if cronJob == nil {
-		return errors.New(fmt.Sprintf("未查询到%s[id:%d]", models.CronJobTool.TableName(), id))
+		return fmt.Errorf("未查询到%s[id:%d]", models.CronJobTool.TableName(), id)
 	}
 	return dao.CronJobDao.DeleteByID(ctx, id)
 }
@@ -106,7 +105,7 @@ func (x *cronJobService) UpdateByID(ctx context.Context, req *types.CronJobUpdat
 		return err
 	}
 	if cronJob == nil {
-		return errors.New(fmt.Sprintf("未查询到%s[id:%d]", models.CronJobTool.TableName(), req.ID))
+		return fmt.Errorf("未查询到%s[id:%d]", models.CronJobTool.TableName(), req.ID)
 	}
 	cronJobUpd := &models.CronJob{}
 	_ = copier.Copy(cronJobUpd, req)

xutils/gormutils/gormutils.go

+package gormutils
+
+import (
+	"fmt"
+	"reflect"
+	"regexp"
+	"strings"
+
+	"gorm.io/gorm"
+)
+
+const (
+	Tag1 = "gq"
+	Tag2 = "query"
+)
+
+func Scopes(where any) func(db *gorm.DB) *gorm.DB {
+	return func(db *gorm.DB) *gorm.DB {
+		return makeScopes(db, where)
+	}
+}
+
+func makeScopes(db *gorm.DB, where any) *gorm.DB {
+	condition := &GormCondition{}
+
+	ResolveSearchQuery(where, condition)
+
+	for k, v := range condition.Where {
+		db = db.Where(k, v...)
+	}
+	for k, v := range condition.Or {
+		db = db.Or(k, v...)
+	}
+	for _, o := range condition.Order {
+		db = db.Order(o)
+	}
+
+	return db
+}
+
+type GormCondition struct {
+	Where map[string][]any
+	Order []string
+	Or    map[string][]any
+}
+
+type Condition interface {
+	SetWhere(k string, v []any)
+	SetOr(k string, v []any)
+	SetOrder(k string)
+	SetJoinOn(t, on string) Condition
+}
+
+func (e *GormCondition) SetJoinOn(t, on string) Condition {
+	return nil
+}
+
+func (e *GormCondition) SetWhere(k string, v []any) {
+	if e.Where == nil {
+		e.Where = make(map[string][]any)
+	}
+	e.Where[k] = v
+}
+
+func (e *GormCondition) SetOr(k string, v []any) {
+	if e.Or == nil {
+		e.Or = make(map[string][]any)
+	}
+	e.Or[k] = v
+}
+
+func (e *GormCondition) SetOrder(k string) {
+	if e.Order == nil {
+		e.Order = make([]string, 0)
+	}
+	e.Order = append(e.Order, k)
+}
+
+type JoinCondition struct {
+	Type   string
+	JoinOn string
+	GormCondition
+}
+
+// ResolveSearchQuery 解析
+/**
+ * 	eq  等于(默认不填都可以)
+ * 	like  包含
+ *	gt / gte 大于 / 大于等于
+ *	lt / lte 小于 / 小于等于
+ *	left  / ileft ：like xxx%
+ *	right / iright  : like %xxx
+ *	in
+ *	isnull
+ *  order 排序		e.g. order[key]=desc     order[key]=asc
+ */
+func ResolveSearchQuery(q any, condition Condition) {
+	qType := reflect.TypeOf(q)
+	qValue := reflect.ValueOf(q)
+
+	var tag string
+	var ok bool
+	var t *resolveSearchTag
+	var tname string
+
+	if qType.Kind() == reflect.Ptr {
+		qType = qType.Elem()
+		qValue = qValue.Elem()
+	}
+
+	if qType.Kind() != reflect.Struct {
+		return
+	}
+
+	for i := 0; i < qType.NumField(); i++ {
+
+		field := qType.Field(i)
+		fieldName := field.Name
+		fieldValue := qValue.Field(i)
+		kind := field.Type.Kind()
+		tag, ok = field.Tag.Lookup(Tag1)
+		if !ok {
+			tag, ok = field.Tag.Lookup(Tag2)
+			if !ok {
+				ResolveSearchQuery(fieldValue.Interface(), condition)
+				continue
+			}
+		}
+
+		if tag == "-" {
+			continue
+		}
+
+		t = parseTag(tag)
+		if t.Join == "" {
+			if !fieldValue.IsValid() {
+				continue
+			}
+
+			switch fieldValue.Kind() {
+			case reflect.Ptr, reflect.Slice, reflect.Map, reflect.Chan, reflect.Interface:
+				if fieldValue.IsNil() {
+					continue
+				}
+			default:
+				if fieldValue.IsZero() {
+					continue
+				}
+			}
+		}
+
+		if t.Column == "" {
+			if kind == reflect.Array || kind == reflect.Slice {
+				if strings.HasSuffix(fieldName, "s") {
+					t.Column = snakeCase(strings.TrimSuffix(fieldName, "s"), false)
+				} else {
+					t.Column = snakeCase(fieldName, false)
+				}
+			} else {
+				t.Column = snakeCase(fieldName, false)
+			}
+		}
+
+		if t.Table == "" {
+			t.Table = tname
+		}
+
+		//解析 Postgres `语法不支持，单独适配
+		otherSql(t, condition, qValue, i)
+	}
+}
+
+func otherSql(t *resolveSearchTag, condition Condition, qValue reflect.Value, i int) {
+	if t.Type == "" {
+		condition.SetWhere(fmt.Sprintf("`%s`.`%s` = ?", t.Table, t.Column), []any{qValue.Field(i).Interface()})
+		return
+	}
+	qtag := QueryTag(t.Type)
+	switch qtag {
+	case EQ:
+		condition.SetWhere(fmt.Sprintf("`%s`.`%s` = ?", t.Table, t.Column), []any{qValue.Field(i).Interface()})
+		return
+	case GT:
+		condition.SetWhere(fmt.Sprintf("`%s`.`%s` > ?", t.Table, t.Column), []any{qValue.Field(i).Interface()})
+		return
+	case GTE:
+		condition.SetWhere(fmt.Sprintf("`%s`.`%s` >= ?", t.Table, t.Column), []any{qValue.Field(i).Interface()})
+		return
+	case LT:
+		condition.SetWhere(fmt.Sprintf("`%s`.`%s` < ?", t.Table, t.Column), []any{qValue.Field(i).Interface()})
+		return
+	case LTE:
+		condition.SetWhere(fmt.Sprintf("`%s`.`%s` <= ?", t.Table, t.Column), []any{qValue.Field(i).Interface()})
+		return
+	case LEFT:
+		condition.SetWhere(fmt.Sprintf("`%s`.`%s` like ?", t.Table, t.Column), []any{qValue.Field(i).String() + "%"})
+		return
+	case LIKE:
+		condition.SetWhere(fmt.Sprintf("`%s`.`%s` like ?", t.Table, t.Column), []any{"%" + qValue.Field(i).String() + "%"})
+		return
+	case RIGHT:
+		condition.SetWhere(fmt.Sprintf("`%s`.`%s` like ?", t.Table, t.Column), []any{"%" + qValue.Field(i).String()})
+		return
+	case IN:
+		condition.SetWhere(fmt.Sprintf("`%s`.`%s` in (?)", t.Table, t.Column), []any{qValue.Field(i).Interface()})
+		return
+	case ISNULL:
+		if !(qValue.Field(i).IsZero() && qValue.Field(i).IsNil()) {
+			condition.SetWhere(fmt.Sprintf("`%s`.`%s` is null", t.Table, t.Column), make([]any, 0))
+		}
+		return
+	case ISNOTNULL:
+		if !(qValue.Field(i).IsZero() && qValue.Field(i).IsNil()) {
+			condition.SetWhere(fmt.Sprintf("%s.%s is not null", t.Table, t.Column), make([]any, 0))
+		}
+		return
+	case ORDER:
+		val := strings.TrimSpace(qValue.Field(i).String())
+		if val != "" {
+			order, success := parseOrder(val)
+			if success {
+				if !strings.Contains(order, ",") && !strings.Contains(order, ".") {
+					condition.SetOrder(fmt.Sprintf("%s.%s", t.Table, order))
+				} else {
+					condition.SetOrder(order)
+				}
+			} else {
+				switch strings.ToLower(qValue.Field(i).String()) {
+				case "desc", "asc":
+					condition.SetOrder(fmt.Sprintf("`%s`.`%s` %s", t.Table, t.Column, qValue.Field(i).String()))
+				}
+			}
+		} else {
+			switch strings.ToLower(qValue.Field(i).String()) {
+			case "desc", "asc":
+				condition.SetOrder(fmt.Sprintf("`%s`.`%s` %s", t.Table, t.Column, qValue.Field(i).String()))
+			}
+		}
+		return
+	default:
+		condition.SetWhere(fmt.Sprintf("`%s`.`%s` = ?", t.Table, t.Column), []any{qValue.Field(i).Interface()})
+	}
+}
+
+type resolveSearchTag struct {
+	Type   string
+	Column string
+	Table  string
+	On     string
+	Join   string
+}
+
+// parseTag 解析search的tag标签
+func parseTag(tag string) *resolveSearchTag {
+	r := &resolveSearchTag{}
+	tags := strings.Split(tag, ";")
+	var ts []string
+	for _, t := range tags {
+		ts = strings.Split(t, ":")
+		if len(ts) == 0 {
+			continue
+		}
+		switch ts[0] {
+		case "type":
+			if len(ts) > 1 {
+				r.Type = ts[1]
+			}
+		case "column":
+			if len(ts) > 1 {
+				r.Column = ts[1]
+			}
+		case "table":
+			if len(ts) > 1 {
+				r.Table = ts[1]
+			}
+		case "on":
+			if len(ts) > 1 {
+				r.On = ts[1]
+			}
+		case "join":
+			if len(ts) > 1 {
+				r.Join = ts[1]
+			}
+		}
+	}
+	return r
+}
+
+type QueryTag string
+
+const (
+	EQ        QueryTag = "eq"
+	LIKE      QueryTag = "like"
+	ILIKE     QueryTag = "ilike"
+	LEFT      QueryTag = "left"
+	ILEFT     QueryTag = "ileft"
+	RIGHT     QueryTag = "right"
+	IRIGHT    QueryTag = "iright"
+	GT        QueryTag = "gt"
+	GTE       QueryTag = "gte"
+	LT        QueryTag = "lt"
+	LTE       QueryTag = "lte"
+	IN        QueryTag = "in"
+	ISNULL    QueryTag = "isnull"
+	ISNOTNULL QueryTag = "isnotnull"
+	ORDER     QueryTag = "order"
+)
+
+/**
+ * 驼峰转蛇形 snake string
+ **/
+func snakeCase(s string, allMode bool) string {
+	num := len(s)
+	data := make([]byte, 0, num*2)
+	for i := 0; i < num; i++ {
+		d := s[i]
+		// or通过ASCII码进行大小写的转化
+		// 65-90（A-Z），97-122（a-z）
+		//判断如果字母为大写的A-Z就在前面拼接一个_
+		if d >= 'A' && d <= 'Z' {
+			if i > 0 {
+				if allMode {
+					data = append(data, '_', d+32)
+				} else {
+					if s[i-1] >= 'A' && s[i-1] <= 'Z' {
+						data = append(data, d+32)
+					} else {
+						data = append(data, '_', d+32)
+					}
+				}
+			} else {
+				data = append(data, d+32)
+			}
+		} else {
+			data = append(data, d)
+		}
+	}
+	//ToLower把大写字母统一转小写
+	return string(data[:])
+}
+
+func parseOrder(order string) (string, bool) {
+	if detectSQLInjection(order) {
+		return "", false
+	}
+	columns := strings.Split(order, ",")
+	orderColumns := make([]string, 0, len(columns))
+	for _, column := range columns {
+		column = strings.TrimSpace(column)
+		if len(column) == 0 {
+			continue
+		}
+		order := "asc"
+		if column[:1] == "-" {
+			order = "desc"
+			column = column[1:]
+		}
+		orderColumns = append(orderColumns, column+" "+order)
+	}
+	if len(orderColumns) == 0 {
+		return "", false
+	}
+	return strings.Join(orderColumns, ","), true
+}
+func detectSQLInjection(input string) bool {
+	return detectSQLInjectionRe.MatchString(input)
+}
+
+var (
+	orderReg             = regexp.MustCompile(`order\[([^\]]+)\]=([^=]+)`)
+	detectSQLInjectionRe = regexp.MustCompile(`['";]+|UNION|SELECT|INSERT|UPDATE|DELETE|DROP|GRANT|EXEC|CREATE|ALTER|TRUNCATE|COUNT|\*|--|\/\*|;|\+|\/`)
+)

xutils/gormutils/gormutils_test.go

+package gormutils_test
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"testing"
+	"time"
+
+	"gitlab.wanzhuangkj.com/tush/xpkg/pkg/sgorm/query"
+	"gitlab.wanzhuangkj.com/tush/xpkg/xutils/gormutils"
+	"gitlab.wanzhuangkj.com/tush/xpkg/xutils/xsf"
+	"gorm.io/driver/mysql"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+)
+
+func TestMakeWhere(t *testing.T) {
+	// in := PageReq{}
+	in := PageReq2{}
+	in.PageIndex = 1
+	in.PageSize = 50
+	in.IDs = []xsf.ID{10, 2, 3}
+	in.UserID = 10
+	in.SiteName = "优盟"
+	scopes := gormutils.Scopes(in)
+	conn := getConn()
+	var rs []*User
+	if e := conn.WithContext(context.TODO()).
+		Table("operator as op").
+		Select("op.id as id ,su.site_id as site_id,s.name as site_name").
+		Joins(" left join site_user su on su.user_type = 2 and su.user_id = op.id").
+		Joins(" left join site s on su.site_id = s.id").
+		Scopes(scopes).
+		Limit(in.GetLimit()).
+		Offset(in.GetOffset()).
+		Find(&rs).Error; e != nil {
+		fmt.Println(e.Error())
+	}
+	for _, r := range rs {
+		fmt.Println(r)
+	}
+}
+
+func getConn() *gorm.DB {
+	newLogger := logger.New(
+		log.New(os.Stdout, "\r\n", log.LstdFlags),
+		logger.Config{
+			SlowThreshold: time.Second,
+			LogLevel:      logger.Info, // 设置为Info级别打印所有SQL
+			Colorful:      true,
+		},
+	)
+	dsn := "root:123456@tcp(localhost:30006)/qitu_operator?charset=utf8mb4&parseTime=True&loc=Local"
+	// 方法2：全局DryRun配置
+	con, _ := gorm.Open(mysql.Open(dsn), &gorm.Config{Logger: newLogger})
+	return con
+}
+
+type PageReq struct {
+	query.Pagination
+	IDs      []xsf.ID `json:"ids" gq:"type:in;table:op;column:id" example:"10"`
+	UserID   xsf.ID   `json:"userID" gq:"table:su;column:user_id" example:"10"`
+	SiteName string   `json:"siteName" gq:"type:left;table:s;column:name" example:"优盟"`
+}
+
+type PageReq2 struct {
+	query.Pagination
+	IDs      []xsf.ID `json:"ids" query:"type:in;table:op;column:id" example:"10"`
+	UserID   xsf.ID   `json:"userID" query:"table:su;column:user_id" example:"10"`
+	SiteName string   `json:"siteName" query:"type:left;table:s;column:name" example:"优盟"`
+}
+
+type User struct {
+	ID       xsf.ID `json:"id" gorm:"id"`
+	SiteID   xsf.ID `json:"siteID" gorm:"site_id"`
+	SiteName string `json:"siteName" gorm:"site_name"`
+}