Skip to content

X
xpkg
提交 469d7176 authored 作者: mooncake's avatar mooncake
浏览文件
操作

update

上级 6f55882a
隐藏空白字符变更
内嵌 并排
正在显示 包含 2 行增加 和 1174 行删除
config/consul.go
浏览文件 @ 469d7176
...@@ -6,6 +6,7 @@ import ( ...@@ -6,6 +6,7 @@ import (
"time" "time"
"gitlab.wanzhuangkj.com/tush/xpkg/pkg/eventbus" "gitlab.wanzhuangkj.com/tush/xpkg/pkg/eventbus"
hashutils "gitlab.wanzhuangkj.com/tush/xpkg/utils/hashutils"
"github.com/hashicorp/consul/api" "github.com/hashicorp/consul/api"
"github.com/spf13/viper" "github.com/spf13/viper"
...@@ -52,7 +53,7 @@ func (x *ConsulConfFetcher) sync() *ConsulConfFetcher { ...@@ -52,7 +53,7 @@ func (x *ConsulConfFetcher) sync() *ConsulConfFetcher {
x.err = xerror.New("[consul]请先配置consul") x.err = xerror.New("[consul]请先配置consul")
return x return x
} }
if hashutil.Md5(kv.Value) != hashutil.Md5(x.cfg.GetContent()) { if hashutils.Md5(kv.Value) != hashutils.Md5(x.cfg.GetContent()) {
x.count++ x.count++
x.isNew = true x.isNew = true
x.cfg.SetContent(kv.Value) x.cfg.SetContent(kv.Value)
......
utils/fileutils/file_path.go deleted 100644 → 0
浏览文件 @ 6f55882a
package fileutils
import (
"os"
"path"
"path/filepath"
"runtime"
"strings"
)
// IsExists determine if a file or folder exists
func IsExists(f string) bool {
_, err := os.Stat(f)
if err != nil {
return !os.IsNotExist(err)
}
return true
}
// GetRunPath get the absolute path of the program execution
func GetRunPath() string {
dir, err := os.Executable()
if err != nil {
return ""
}
return filepath.Dir(dir)
}
// GetFilename get file name
func GetFilename(filePath string) string {
_, name := filepath.Split(filePath)
return name
}
// GetFileSuffixName get file suffix name, example: ".txt"
func GetFileSuffixName(filePath string) string {
return filepath.Ext(filePath)
}
// GetDir get dir, not include the last separator
func GetDir(filePath string) string {
return filepath.Dir(filePath)
}
// GetSuffixDir get suffix dir, not include the last separator
func GetSuffixDir(filePath string) string {
fileInfo, err := os.Stat(filePath)
if err != nil {
return filepath.Base(filePath)
}
if !fileInfo.IsDir() {
filePath = strings.TrimSuffix(filePath, fileInfo.Name())
}
return filepath.Base(filePath)
}
// GetFileDir get dir, include the last separator
func GetFileDir(filePath string) string {
dir, _ := filepath.Split(filePath)
return dir
}
// MakeDir create dir
func MakeDir(dir string) error {
if _, err := os.Stat(dir); os.IsNotExist(err) {
return os.MkdirAll(dir, 0766)
}
return nil
}
// GetFilenameWithoutSuffix get file name without suffix
func GetFilenameWithoutSuffix(filePath string) string {
_, name := filepath.Split(filePath)
return strings.TrimSuffix(name, path.Ext(name))
}
// Join joins any number of path elements into a single path
func Join(elem ...string) string {
dir := strings.Join(elem, "/")
if IsWindows() {
return strings.ReplaceAll(dir, "/", "\\")
}
return dir
}
// IsWindows determining whether a window environment
func IsWindows() bool {
return runtime.GOOS == "windows"
}
// GetPathDelimiter get separator by system type
func GetPathDelimiter() string {
delimiter := "/"
if IsWindows() {
delimiter = "\\"
}
return delimiter
}
// ListFiles iterates over all files in the specified directory, returning the absolute path to the file
func ListFiles(dirPath string, opts ...Option) ([]string, error) {
files := []string{}
err := error(nil)
o := defaultOptions()
o.apply(opts...)
if !o.noAbsolutePath {
dirPath, err = filepath.Abs(dirPath)
if err != nil {
return files, err
}
}
switch o.filter {
case prefix:
return files, walkDirWithFilter(dirPath, &files, matchPrefix(o.name))
case suffix:
return files, walkDirWithFilter(dirPath, &files, matchSuffix(o.name))
case contain:
return files, walkDirWithFilter(dirPath, &files, matchContain(o.name))
}
return files, walkDir(dirPath, &files)
}
// ListDirsAndFiles iterates through all subdirectories of the specified directory, returning the absolute path to the file
func ListDirsAndFiles(dirPath string) (map[string][]string, error) {
df := make(map[string][]string, 2)
dirPath, err := filepath.Abs(dirPath)
if err != nil {
return df, err
}
dirs := []string{}
files := []string{}
err = walkDir2(dirPath, &dirs, &files)
if err != nil {
return df, err
}
df["dirs"] = dirs
df["files"] = files
return df, nil
}
// FuzzyMatchFiles fuzzy matching of documents, * only
func FuzzyMatchFiles(f string) []string {
var files []string
dir, filenameReg := filepath.Split(f)
if !strings.Contains(filenameReg, "*") {
files = append(files, f)
return files
}
lFiles, err := ListFiles(dir)
if err != nil {
return files
}
for _, file := range lFiles {
_, filename := filepath.Split(file)
isMatch, _ := path.Match(filenameReg, filename)
if isMatch {
files = append(files, file)
}
}
return files
}
// ListDirs list all sub dirs, not including itself
func ListDirs(specifiedDir string) ([]string, error) {
dir, err := os.ReadDir(specifiedDir)
if err != nil {
return nil, err
}
var dirs []string
for _, fi := range dir {
if fi.IsDir() {
dirs = append(dirs, filepath.Join(specifiedDir, fi.Name()))
tmpDirs, err := ListDirs(filepath.Join(specifiedDir, fi.Name()))
if err != nil {
return nil, err
}
dirs = append(dirs, tmpDirs...)
}
}
return dirs, nil
}
// FilterDirs filter directories that meet the criteria
func FilterDirs(dirs []string, opts Option) []string {
o := defaultOptions()
o.apply(opts)
var filteredDirs []string
for _, dir := range dirs {
files, err := os.ReadDir(dir)
if err != nil {
continue
}
existDir := map[string]struct{}{}
for _, file := range files {
if file.IsDir() {
continue
}
switch o.filter {
case prefix:
if matchPrefix(o.name)(file.Name()) {
if _, ok := existDir[dir]; !ok {
existDir[dir] = struct{}{}
filteredDirs = append(filteredDirs, dir)
}
}
case suffix:
if matchSuffix(o.name)(file.Name()) {
if _, ok := existDir[dir]; !ok {
existDir[dir] = struct{}{}
filteredDirs = append(filteredDirs, dir)
}
}
case contain:
if matchContain(o.name)(file.Name()) {
if _, ok := existDir[dir]; !ok {
existDir[dir] = struct{}{}
filteredDirs = append(filteredDirs, dir)
}
}
}
}
}
return filteredDirs
}
// iterative traversal of documents with filter conditions
func walkDirWithFilter(dirPath string, allFiles *[]string, filter filterFn) error {
files, err := os.ReadDir(dirPath)
if err != nil {
return err
}
for _, file := range files {
deepFile := dirPath + GetPathDelimiter() + file.Name()
if file.IsDir() {
err = walkDirWithFilter(deepFile, allFiles, filter)
if err != nil {
return err
}
continue
}
if filter(deepFile) {
*allFiles = append(*allFiles, deepFile)
}
}
return nil
}
func walkDir2(dirPath string, allDirs *[]string, allFiles *[]string) error {
files, err := os.ReadDir(dirPath)
if err != nil {
return err
}
for _, file := range files {
deepFile := dirPath + GetPathDelimiter() + file.Name()
if file.IsDir() {
*allDirs = append(*allDirs, deepFile)
err = walkDir2(deepFile, allDirs, allFiles)
if err != nil {
return err
}
continue
}
*allFiles = append(*allFiles, deepFile)
}
return nil
}
type filterFn func(string) bool
// suffix matching
func matchSuffix(suffixName string) filterFn {
return func(filename string) bool {
if suffixName == "" {
return false
}
size := len(filename) - len(suffixName)
if size >= 0 && filename[size:] == suffixName {
return true
}
return false
}
}
// prefix Matching
func matchPrefix(prefixName string) filterFn {
return func(filePath string) bool {
if prefixName == "" {
return false
}
filename := GetFilename(filePath)
size := len(filename) - len(prefixName)
if size >= 0 && filename[:len(prefixName)] == prefixName {
return true
}
return false
}
}
// contains the string
func matchContain(containName string) filterFn {
return func(filePath string) bool {
if containName == "" {
return false
}
filename := GetFilename(filePath)
return strings.Contains(filename, containName)
}
}
// traversing the document by iteration
func walkDir(dirPath string, allFiles *[]string) error {
files, err := os.ReadDir(dirPath)
if err != nil {
return err
}
for _, file := range files {
deepFile := dirPath + GetPathDelimiter() + file.Name()
if file.IsDir() {
err = walkDir(deepFile, allFiles)
if err != nil {
return err
}
continue
}
*allFiles = append(*allFiles, deepFile)
}
return nil
}
// ListSubDirs list all sub dirs that have the specified sub dir, if sub dir is empty, return all sub dirs
func ListSubDirs(root string, subDir string) ([]string, error) {
var dirs []string
err := filepath.Walk(root, func(dirPath string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() && hasSubDir(dirPath, subDir) {
if subDir == "" {
dirs = append(dirs, dirPath)
} else {
dirs = append(dirs, dirPath+GetPathDelimiter()+subDir)
}
}
return nil
})
return dirs, err
}
func hasSubDir(dirPath string, subDir string) bool {
_, err := os.Stat(filepath.Join(dirPath, subDir))
return err == nil || os.IsExist(err)
}
utils/fileutils/option.go deleted 100644 → 0
浏览文件 @ 6f55882a
package fileutils
const (
prefix = "prefix"
suffix = "suffix"
contain = "contain"
)
var (
defaultFilterType = "" // with prefix, suffix, contain, no filter by default
)
type options struct {
filter string
name string
noAbsolutePath bool
}
func defaultOptions() *options {
return &options{
filter: defaultFilterType,
}
}
// Option set the file options.
type Option func(*options)
func (o *options) apply(opts ...Option) {
for _, opt := range opts {
opt(o)
}
}
// WithSuffix set suffix matching
func WithSuffix(name string) Option {
return func(o *options) {
o.filter = suffix
o.name = name
}
}
// WithPrefix set prefix matching
func WithPrefix(name string) Option {
return func(o *options) {
o.filter = prefix
o.name = name
}
}
// WithContain set contain matching
func WithContain(name string) Option {
return func(o *options) {
o.filter = contain
o.name = name
}
}
// WithNoAbsolutePath set no absolute path
func WithNoAbsolutePath() Option {
return func(o *options) {
o.noAbsolutePath = true
}
}
utils/hashutils/md5.go deleted 100644 → 0
浏览文件 @ 6f55882a
package hashutils
import (
"crypto/md5"
"encoding/hex"
)
func Md5(d []byte) string {
hashed := md5.Sum(d)
return hex.EncodeToString(hashed[:])
}
func Md5Stream(d []byte) string {
hash := md5.New()
hash.Write(d)
hashed := hash.Sum(nil)
return hex.EncodeToString(hashed)
}
utils/setutils/xset.go deleted 100644 → 0
浏览文件 @ 6f55882a
package setutils
import (
"gitlab.wanzhuangkj.com/tush/xpkg/pkg/xtype"
)
type Set[T xtype.Key] struct {
m map[any]bool
}
func NewSet[T xtype.Key]() *Set[T] {
return &Set[T]{m: make(map[any]bool)}
}
func (s *Set[T]) Add(item any) {
if !s.Exists(item) {
s.m[item] = true
}
}
func (s *Set[T]) Remove(item any) {
delete(s.m, item)
}
func (s *Set[T]) Clear() {
s.m = make(map[any]bool)
}
func (s *Set[T]) Exists(item any) bool {
_, exists := s.m[item]
return exists
}
func (s *Set[T]) Size() int {
return len(s.m)
}
func (s *Set[T]) Slice() []T {
var list []T
for k := range s.m {
if v, ok := k.(T); ok {
list = append(list, v)
}
}
return list
}
utils/sliceutils/xslice.go deleted 100644 → 0
浏览文件 @ 6f55882a
package sliceutils
import (
"fmt"
"strings"
"gitlab.wanzhuangkj.com/tush/xpkg/pkg/errcode"
"gitlab.wanzhuangkj.com/tush/xpkg/pkg/xtype"
"gitlab.wanzhuangkj.com/tush/xpkg/utils/setutils"
"github.com/jinzhu/copier"
"github.com/spf13/cast"
"gitlab.wanzhuangkj.com/tush/xpkg/pkg/xerrors/xerror"
)
// connector 连接符
func Join[T any](rs []T, connector string) string {
if len(rs) == 0 {
return ""
}
strs := make([]string, 0, len(rs))
for _, r := range rs {
v := cast.ToString(r)
strs = append(strs, v)
}
return strings.Join(strs, connector)
}
func Contains[T comparable](rs []T, r T) bool {
for _, v := range rs {
if v == r {
return true
}
}
return false
}
func NotContains[T comparable](rs []T, r T) bool {
return !Contains(rs, r)
}
func StrToUint64(rs []string) []uint64 {
if len(rs) == 0 {
return nil
}
var IDs []uint64
for _, r := range rs {
IDs = append(IDs, cast.ToUint64(r))
}
return IDs
}
type IID[ID xtype.Key] interface {
GetID() ID
}
type IIDTable[ID xtype.Key] interface {
GetID() ID
TableName() string
}
type ITable interface {
TableName() string
}
func GetIDs[ID xtype.Key, T IID[ID]](rs []*T) []ID {
if len(rs) == 0 {
return nil
}
set := setutils.NewSet[ID]()
for _, r := range rs {
set.Add((*r).GetID())
}
return set.Slice()
}
func SliceToIDMap[ID xtype.Key, T IID[ID]](rs []*T) map[ID]*T {
m := make(map[ID]*T)
for _, r := range rs {
m[(*r).GetID()] = r
}
return m
}
func MapToSlice[T comparable, FromT, ToT any](m map[T]*FromT) []*ToT {
if len(m) == 0 {
return nil
}
ret := make([]*ToT, len(m))
for _, r := range m {
ret = append(ret, TypeToType[FromT, ToT](r))
}
return ret
}
func MapToMap[T comparable, FromT, ToT any](m map[T]*FromT) map[T]*ToT {
if len(m) == 0 {
return nil
}
ret := make(map[T]*ToT, len(m))
for k, r := range m {
ret[k] = TypeToType[FromT, ToT](r)
}
return ret
}
func TypeToType[FromT, ToT any](r *FromT) *ToT {
if r == nil {
return nil
}
var ret ToT
copier.Copy(&ret, r)
return &ret
}
func SliceToSlice[FromT, ToT any](rs []*FromT) []*ToT {
if len(rs) == 0 {
return nil
}
ret := make([]*ToT, 0, len(rs))
for _, r := range rs {
if r == nil {
continue
}
ret = append(ret, TypeToType[FromT, ToT](r))
}
return ret
}
func MapSliceToMapSlice[T comparable, FromT, ToT any](m map[T][]*FromT) map[T][]*ToT {
if len(m) == 0 {
return nil
}
ret := make(map[T][]*ToT, len(m))
for k, rs := range m {
ret[k] = SliceToSlice[FromT, ToT](rs)
}
return ret
}
func MapSliceToSlice[T comparable, FromT, ToT any](m map[T][]*FromT) []*ToT {
if len(m) == 0 {
return nil
}
ret := make([]*ToT, 0)
for _, rs := range m {
ret = append(ret, SliceToSlice[FromT, ToT](rs)...)
}
return ret
}
// SetDifference 计算两个切片的差集 A - B
func SetDifference[T comparable](A, B []T) []T {
result := []T{}
setB := make(map[T]struct{})
for _, elem := range B {
setB[elem] = struct{}{}
}
for _, elem := range A {
if _, found := setB[elem]; !found {
result = append(result, elem)
}
}
return result
}
func StrMapToSlice[T any](m map[string]*T) []*T {
var rs []*T
for _, r := range m {
rs = append(rs, r)
}
return rs
}
func GetStrMapIDs[ID xtype.Key, T IID[ID]](m map[string]*T) []ID {
st := setutils.NewSet[ID]()
for _, v := range m {
st.Add((*v).GetID())
}
return st.Slice()
}
func IDMapToSlice[ID xtype.Key, T any](m map[ID]*T) []*T {
var rs []*T
for _, r := range m {
rs = append(rs, r)
}
return rs
}
func GetIDMapIDs[ID xtype.Key, T IID[ID]](m map[ID]*T) []ID {
st := setutils.NewSet[ID]()
for _, v := range m {
st.Add((*v).GetID())
}
return st.Slice()
}
func StrMapSliceToSlice[T any](m map[string][]*T) []*T {
var rs []*T
for _, r := range m {
rs = append(rs, r...)
}
return rs
}
func GetStrMapSliceIDs[ID xtype.Key, T IID[ID]](m map[string][]*T) []ID {
st := setutils.NewSet[ID]()
for _, rs := range m {
for _, r := range rs {
st.Add((*r).GetID())
}
}
return st.Slice()
}
func IDMapSliceToSlice[ID xtype.Key, T any](m map[ID][]*T) []*T {
var rs []*T
for _, r := range m {
rs = append(rs, r...)
}
return rs
}
func GetIDMapSliceIDs[ID xtype.Key, T IID[ID]](m map[ID][]*T) []ID {
st := setutils.NewSet[ID]()
for _, rs := range m {
for _, r := range rs {
st.Add((*r).GetID())
}
}
return st.Slice()
}
type ICode interface {
Code() int
}
func CompareSlice[ID xtype.Key, T IIDTable[ID], BizSlice ~[]*T](ids []ID, bizSlice BizSlice, errCode error) error {
if len(ids) == 0 {
return nil
}
if len(bizSlice) < len(ids) {
if len(bizSlice) == 0 {
if v, ok := errCode.(*errcode.Error); ok {
return xerror.NewC(v.Code(), v.Msg())
}
return xerror.New(errCode.Error())
}
deltaIDs := SetDifference(ids, GetIDs[ID](bizSlice))
if len(deltaIDs) > 0 {
if v, ok := errCode.(*errcode.Error); ok {
return xerror.NewC(v.Code(), fmt.Sprintf("%s[ids:%s]", v.Msg(), Join(deltaIDs, ",")))
}
return xerror.New(errCode.Error())
}
}
return nil
}
func ArrayUnique[T comparable](array []T) []T {
mp := make(map[T]struct{})
idx := 0
for _, value := range array {
if _, ok := mp[value]; ok {
continue
}
array[idx] = value
idx = idx + 1
mp[value] = struct{}{}
}
return array[:idx]
}
//
//type IId[id xtype.Key] interface {
// GetID() id
//}
//
//func GetIds[id xtype.Key, T IId[id]](rs []*T) []id {
// if len(rs) == 0 {
// return nil
// }
// set := setutils.NewSet[id]()
// for _, r := range rs {
// set.Add((*r).GetID())
// }
// return set.Slice()
//}
//
//func SliceToIdMap[id xtype.Key, T IId[id]](rs []*T) map[id]*T {
// m := make(map[id]*T)
// for _, r := range rs {
// m[(*r).GetID()] = r
// }
// return m
//}
func ChunkSlice[T any](slice []T, chunkSize int) [][]T {
var chunks [][]T
for i := 0; i < len(slice); i += chunkSize {
end := i + chunkSize
if end > len(slice) {
end = len(slice)
}
chunks = append(chunks, slice[i:end])
}
return chunks
}
type IJoin interface {
String() string
}
func JoinStrs[T IJoin](rs []T, connector string) string {
if len(rs) == 0 {
return ""
}
strs := make([]string, 0, len(rs))
for _, r := range rs {
strs = append(strs, r.String())
}
return strings.Join(strs, connector)
}
func AllNotEmpty(rs ...string) bool {
if len(rs) == 0 {
return false
}
ret := true
for _, r := range rs {
if strings.TrimSpace(r) == "" {
ret = false
}
}
return ret
}
func AllEmpty(rs ...string) bool {
if len(rs) == 0 {
return false
}
ret := true
for _, r := range rs {
if strings.TrimSpace(r) != "" {
ret = false
}
}
return ret
}
func AnyEmpty(rs ...string) bool {
if len(rs) == 0 {
return false
}
for _, r := range rs {
if strings.TrimSpace(r) == "" {
return true
}
}
return false
}
utils/stackutils/stack.go deleted 100644 → 0
浏览文件 @ 6f55882a
package stackutils
import "runtime"
func Get() string {
buf := make([]byte, 1024)
for {
n := runtime.Stack(buf, false)
if n < len(buf) {
buf = buf[:n]
break
}
buf = make([]byte, 2*len(buf))
}
return string(buf)
}
utils/stringutils/xstring.go deleted 100644 → 0
浏览文件 @ 6f55882a
package stringutils
import (
"strings"
)
func CamelToUnderscore(s string) string {
var builder strings.Builder
for i, c := range s {
if i > 0 && c >= 'A' && c <= 'Z' {
builder.WriteByte('_')
}
builder.WriteRune(c)
}
return strings.ToLower(builder.String())
}
func UnderscoreToCamel(s string, isUpperFirst bool) string {
var builder strings.Builder
words := strings.Split(s, "_")
for i, word := range words {
if word == "" {
continue
}
// 处理首单词首字母
if i == 0 && !isUpperFirst {
builder.WriteString(strings.ToLower(word[:1]))
} else {
builder.WriteString(strings.ToUpper(word[:1]))
}
// 剩余字符处理
if len(word) > 1 {
builder.WriteString(strings.ToLower(word[1:]))
}
}
return builder.String()
}
func AllNotEmpty(rs ...string) bool {
if len(rs) == 0 {
return false
}
ret := true
for _, r := range rs {
if strings.TrimSpace(r) == "" {
ret = false
}
}
return ret
}
utils/weblogutils/web_log.go deleted 100644 → 0
浏览文件 @ 6f55882a
package weblogutils
import (
"container/list"
"context"
"strings"
"time"
"gitlab.wanzhuangkj.com/tush/xpkg/config"
"gitlab.wanzhuangkj.com/tush/xpkg/database"
"gitlab.wanzhuangkj.com/tush/xpkg/xcommon/api"
"github.com/gin-gonic/gin"
"github.com/jinzhu/copier"
"gitlab.wanzhuangkj.com/tush/xpkg/pkg/httpcli/entity"
"gitlab.wanzhuangkj.com/tush/xpkg/pkg/logger"
merge "gitlab.wanzhuangkj.com/tush/xpkg/pkg/merger"
"gitlab.wanzhuangkj.com/tush/xpkg/utils/ctxutils"
"gitlab.wanzhuangkj.com/tush/xpkg/utils/jsonutils"
"gitlab.wanzhuangkj.com/tush/xpkg/utils/retryutils"
"gitlab.wanzhuangkj.com/tush/xpkg/utils/xsf"
"gitlab.wanzhuangkj.com/tush/xpkg/utils/xtime"
"gorm.io/gorm"
glogger "gorm.io/gorm/logger"
)
var (
schema string
enable bool = false
debug bool = false
ignore = make(map[string]struct{})
)
func SetSchema(v string) {
schema = v
}
func WebLogInterceptor() func(c *gin.Context) {
return func(c *gin.Context) {
if !enable {
c.Next()
return
}
URL := strings.ToLower(c.Request.Method) + "#" + c.Request.URL.Path
if _, ok := ignore[URL]; ok {
c.Next()
return
}
var (
req *entity.CopyHttpReq
rsp *entity.CopyHttpRsp
// respCode int
)
ctx := ctxutils.WrapCtx(c)
xApi, ok := api.XApi.M[URL]
if ok {
uid := ctxutils.GetGinUID(c)
req = entity.GetCopyReq(c)
if uid > 0 {
defer func() {
AddAsync(ctx, xApi.Summary, req, rsp)
}()
} else {
defer func() {
AddAsync(ctx, xApi.Summary, req, rsp)
}()
}
}
c.Next()
if ok {
rsp = entity.GetCopyRsp(c)
}
}
}
var merger *merge.FanIn
func Init() {
var webLogCfg config.WebLog
config.IsProd()
config.Read(func(c *config.Config) {
copier.Copy(&webLogCfg, c.App.WebLog)
})
enable = webLogCfg.Enable
debug = webLogCfg.Debug
if !webLogCfg.Enable {
return
}
schema = webLogCfg.Schema
isProd := config.IsProd()
for _, p := range webLogCfg.IgnorePaths {
ignore[strings.TrimSpace(p)] = struct{}{}
}
duration := 10 * time.Minute
count := 256
if !isProd {
duration = 1 * time.Minute
count = 10
}
merger = merge.NewFanIn(duration, count, func(l *list.List) error {
if l.Len() == 0 {
return nil
}
ctx := context.WithValue(context.Background(), ctxutils.ContextTraceIDKey, xsf.GenerateID().String())
ors := make([]*WebLog, 0, l.Len())
for e := l.Front(); e != nil; e = e.Next() {
le, ok := e.Value.(*WebLog)
if ok {
ors = append(ors, le)
}
}
if len(ors) == 0 {
return nil
}
insertFn := func() error {
if debug {
if err := webLogDaoInstance.CreateSlice(ctx, ors); err != nil {
logger.Error("[webLog] insert fail detail", logger.Err(err), ctxutils.CtxTraceIDField(ctx))
return err
}
} else {
if err := webLogDaoInstance.CreateSliceSilent(ctx, ors); err != nil {
logger.Error("[webLog] insert fail detail", logger.Err(err), ctxutils.CtxTraceIDField(ctx))
return err
}
}
return nil
}
go func() {
if debug {
logger.Info("[webLog]batch insert start")
}
if err := retryutils.Retry(insertFn, retryutils.RetryTimes(10), retryutils.RetryWithLinearBackoff(32*time.Minute)); err != nil {
logger.Error("[webLog] insert fail", logger.String("err", err.Error()), ctxutils.CtxTraceIDField(ctx))
for _, or := range ors {
logger.Error("[webLog] insert fail detail", logger.String("user_id", or.UID.String()), logger.String("operate", or.Operate), ctxutils.CtxTraceIDField(ctx))
}
}
if debug {
logger.Info("[webLog]batch insert success")
}
}()
return nil
})
logger.Info("[webLog]init success")
merger.Start()
webLogDaoInstance.db = database.DB(schema)
}
func AddAsync(ctx context.Context, operate string, req *entity.CopyHttpReq, rsp *entity.CopyHttpRsp) {
wl := &WebLog{}
wl.ID = xsf.GenerateID()
wl.CompanyName = ctxutils.GetCtxCompanyName(ctx)
wl.CompanyID = ctxutils.GetCtxCompanyID(ctx)
wl.UID = ctxutils.GetCtxUID(ctx)
wl.UName = ctxutils.GetCtxUName(ctx)
wl.TraceID = ctxutils.GetCtxTid(ctx)
wl.Operate = operate
wl.CreatedAt = xtime.Now()
if req != nil {
wl.Req = jsonutils.ToJSONStringPtr(req)
}
if rsp != nil {
wl.Rsp = jsonutils.ToJSONStringPtr(rsp)
}
go func() {
merger.Add(wl)
}()
}
type webLogDao struct {
db *gorm.DB
}
var webLogDaoInstance = &webLogDao{}
func (x *webLogDao) CreateSliceSilent(ctx context.Context, ors []*WebLog) error {
return x.db.WithContext(ctx).Session(&gorm.Session{Logger: glogger.Discard}).Create(ors).Error
}
func (x *webLogDao) CreateSlice(ctx context.Context, ors []*WebLog) error {
return x.db.WithContext(ctx).Create(ors).Error
}
type WebLog struct {
ID xsf.ID `json:"id" form:"id" swaggertype:"string" gorm:"column:id;type:bigint(20) unsigned;primaryKey;autoIncrement;comment:主键" example:"1000"` //主键
CompanyID xsf.ID `json:"companyID" form:"companyID" swaggertype:"string" gorm:"column:company_id;type:bigint(20) unsigned;comment:公司ID" example:"1008"` //公司ID
UID xsf.ID `json:"uID" form:"uID" swaggertype:"string" gorm:"column:u_id;type:bigint(20) unsigned;comment:用户ID" example:"2008"` //用户ID
CompanyName string `json:"companyName" form:"companyName" gorm:"column:company_name;type:varchar(128);comment:公司名称" example:"tomcat"` //公司名称
UName string `json:"uName" form:"uName" gorm:"column:u_name;type:varchar(128);comment:名称" example:"tomcat"` //名称
Operate string `json:"operate" form:"operate" gorm:"column:operate;type:varchar(128);comment:操作" example:"用户登出"` //操作
TraceID string `json:"traceID" form:"traceID" gorm:"column:trace_id;type:varchar(64);comment:溯源id" example:"TRACE_ID12345"` //溯源id
Req *string `json:"req" form:"req" gorm:"column:req;type:json;comment:请求" example:""` //请求
Rsp *string `json:"rsp" form:"rsp" gorm:"column:rsp;type:json;comment:响应" example:""` //响应
CreatedAt xtime.DateTime `json:"createdAt" form:"createdAt" gorm:"column:created_at;type:datetime;comment:创建时间" example:"2025-01-07 12:20:43"` //创建时间
}
func (x WebLog) GetID() xsf.ID {
return x.ID
}
const TBWebLog = "web_log"
func (WebLog) TableName() string {
return TBWebLog
}
func (x *WebLog) BeforeCreate(tx *gorm.DB) (err error) {
if x.CreatedAt.IsZero() {
x.CreatedAt = xtime.Now()
}
return nil
}
func (x *WebLog) BeforeUpdate(tx *gorm.DB) (err error) {
return nil
}
utils/weblogutils/weblog.sql deleted 100644 → 0
浏览文件 @ 6f55882a
DROP TABLE if EXISTS `web_log`;
CREATE TABLE `web_log` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID eg[10001]',
`company_id` bigint(20) DEFAULT NULL COMMENT '公司ID eg[20001]',
`u_id` bigint(20) DEFAULT NULL COMMENT '用户ID eg[30001]',
`company_name` varchar(32) COLLATE utf8mb4_unicode_ci DEFAULT NULL DEFAULT '' COMMENT '公司名称 eg[万桩]',
`u_name` varchar(32) COLLATE utf8mb4_unicode_ci DEFAULT NULL DEFAULT '' COMMENT '名称 eg[tomcat]',
`operate` varchar(32) COLLATE utf8mb4_unicode_ci DEFAULT NULL DEFAULT '' COMMENT '操作 eg[用户登出]',
`trace_id` varchar(64) COLLATE utf8mb4_unicode_ci DEFAULT NULL DEFAULT '' COMMENT '溯源id eg[TRACE_ID12345]',
`req` json DEFAULT NULL COMMENT '请求 eg[]',
`rsp` json DEFAULT NULL COMMENT '响应 eg[]',
`created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间 eg[2025-01-07 12:20:43]',
PRIMARY KEY (`id`),
KEY `idx_uid_ct` (`u_id`, `created_at`)
) ENGINE = InnoDB AUTO_INCREMENT = 1 DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci ROW_FORMAT = DYNAMIC COMMENT = '操作记录';
alter table web_log add column `company_name` varchar(32) COLLATE utf8mb4_unicode_ci DEFAULT NULL DEFAULT '' COMMENT '公司名称 eg[万桩]';
alter table web_log add column `u_name` varchar(32) COLLATE utf8mb4_unicode_ci DEFAULT NULL DEFAULT '' COMMENT '名称 eg[tomcat]';
\ No newline at end of file
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论