提交 105f3c9a authored 作者: mooncake9527's avatar mooncake9527

update

上级 bbc4005a
// Package consulcli is connecting to the consul service client.
package consulcli
import (
"fmt"
"gitlab.wanzhuangkj.com/tush/xpkg/xerrors/xerror"
"github.com/hashicorp/consul/api"
)
// Init connecting to the consul service
// Note: If the WithConfig(*api.Config) parameter is set, the addr parameter is ignored!
func Init(addr string, opts ...Option) (*api.Client, error) {
o := defaultOptions()
o.apply(opts...)
if o.config != nil {
return api.NewClient(o.config)
}
if addr == "" {
return nil, fmt.Errorf("consul address cannot be empty")
}
c, err := api.NewClient(&api.Config{
Address: addr,
Scheme: o.scheme,
WaitTime: o.waitTime,
Datacenter: o.datacenter,
})
if err != nil {
return nil, xerror.New(err.Error())
}
return c, nil
}
package consulcli
import (
"testing"
"time"
"github.com/hashicorp/consul/api"
)
func TestInit(t *testing.T) {
addr := "192.168.3.37:8500"
cli, err := Init(addr,
WithScheme("http"),
WithWaitTime(time.Second*2),
WithDatacenter(""),
WithToken("your-token"),
)
t.Log(err, cli)
cli, err = Init("", WithConfig(&api.Config{
Address: addr,
Scheme: "http",
WaitTime: time.Second * 2,
Datacenter: "",
}))
t.Log(err, cli)
}
package consulcli
import (
"time"
"github.com/hashicorp/consul/api"
)
// Option set the consul client options.
type Option func(*options)
type options struct {
scheme string
waitTime time.Duration
datacenter string
token string
// if you set this parameter, all fields above are invalid
config *api.Config
}
func defaultOptions() *options {
return &options{
scheme: "http",
waitTime: time.Second * 5,
}
}
func (o *options) apply(opts ...Option) {
for _, opt := range opts {
opt(o)
}
}
// WithWaitTime set wait time
func WithWaitTime(waitTime time.Duration) Option {
return func(o *options) {
o.waitTime = waitTime
}
}
// WithScheme set scheme
func WithScheme(scheme string) Option {
return func(o *options) {
o.scheme = scheme
}
}
// WithDatacenter set datacenter
func WithDatacenter(datacenter string) Option {
return func(o *options) {
o.datacenter = datacenter
}
}
// WithToken set token
func WithToken(token string) Option {
return func(o *options) {
o.token = token
}
}
// WithConfig set consul config
func WithConfig(c *api.Config) Option {
return func(o *options) {
o.config = c
}
}
// Package dlock provides distributed locking primitives, supports redis and etcd.
package dlock
import "context"
// Locker is the interface that wraps the basic locking operations.
type Locker interface {
Lock(ctx context.Context) error
Unlock(ctx context.Context) error
TryLock(ctx context.Context) (bool, error)
Close() error
}
package dlock
import (
"context"
"errors"
"time"
clientv3 "go.etcd.io/etcd/client/v3"
"go.etcd.io/etcd/client/v3/concurrency"
)
var defaultTTL = 15 // seconds
type EtcdLock struct {
session *concurrency.Session
mutex *concurrency.Mutex
}
// NewEtcd creates a new etcd locker with the given key and ttl.
func NewEtcd(client *clientv3.Client, key string, ttl int) (Locker, error) {
if client == nil {
return nil, errors.New("etcd client is nil")
}
if key == "" {
return nil, errors.New("key is empty")
}
if ttl <= 0 {
ttl = defaultTTL
}
expiration := time.Duration(ttl) * time.Second
ctx, _ := context.WithTimeout(context.Background(), expiration) //nolint
session, err := concurrency.NewSession(
client,
concurrency.WithTTL(ttl),
concurrency.WithContext(ctx),
)
if err != nil {
return nil, err
}
mutex := concurrency.NewMutex(session, key)
locker := &EtcdLock{
session: session,
mutex: mutex,
}
return locker, nil
}
// Lock blocks until the lock is acquired or the context is canceled.
func (l *EtcdLock) Lock(ctx context.Context) error {
return l.mutex.Lock(ctx)
}
// Unlock releases the lock.
func (l *EtcdLock) Unlock(ctx context.Context) error {
return l.mutex.Unlock(ctx)
}
// TryLock tries to acquire the lock without blocking.
func (l *EtcdLock) TryLock(ctx context.Context) (bool, error) {
err := l.mutex.TryLock(ctx)
if err == nil {
return true, nil
}
if err == concurrency.ErrLocked {
return false, nil
}
return false, err
}
// Close releases the lock and the etcd session.
func (l *EtcdLock) Close() error {
if l.session != nil {
return l.session.Close()
}
return nil
}
package dlock
import (
"fmt"
"testing"
"time"
"gitlab.wanzhuangkj.com/tush/xpkg/etcdcli"
"go.uber.org/zap"
)
func TestEtcdLock_TryLock(t *testing.T) {
initLocker := func() Locker {
return getEtcdLock()
}
testLockAndUnlock(initLocker, false, t)
}
func TestEtcdLock_Lock(t *testing.T) {
initLocker := func() Locker {
return getEtcdLock()
}
testLockAndUnlock(initLocker, true, t)
}
func getEtcdLock() Locker {
endpoints := []string{"127.0.0.1:2379"}
cli, err := etcdcli.Init(endpoints,
etcdcli.WithDialTimeout(time.Second*2),
etcdcli.WithAuth("", ""),
etcdcli.WithAutoSyncInterval(0),
etcdcli.WithLog(zap.NewNop()),
)
if err != nil {
fmt.Println(err)
return nil
}
locker, err := NewEtcd(cli, "xmall/dlock", 10)
if err != nil {
fmt.Println(err)
return nil
}
return locker
}
package dlock
import (
"context"
"errors"
"github.com/go-redsync/redsync/v4"
"github.com/go-redsync/redsync/v4/redis/goredis/v9"
"github.com/redis/go-redis/v9"
)
// RedisLock implements Locker using Redis.
type RedisLock struct {
mutex *redsync.Mutex
}
// NewRedisLock creates a new RedisLock.
func NewRedisLock(client *redis.Client, key string, options ...redsync.Option) (Locker, error) {
if client == nil {
return nil, errors.New("redis client is nil")
}
if key == "" {
return nil, errors.New("key is empty")
}
return newLocker(client, key, options...), nil
}
// NewRedisClusterLock creates a new RedisClusterLock.
func NewRedisClusterLock(clusterClient *redis.ClusterClient, key string, options ...redsync.Option) (Locker, error) {
if clusterClient == nil {
return nil, errors.New("cluster redis client is nil")
}
if key == "" {
return nil, errors.New("key is empty")
}
return newLocker(clusterClient, key, options...), nil
}
func newLocker(delegate redis.UniversalClient, key string, options ...redsync.Option) Locker {
pool := goredis.NewPool(delegate)
rs := redsync.New(pool)
mutex := rs.NewMutex(key, options...)
return &RedisLock{
mutex: mutex,
}
}
// TryLock tries to acquire the lock without blocking.
func (l *RedisLock) TryLock(ctx context.Context) (bool, error) {
err := l.mutex.TryLockContext(ctx)
if err == nil {
return true, nil
}
return false, err
}
// Lock blocks until the lock is acquired or the context is canceled.
func (l *RedisLock) Lock(ctx context.Context) error {
return l.mutex.LockContext(ctx)
}
// Unlock releases the lock, if unlocking the key is successful, the key will be automatically deleted
func (l *RedisLock) Unlock(ctx context.Context) error {
_, err := l.mutex.UnlockContext(ctx)
return err
}
// Close no-op for RedisLock.
func (l *RedisLock) Close() error {
return nil
}
package dlock
import (
"context"
"fmt"
"sync"
"testing"
"time"
"gitlab.wanzhuangkj.com/tush/xpkg/goredis"
)
func TestRedisLock_TryLock(t *testing.T) {
initLocker := func() Locker {
return getRedisLock()
}
testLockAndUnlock(initLocker, false, t)
}
func TestRedisLock_Lock(t *testing.T) {
initLocker := func() Locker {
return getRedisLock()
}
testLockAndUnlock(initLocker, true, t)
}
func TestClusterRedis_TryLock(t *testing.T) {
initLocker := func() Locker {
return getClusterRedisLock()
}
testLockAndUnlock(initLocker, false, t)
}
func TestClusterRedis_Lock(t *testing.T) {
initLocker := func() Locker {
return getClusterRedisLock()
}
testLockAndUnlock(initLocker, true, t)
}
func getRedisLock() Locker {
redisCli, err := goredis.Init("default:123456@127.0.0.1:6379")
if err != nil {
fmt.Println(err)
return nil
}
locker, err := NewRedisLock(redisCli, "test_lock")
if err != nil {
return nil
}
return locker
}
func getClusterRedisLock() Locker {
addrs := []string{"127.0.0.1:6380", "127.0.0.1:6381", "127.0.0.1:6382"}
clusterClient, err := goredis.InitCluster(addrs, "", "123456")
if err != nil {
fmt.Println(err)
return nil
}
locker, err := NewRedisClusterLock(clusterClient, "test_cluster_lock")
if err != nil {
return nil
}
return locker
}
func testLockAndUnlock(initLocker func() Locker, isBlock bool, t *testing.T) {
waitGroup := &sync.WaitGroup{}
for i := 1; i <= 10; i++ {
waitGroup.Add(1)
go func(i int) {
defer waitGroup.Done()
ctx, _ := context.WithTimeout(context.Background(), time.Second*10)
NO := fmt.Sprintf("[NO-%d] ", i)
locker := initLocker()
if locker == nil {
t.Log("logger init failed")
return
}
var err error
var ok bool
for {
select {
case <-ctx.Done():
return
default:
}
time.Sleep(time.Millisecond * 50)
if isBlock {
err = locker.Lock(ctx)
if err == nil {
ok = true
}
} else {
ok, err = locker.TryLock(ctx)
}
if err != nil {
//t.Log(NO+"try lock error:", err)
continue
}
if ok {
t.Log(NO + "acquire lock success, and do something")
time.Sleep(time.Millisecond * 200)
err = locker.Unlock(ctx)
if err != nil {
return
}
t.Log(NO + "unlock done")
return
}
}
}(i)
}
waitGroup.Wait()
}
// Package etcdcli is use for connecting to the etcd service
package etcdcli
import (
"fmt"
"gitlab.wanzhuangkj.com/tush/xpkg/xerrors/xerror"
"time"
clientv3 "go.etcd.io/etcd/client/v3"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/credentials/insecure"
)
// Init connecting to the etcd service
// Note: If the WithConfig(*clientv3.Config) parameter is set, the endpoints parameter is ignored!
func Init(endpoints []string, opts ...Option) (*clientv3.Client, error) {
o := defaultOptions()
o.apply(opts...)
if o.config != nil {
return clientv3.New(*o.config)
}
if len(endpoints) == 0 {
return nil, fmt.Errorf("etcd endpoints cannot be empty")
}
conf := clientv3.Config{
Endpoints: endpoints,
DialTimeout: o.dialTimeout,
DialKeepAliveTime: 20 * time.Second,
DialKeepAliveTimeout: 10 * time.Second,
AutoSyncInterval: o.autoSyncInterval,
Logger: o.logger,
Username: o.username,
Password: o.password,
}
if !o.isSecure {
conf.DialOptions = append(conf.DialOptions, grpc.WithTransportCredentials(insecure.NewCredentials()))
} else {
cred, err := credentials.NewClientTLSFromFile(o.certFile, o.serverNameOverride)
if err != nil {
return nil, fmt.Errorf("NewClientTLSFromFile error: %v", err)
}
conf.DialOptions = append(conf.DialOptions, grpc.WithTransportCredentials(cred))
}
cli, err := clientv3.New(conf)
if err != nil {
return nil, xerror.Errorf("connecting to the etcd service error: %v", err)
}
return cli, nil
}
package etcdcli
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/zap"
)
func TestInit(t *testing.T) {
endpoints := []string{"192.168.3.37:2379"}
cli, err := Init(endpoints,
WithDialTimeout(time.Second*2),
WithAuth("", ""),
WithAutoSyncInterval(0),
WithLog(zap.NewNop()),
)
t.Log(err, cli)
cli, err = Init(nil, WithConfig(&clientv3.Config{
Endpoints: endpoints,
DialTimeout: time.Second * 2,
Username: "",
Password: "",
}))
t.Log(err, cli)
// test error
_, err = Init(endpoints,
WithDialTimeout(time.Second),
WithSecure("foo", "notfound.crt"))
assert.Error(t, err)
endpoints = nil
_, err = Init(endpoints)
assert.Error(t, err)
}
package etcdcli
import (
"time"
clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/zap"
)
// Option set the etcd client options.
type Option func(*options)
type options struct {
dialTimeout time.Duration // connection timeout, unit(second)
username string
password string
isSecure bool
serverNameOverride string // etcd domain
certFile string // path to certificate file
autoSyncInterval time.Duration // automatic synchronization of member list intervals
logger *zap.Logger
// if you set this parameter, all fields above are invalid
config *clientv3.Config
}
func defaultOptions() *options {
return &options{
dialTimeout: time.Second * 5,
}
}
func (o *options) apply(opts ...Option) {
for _, opt := range opts {
opt(o)
}
}
// WithDialTimeout set dial timeout
func WithDialTimeout(duration time.Duration) Option {
return func(o *options) {
o.dialTimeout = duration
}
}
// WithAuth set authentication
func WithAuth(username string, password string) Option {
return func(o *options) {
o.username = username
o.password = password
}
}
// WithSecure set tls
func WithSecure(serverNameOverride string, certFile string) Option {
return func(o *options) {
o.isSecure = true
o.serverNameOverride = serverNameOverride
o.certFile = certFile
}
}
// WithAutoSyncInterval set auto sync interval value
func WithAutoSyncInterval(duration time.Duration) Option {
return func(o *options) {
o.autoSyncInterval = duration
}
}
// WithLog set logger
func WithLog(l *zap.Logger) Option {
return func(o *options) {
o.logger = l
}
}
// WithConfig set etcd client config
func WithConfig(c *clientv3.Config) Option {
return func(o *options) {
o.config = c
}
}
差异被折叠。
差异被折叠。
// Package gobash provides the ability to execute commands, scripts, executables in the go environment with live log output.
package gobash
import (
"bufio"
"context"
"errors"
"fmt"
"io"
"os/exec"
"strings"
)
// Exec suitable for executing a single non-blocking command, outputting standard and error logs,
// but the log output is not real time, no execution, command name must be in system path,
// Note: If the execution of a command blocks permanently, it can cause a concurrent leak.
func Exec(name string, args ...string) ([]byte, error) {
cmdName, err := exec.LookPath(name) // cmdName is absolute path
if err != nil {
return nil, err
}
cmd := exec.Command(cmdName, args...)
return getResult(cmd)
}
// Result of the execution of the command
type Result struct {
StdOut chan string
Err error // If nil after the command is executed, the command is executed successfully
}
// Run execute the command, no execution, command name must be in system path,
// you can actively end the command, the execution results are returned in real time in Result.StdOut
func Run(ctx context.Context, name string, args ...string) *Result {
result := &Result{StdOut: make(chan string), Err: error(nil)}
go func() {
defer func() { close(result.StdOut) }() // execution complete, channel closed
cmdName, err := exec.LookPath(name) // cmdName is absolute path
if err != nil {
result.Err = err
return
}
cmd := exec.CommandContext(ctx, cmdName, args...)
handleExec(ctx, cmd, result)
}()
return result
}
func handleExec(ctx context.Context, cmd *exec.Cmd, result *Result) {
result.StdOut <- strings.Join(cmd.Args, " ") + "\n"
stdout, stderr, err := getCmdReader(cmd)
if err != nil {
result.Err = err
return
}
reader := bufio.NewReader(stdout)
// reads each line in real time
line := ""
for {
line, err = reader.ReadString('\n')
if err != nil {
if errors.Is(err, io.EOF) { // determine if it has been read
break
}
result.Err = err
break
}
select {
case result.StdOut <- line:
case <-ctx.Done():
result.Err = fmt.Errorf("%v", ctx.Err())
return
}
}
// capture error logs
bytesErr, err := io.ReadAll(stderr)
if err != nil {
result.Err = err
return
}
err = cmd.Wait()
if err != nil {
if len(bytesErr) != 0 {
result.Err = errors.New(string(bytesErr))
return
}
result.Err = err
}
}
func getCmdReader(cmd *exec.Cmd) (io.ReadCloser, io.ReadCloser, error) {
stdout, err := cmd.StdoutPipe()
if err != nil {
return nil, nil, err
}
stderr, err := cmd.StderrPipe()
if err != nil {
return nil, nil, err
}
err = cmd.Start()
if err != nil {
return nil, nil, err
}
return stdout, stderr, nil
}
func getResult(cmd *exec.Cmd) ([]byte, error) {
stdout, stderr, err := getCmdReader(cmd)
if err != nil {
return nil, err
}
bytes, err := io.ReadAll(stdout)
if err != nil {
return nil, err
}
bytesErr, err := io.ReadAll(stderr)
if err != nil {
return nil, err
}
err = cmd.Wait()
if err != nil {
if len(bytesErr) != 0 {
return nil, errors.New(string(bytesErr))
}
return nil, err
}
return bytes, nil
}
package gobash
import (
"context"
"fmt"
"testing"
"time"
)
func TestRun(t *testing.T) {
cmds := map[string][]string{
"pwd": {},
"go": {"env", "GOPATH"},
"bash": {"-c", "for i in $(seq 1 5); do echo 'test cmd' $i;sleep 0.1; done"},
}
for cmd, args := range cmds {
ctx, _ := context.WithTimeout(context.Background(), time.Second)
result := Run(ctx, cmd, args...)
for v := range result.StdOut { // Real-time output of logs and error messages
t.Logf(v)
}
if result.Err != nil {
t.Logf("execute command failed, %v", result.Err)
}
fmt.Println()
}
}
func TestExec(t *testing.T) {
cmds := map[string][]string{
"pwd": {},
"go": {"env", "GOROOT"},
"bash": {"-c", "for i in $(seq 1 5); do echo 'test cmd' $i;sleep 0.1; done"},
}
for cmd, args := range cmds {
out, err := Exec(cmd, args...)
if err != nil {
t.Logf("execute command[%s] failed, %v\n", cmd, err)
continue
}
t.Logf("%s\n", out)
}
}
// Package benchmark is compression testing of rpc methods and generation of reported results.
package benchmark
import (
"fmt"
"os"
"github.com/bojand/ghz/printer"
"github.com/bojand/ghz/runner"
"google.golang.org/protobuf/proto"
)
type Option = runner.Option
// Runner interface
type Runner interface {
Run() error
}
// bench pressing parameters
type bench struct {
rpcServerHost string // rpc server address
protoFile string // proto file
packageName string // proto file package name
serviceName string // proto file service name
methodName string // name of pressure test method
methodRequest proto.Message // input parameters corresponding to the method
dependentProtoFilePath []string // dependent proto file path
total uint // number of requests
options []runner.Option
}
// New create a pressure test instance
//
// invalid parameter total if the option runner.WithRunDuration is set
func New(host string, protoFile string, methodName string, req proto.Message, dependentProtoFilePath []string, total int, options ...runner.Option) (Runner, error) {
data, err := os.ReadFile(protoFile)
if err != nil {
return nil, err
}
packageName := getName(data, packagePattern)
if packageName == "" {
return nil, fmt.Errorf("not found package name in protobuf file %s", protoFile)
}
serviceName := getName(data, servicePattern)
if serviceName == "" {
return nil, fmt.Errorf("not found service name in protobuf file %s", protoFile)
}
methodNames := getMethodNames(data, methodPattern)
mName := matchName(methodNames, methodName)
if mName == "" {
return nil, fmt.Errorf("not found name %s in protobuf file %s", methodName, protoFile)
}
return &bench{
protoFile: protoFile,
packageName: packageName,
serviceName: serviceName,
methodName: mName,
methodRequest: req,
rpcServerHost: host,
total: uint(total),
dependentProtoFilePath: dependentProtoFilePath,
options: options,
}, nil
}
// Run operational performance benchmarking
func (b *bench) Run() error {
callMethod := fmt.Sprintf("%s.%s/%s", b.packageName, b.serviceName, b.methodName)
data, err := proto.Marshal(b.methodRequest)
if err != nil {
return err
}
opts := []runner.Option{
runner.WithTotalRequests(b.total),
runner.WithProtoFile(b.protoFile, b.dependentProtoFilePath),
runner.WithBinaryData(data),
runner.WithInsecure(true),
// more parameter settings https://github.com/bojand/ghz/blob/master/runner/options.go#L41
// example settings: https://github.com/bojand/ghz/blob/master/runner/options_test.go#L79
}
opts = append(opts, b.options...)
report, err := runner.Run(callMethod, b.rpcServerHost, opts...)
if err != nil {
return err
}
return b.saveReport(callMethod, report)
}
func (b *bench) saveReport(callMethod string, report *runner.Report) error {
// specify the output path
outDir := os.TempDir() + string(os.PathSeparator) + "xmall_grpc_benchmark"
_ = os.MkdirAll(outDir, 0777)
outputFile := fmt.Sprintf("%sreport_%s.html", outDir+string(os.PathSeparator), b.methodName)
file, err := os.Create(outputFile)
if err != nil {
return err
}
defer func() {
_ = file.Close()
}()
rp := printer.ReportPrinter{
Out: file,
Report: report,
}
fmt.Printf("\nperformance testing of api '%s' is now complete, copy the report file path to your browser to view,\nreport file: %s\n\n", callMethod, outputFile)
return rp.Print("html")
}
package benchmark
import (
"testing"
"github.com/bojand/ghz/runner"
"github.com/stretchr/testify/assert"
"google.golang.org/protobuf/compiler/protogen"
"google.golang.org/protobuf/types/pluginpb"
)
func TestNew(t *testing.T) {
importProtoFiles := []string{}
_, err := New("localhost", "testProto/test.proto", "Create", nil, importProtoFiles, 100)
assert.NoError(t, err)
_, err = New("localhost", "testProto/test2.proto", "Create", nil, importProtoFiles, 100)
assert.Error(t, err)
_, err = New("localhost", "testProto/test3.proto", "Create", nil, importProtoFiles, 100)
assert.Error(t, err)
_, err = New("localhost", "testProto/test4.proto", "Create", nil, importProtoFiles, 100)
assert.Error(t, err)
}
func Test_params_Run(t *testing.T) {
req := &pluginpb.CodeGeneratorRequest{}
opts := protogen.Options{}
gen, err := opts.New(req)
o1 := gen.Response()
importProtoFiles := []string{}
b, err := New("localhost", "testProto/test.proto", "Create", o1, importProtoFiles, 2)
assert.NoError(t, err)
err = b.Run()
t.Log(err)
}
func Test_bench_saveReport(t *testing.T) {
bc := &bench{methodName: "foo"}
err := bc.saveReport("test", &runner.Report{Name: "foo"})
assert.NoError(t, err)
}
package benchmark
import (
"regexp"
"strings"
)
const (
packagePattern = `\npackage (.*);`
servicePattern = `\nservice (\w+)`
methodPattern = `rpc (\w+)`
)
func getName(data []byte, pattern string) string {
re := regexp.MustCompile(pattern)
matchArr := re.FindStringSubmatch(string(data))
if len(matchArr) == 2 {
return strings.ReplaceAll(matchArr[1], " ", "")
}
return ""
}
func getMethodNames(data []byte, methodPattern string) []string {
re := regexp.MustCompile(methodPattern)
matchArr := re.FindAllStringSubmatch(string(data), -1)
names := []string{}
for _, arr := range matchArr {
if len(arr) == 2 {
names = append(names, strings.ReplaceAll(arr[1], " ", ""))
}
}
return names
}
// match name, not case-sensitive
func matchName(names []string, name string) string {
out := ""
for _, s := range names {
if strings.EqualFold(s, name) {
out = s
break
}
}
return out
}
package benchmark
import (
"testing"
"github.com/stretchr/testify/assert"
)
var testData = []byte(`
syntax = "proto3";
package api.use.v1;
option go_package = "./v1;v1";
service useService {
rpc Create(CreateUseRequest) returns (CreateUseReply) {}
rpc DeleteByID(DeleteUseByIDRequest) returns (DeleteUseByIDReply) {}
}
`)
func Test_getName(t *testing.T) {
actual := getName(testData, packagePattern)
assert.Equal(t, "api.use.v1", actual)
actual = getName(testData, servicePattern)
assert.Equal(t, "useService", actual)
}
func Test_getMethodNames(t *testing.T) {
actual := getMethodNames(testData, methodPattern)
assert.EqualValues(t, []string{"Create", "DeleteByID"}, actual)
}
func Test_matchName(t *testing.T) {
methodNames := []string{"Create", "DeleteByID"}
actual := matchName(methodNames, "Create")
assert.NotEmpty(t, actual)
actual = matchName(methodNames, "a")
assert.Empty(t, actual)
}
syntax = "proto3";
package v1;
option go_package = "./v1;v1";
service userService {
rpc Create(CreateUserRequest) returns (CreateUserReply) {}
rpc DeleteByID(DeleteUserByIDRequest) returns (DeleteUserByIDReply) {}
}
message CreateUserRequest {
string name = 1;
}
message CreateUserReply {
uint64 id = 1;
}
message DeleteUserByIDRequest {
uint64 id = 1;
}
message DeleteUserByIDReply {
}
syntax = "proto3";
// no package error
option go_package = "./v1;v1";
service userService {
rpc Create(CreateUserRequest) returns (CreateUserReply) {}
rpc DeleteByID(DeleteUserByIDRequest) returns (DeleteUserByIDReply) {}
}
message CreateUserRequest {
string name = 1;
}
message CreateUserReply {
uint64 id = 1;
}
message DeleteUserByIDRequest {
uint64 id = 1;
}
message DeleteUserByIDReply {
}
syntax = "proto3";
package v1;
option go_package = "./v1;v1";
// no service error
syntax = "proto3";
package v1;
option go_package = "./v1;v1";
service userService {
// no method error
}
// Package client is generic grpc client-side.
package client
import (
"context"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/resolver"
)
// Option client option func
type Option func(*options)
type options struct {
builders []resolver.Builder
isLoadBalance bool
credentials credentials.TransportCredentials
unaryInterceptors []grpc.UnaryClientInterceptor
streamInterceptors []grpc.StreamClientInterceptor
dialOptions []grpc.DialOption
}
func defaultOptions() *options {
return &options{}
}
func (o *options) apply(opts ...Option) {
for _, opt := range opts {
opt(o)
}
}
// WithServiceDiscover set service discover
func WithServiceDiscover(builders ...resolver.Builder) Option {
return func(o *options) {
o.builders = builders
}
}
// WithLoadBalance set load balance
func WithLoadBalance() Option {
return func(o *options) {
o.isLoadBalance = true
}
}
// WithSecure set secure
func WithSecure(credential credentials.TransportCredentials) Option {
return func(o *options) {
o.credentials = credential
}
}
// WithUnaryInterceptor set unary interceptor
func WithUnaryInterceptor(interceptors ...grpc.UnaryClientInterceptor) Option {
return func(o *options) {
o.unaryInterceptors = interceptors
}
}
// WithStreamInterceptor set stream interceptor
func WithStreamInterceptor(interceptors ...grpc.StreamClientInterceptor) Option {
return func(o *options) {
o.streamInterceptors = interceptors
}
}
// WithDialOption set DialOption
func WithDialOption(dialOptions ...grpc.DialOption) Option {
return func(o *options) {
o.dialOptions = dialOptions
}
}
// NewClient create a new grpc client
func NewClient(endpoint string, opts ...Option) (*grpc.ClientConn, error) {
o := defaultOptions()
o.apply(opts...)
var dialOptions []grpc.DialOption
// service discovery
if len(o.builders) > 0 {
dialOptions = append(dialOptions, grpc.WithResolvers(o.builders...))
}
// load balance option
if o.isLoadBalance {
dialOptions = append(dialOptions, grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"round_robin":{}}]}`))
}
// secure option
if o.credentials == nil {
dialOptions = append(dialOptions, grpc.WithTransportCredentials(insecure.NewCredentials()))
} else {
dialOptions = append(dialOptions, grpc.WithTransportCredentials(o.credentials))
}
// custom dial option
if len(o.dialOptions) > 0 {
dialOptions = append(dialOptions, o.dialOptions...)
}
// custom unary interceptor option
if len(o.unaryInterceptors) > 0 {
dialOptions = append(dialOptions, grpc.WithChainUnaryInterceptor(o.unaryInterceptors...))
}
// custom stream interceptor option
if len(o.streamInterceptors) > 0 {
dialOptions = append(dialOptions, grpc.WithChainStreamInterceptor(o.streamInterceptors...))
}
return grpc.NewClient(endpoint, dialOptions...)
}
// Dial to grpc server
func Dial(_ context.Context, endpoint string, opts ...Option) (*grpc.ClientConn, error) {
return NewClient(endpoint, opts...)
}
package client
import (
"context"
"testing"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/resolver"
)
type builder struct{}
func (b *builder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) {
return nil, nil
}
func (b *builder) Scheme() string {
return ""
}
var unaryInterceptors = []grpc.UnaryClientInterceptor{
func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
return nil
},
}
var streamInterceptors = []grpc.StreamClientInterceptor{
func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
return nil, nil
},
}
func TestNewClient(t *testing.T) {
conn, err := NewClient("127.0.0.1:50082",
WithServiceDiscover(new(builder)),
WithLoadBalance(),
WithSecure(insecure.NewCredentials()),
WithUnaryInterceptor(unaryInterceptors...),
WithStreamInterceptor(streamInterceptors...),
WithDialOption(grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"round_robin":{}}]}`)),
)
defer conn.Close()
t.Log(conn, err)
time.Sleep(time.Second)
}
// Package grpccli is grpc client with support for service discovery, logging, load balancing, trace, metrics, retries, circuit breaker.
package grpccli
import (
"context"
"errors"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"gitlab.wanzhuangkj.com/tush/xpkg/grpc/gtls"
"gitlab.wanzhuangkj.com/tush/xpkg/grpc/interceptor"
"gitlab.wanzhuangkj.com/tush/xpkg/logger"
"gitlab.wanzhuangkj.com/tush/xpkg/servicerd/discovery"
)
// NewClient creates a new grpc client
func NewClient(endpoint string, opts ...Option) (*grpc.ClientConn, error) {
o := defaultOptions()
o.apply(opts...)
var clientOptions []grpc.DialOption
// service discovery
if o.discovery != nil {
clientOptions = append(clientOptions, grpc.WithResolvers(
discovery.NewBuilder(
o.discovery,
discovery.WithInsecure(o.discoveryInsecure),
)))
}
// load balance option
if o.enableLoadBalance {
clientOptions = append(clientOptions, grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"round_robin":{}}]}`))
}
// secure option
so, err := secureOption(o)
if err != nil {
return nil, err
}
clientOptions = append(clientOptions, so)
// token option
if o.enableToken {
clientOptions = append(clientOptions, interceptor.ClientTokenOption(
o.appID,
o.appKey,
o.isSecure(),
))
}
// unary options
clientOptions = append(clientOptions, unaryClientOptions(o))
// stream options
clientOptions = append(clientOptions, streamClientOptions(o))
// custom options
clientOptions = append(clientOptions, o.dialOptions...)
return grpc.NewClient(endpoint, clientOptions...)
}
// Dial to grpc server
// Deprecated: use NewClient instead
func Dial(_ context.Context, endpoint string, opts ...Option) (*grpc.ClientConn, error) {
return NewClient(endpoint, opts...)
}
func secureOption(o *options) (grpc.DialOption, error) {
switch o.secureType {
case secureOneWay: // server side certification
if o.certFile == "" {
return nil, errors.New("cert file is empty")
}
credentials, err := gtls.GetClientTLSCredentials(o.serverName, o.certFile)
if err != nil {
return nil, err
}
return grpc.WithTransportCredentials(credentials), nil
case secureTwoWay: // both client and server side certification
if o.caFile == "" {
return nil, errors.New("ca file is empty")
}
if o.certFile == "" {
return nil, errors.New("cert file is empty")
}
if o.keyFile == "" {
return nil, errors.New("key file is empty")
}
credentials, err := gtls.GetClientTLSCredentialsByCA(
o.serverName,
o.caFile,
o.certFile,
o.keyFile,
)
if err != nil {
return nil, err
}
return grpc.WithTransportCredentials(credentials), nil
default:
return grpc.WithTransportCredentials(insecure.NewCredentials()), nil
}
}
func unaryClientOptions(o *options) grpc.DialOption {
var unaryClientInterceptors []grpc.UnaryClientInterceptor
unaryClientInterceptors = append(unaryClientInterceptors, interceptor.UnaryClientRecovery())
if o.requestTimeout > 0 {
unaryClientInterceptors = append(unaryClientInterceptors, interceptor.UnaryClientTimeout(o.requestTimeout))
}
// request id
if o.enableRequestID {
unaryClientInterceptors = append(unaryClientInterceptors, interceptor.UnaryClientRequestID())
}
// logging
if o.enableLog {
unaryClientInterceptors = append(unaryClientInterceptors, interceptor.UnaryClientLog(logger.Get()))
}
// metrics
if o.enableMetrics {
unaryClientInterceptors = append(unaryClientInterceptors, interceptor.UnaryClientMetrics())
}
// circuit breaker
//if o.enableCircuitBreaker {
// unaryClientInterceptors = append(unaryClientInterceptors, interceptor.UnaryClientCircuitBreaker(
// // set rpc code for circuit breaker, default already includes codes.Internal and codes.Unavailable
// //interceptor.WithValidCode(codes.PermissionDenied),
// ))
//}
// retry
if o.enableRetry {
unaryClientInterceptors = append(unaryClientInterceptors, interceptor.UnaryClientRetry())
}
// trace
if o.enableTrace {
unaryClientInterceptors = append(unaryClientInterceptors, interceptor.UnaryClientTracing())
}
// custom unary interceptors
unaryClientInterceptors = append(unaryClientInterceptors, o.unaryInterceptors...)
return grpc.WithChainUnaryInterceptor(unaryClientInterceptors...)
}
func streamClientOptions(o *options) grpc.DialOption {
var streamClientInterceptors []grpc.StreamClientInterceptor
streamClientInterceptors = append(streamClientInterceptors, interceptor.StreamClientRecovery())
// request id
if o.enableRequestID {
streamClientInterceptors = append(streamClientInterceptors, interceptor.StreamClientRequestID())
}
// logging
if o.enableLog {
streamClientInterceptors = append(streamClientInterceptors, interceptor.StreamClientLog(logger.Get()))
}
// metrics
if o.enableMetrics {
streamClientInterceptors = append(streamClientInterceptors, interceptor.StreamClientMetrics())
}
// circuit breaker
//if o.enableCircuitBreaker {
// streamClientInterceptors = append(streamClientInterceptors, interceptor.StreamClientCircuitBreaker(
// // set rpc code for circuit breaker, default already includes codes.Internal and codes.Unavailable
// //interceptor.WithValidCode(codes.PermissionDenied),
// ))
//}
// retry
if o.enableRetry {
streamClientInterceptors = append(streamClientInterceptors, interceptor.StreamClientRetry())
}
// trace
if o.enableTrace {
streamClientInterceptors = append(streamClientInterceptors, interceptor.StreamClientTracing())
}
// custom stream interceptors
streamClientInterceptors = append(streamClientInterceptors, o.streamInterceptors...)
return grpc.WithChainStreamInterceptor(streamClientInterceptors...)
}
package grpccli
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/assert"
clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/zap"
"gitlab.wanzhuangkj.com/tush/xpkg/grpc/gtls/certfile"
"gitlab.wanzhuangkj.com/tush/xpkg/servicerd/registry/etcd"
)
func TestNewClient(t *testing.T) {
_, err := NewClient("localhost:8282")
assert.NoError(t, err)
_, err = Dial(context.Background(), "localhost:8282")
assert.NoError(t, err)
}
func TestNewClient2(t *testing.T) {
_, err := NewClient("localhost:8282",
WithEnableLog(zap.NewNop()),
WithEnableMetrics(),
WithToken(true, "grpc", "123456"),
WithEnableLoadBalance(),
WithEnableCircuitBreaker(),
WithEnableRetry(),
WithDiscovery(etcd.New(&clientv3.Client{})),
)
assert.NoError(t, err)
time.Sleep(time.Millisecond * 50)
}
func Test_unaryClientOptions(t *testing.T) {
o := &options{
enableToken: true,
enableLog: true,
enableRequestID: true,
enableTrace: true,
enableMetrics: true,
enableRetry: true,
enableLoadBalance: true,
enableCircuitBreaker: true,
}
scOpt := unaryClientOptions(o)
assert.NotNil(t, scOpt)
}
func Test_streamClientOptions(t *testing.T) {
o := &options{
enableToken: true,
enableLog: true,
enableRequestID: true,
enableTrace: true,
enableMetrics: true,
enableRetry: true,
enableLoadBalance: true,
enableCircuitBreaker: true,
}
scOpt := streamClientOptions(o)
assert.NotNil(t, scOpt)
}
func Test_secureOption(t *testing.T) {
o := &options{
secureType: "one-way",
serverName: "localhost",
certFile: certfile.Path("one-way/server.crt"),
}
// correct
opt, err := secureOption(o)
assert.NoError(t, err)
assert.NotNil(t, opt)
// error
o.certFile = ""
_, err = secureOption(o)
assert.Error(t, err)
o.certFile = "not found"
_, err = secureOption(o)
assert.Error(t, err)
o = &options{
secureType: "two-way",
serverName: "localhost",
caFile: certfile.Path("two-way/ca.pem"),
certFile: certfile.Path("two-way/client/client.pem"),
keyFile: certfile.Path("two-way/client/client.key"),
}
// correct
opt, err = secureOption(o)
assert.NoError(t, err)
assert.NotNil(t, opt)
// error
o.certFile = "not found"
_, err = secureOption(o)
assert.Error(t, err)
o.caFile = ""
_, err = secureOption(o)
assert.Error(t, err)
o.caFile = "not found"
o.certFile = ""
_, err = secureOption(o)
assert.Error(t, err)
o.certFile = "not found"
o.keyFile = ""
_, err = secureOption(o)
assert.Error(t, err)
o.secureType = ""
opt, err = secureOption(o)
assert.NoError(t, err)
assert.NotNil(t, opt)
}
package grpccli
import (
"time"
"go.uber.org/zap"
"google.golang.org/grpc"
"gitlab.wanzhuangkj.com/tush/xpkg/servicerd/registry"
)
var (
secureOneWay = "one-way"
secureTwoWay = "two-way"
)
// Option grpc dial options
type Option func(*options)
// options grpc dial options
type options struct {
requestTimeout time.Duration // request timeout, valid only for unary
// secure setting
secureType string // secure type "","one-way","two-way"
serverName string // server name
caFile string // ca file
certFile string // cert file
keyFile string // key file
// token setting
enableToken bool // whether to turn on token
appID string
appKey string
// interceptor setting
enableLog bool // whether to turn on the log
log *zap.Logger
enableRequestID bool // whether to turn on the request id
enableTrace bool // whether to turn on tracing
enableMetrics bool // whether to turn on metrics
enableRetry bool // whether to turn on retry
enableLoadBalance bool // whether to turn on load balance
enableCircuitBreaker bool // whether to turn on circuit breaker
discovery registry.Discovery // if not nil means use service discovery
discoveryInsecure bool
// custom setting
dialOptions []grpc.DialOption // custom options
unaryInterceptors []grpc.UnaryClientInterceptor // custom unary interceptor
streamInterceptors []grpc.StreamClientInterceptor // custom stream interceptor
}
func defaultOptions() *options {
return &options{
secureType: "",
serverName: "localhost",
certFile: "",
keyFile: "",
caFile: "",
enableLog: false,
discoveryInsecure: true,
dialOptions: nil,
unaryInterceptors: nil,
streamInterceptors: nil,
discovery: nil,
}
}
func (o *options) apply(opts ...Option) {
for _, opt := range opts {
opt(o)
}
}
// WithTimeout set dial timeout
func WithTimeout(d time.Duration) Option {
return func(o *options) {
o.requestTimeout = d
}
}
// WithEnableRequestID enable request id
func WithEnableRequestID() Option {
return func(o *options) {
o.enableRequestID = true
}
}
// WithEnableLog enable log
func WithEnableLog(log *zap.Logger) Option {
return func(o *options) {
o.enableLog = true
if log != nil {
o.log = log
return
}
o.log, _ = zap.NewProduction()
}
}
// WithEnableTrace enable trace
func WithEnableTrace() Option {
return func(o *options) {
o.enableTrace = true
}
}
// WithEnableMetrics enable metrics
func WithEnableMetrics() Option {
return func(o *options) {
o.enableMetrics = true
}
}
// WithEnableLoadBalance enable load balance
func WithEnableLoadBalance() Option {
return func(o *options) {
o.enableLoadBalance = true
}
}
// WithEnableRetry enable registry
func WithEnableRetry() Option {
return func(o *options) {
o.enableRetry = true
}
}
// WithEnableCircuitBreaker enable circuit breaker
func WithEnableCircuitBreaker() Option {
return func(o *options) {
o.enableCircuitBreaker = true
}
}
// WithDiscoveryInsecure setting discovery insecure
func WithDiscoveryInsecure(b bool) Option {
return func(o *options) {
o.discoveryInsecure = b
}
}
func (o *options) isSecure() bool {
if o.secureType == secureOneWay || o.secureType == secureTwoWay {
return true
}
return false
}
// WithSecure support setting one-way or two-way secure
func WithSecure(t string, serverName string, caFile string, certFile string, keyFile string) Option {
switch t {
case secureOneWay:
return WithOneWaySecure(serverName, certFile)
case secureTwoWay:
return WithTwoWaySecure(serverName, caFile, certFile, keyFile)
}
return func(o *options) {
o.secureType = t
}
}
// WithOneWaySecure set one-way secure
func WithOneWaySecure(serverName string, certFile string) Option {
return func(o *options) {
if serverName == "" {
serverName = "localhost"
}
o.secureType = secureOneWay
o.serverName = serverName
o.certFile = certFile
}
}
// WithTwoWaySecure set two-way secure
func WithTwoWaySecure(serverName string, caFile string, certFile string, keyFile string) Option {
return func(o *options) {
if serverName == "" {
serverName = "localhost"
}
o.secureType = secureTwoWay
o.serverName = serverName
o.caFile = caFile
o.certFile = certFile
o.keyFile = keyFile
}
}
// WithToken set token
func WithToken(enable bool, appID string, appKey string) Option {
return func(o *options) {
o.enableToken = enable
o.appID = appID
o.appKey = appKey
}
}
// WithDialOptions set dial options
func WithDialOptions(dialOptions ...grpc.DialOption) Option {
return func(o *options) {
o.dialOptions = append(o.dialOptions, dialOptions...)
}
}
// WithUnaryInterceptors set dial unaryInterceptors
func WithUnaryInterceptors(unaryInterceptors ...grpc.UnaryClientInterceptor) Option {
return func(o *options) {
o.unaryInterceptors = append(o.unaryInterceptors, unaryInterceptors...)
}
}
// WithStreamInterceptors set dial streamInterceptors
func WithStreamInterceptors(streamInterceptors ...grpc.StreamClientInterceptor) Option {
return func(o *options) {
o.streamInterceptors = append(o.streamInterceptors, streamInterceptors...)
}
}
// WithDiscovery set dial discovery
func WithDiscovery(discovery registry.Discovery) Option {
return func(o *options) {
o.discovery = discovery
}
}
package grpccli
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"gitlab.wanzhuangkj.com/tush/xpkg/grpc/interceptor"
"gitlab.wanzhuangkj.com/tush/xpkg/servicerd/registry"
)
func TestWithDialOptions(t *testing.T) {
testData := grpc.WithTransportCredentials(insecure.NewCredentials())
opt := WithDialOptions(testData)
o := new(options)
o.apply(opt)
assert.Equal(t, testData, o.dialOptions[0])
}
func TestWithDiscovery(t *testing.T) {
testData := new(registry.Discovery)
opt := WithDiscovery(*testData)
o := new(options)
o.apply(opt)
assert.NotEqual(t, testData, o.discovery)
}
func TestWithEnableCircuitBreaker(t *testing.T) {
opt := WithEnableCircuitBreaker()
o := new(options)
o.apply(opt)
assert.Equal(t, true, o.enableCircuitBreaker)
}
func TestWithEnableLoadBalance(t *testing.T) {
opt := WithEnableLoadBalance()
o := new(options)
o.apply(opt)
assert.Equal(t, true, o.enableLoadBalance)
}
func TestWithEnableRequestID(t *testing.T) {
opt := WithEnableRequestID()
o := new(options)
o.apply(opt)
assert.Equal(t, true, o.enableRequestID)
}
func TestWithEnableLog(t *testing.T) {
opt := WithEnableLog(nil)
testData := zap.NewNop()
opt = WithEnableLog(testData)
o := new(options)
o.apply(opt)
assert.Equal(t, testData, o.log)
}
func TestWithEnableMetrics(t *testing.T) {
opt := WithEnableMetrics()
o := new(options)
o.apply(opt)
assert.Equal(t, true, o.enableMetrics)
}
func TestWithEnableRetry(t *testing.T) {
opt := WithEnableRetry()
o := new(options)
o.apply(opt)
assert.Equal(t, true, o.enableRetry)
}
func TestWithEnableTrace(t *testing.T) {
opt := WithEnableTrace()
o := new(options)
o.apply(opt)
assert.Equal(t, true, o.enableTrace)
}
func TestWithStreamInterceptors(t *testing.T) {
testData := interceptor.StreamClientRetry()
opt := WithStreamInterceptors(testData)
o := new(options)
o.apply(opt)
assert.LessOrEqual(t, 1, len(o.streamInterceptors))
}
func TestWithTimeout(t *testing.T) {
testData := time.Second
opt := WithTimeout(testData)
o := new(options)
o.apply(opt)
assert.Equal(t, testData, o.requestTimeout)
}
func TestWithDiscoveryInsecure(t *testing.T) {
var testData bool
opt := WithDiscoveryInsecure(testData)
o := new(options)
o.apply(opt)
assert.Equal(t, testData, o.discoveryInsecure)
}
func TestWithUnaryInterceptors(t *testing.T) {
testData := interceptor.UnaryClientRetry()
opt := WithUnaryInterceptors(testData)
o := new(options)
o.apply(opt)
assert.LessOrEqual(t, 1, len(o.unaryInterceptors))
}
func Test_defaultOptions(t *testing.T) {
o := defaultOptions()
assert.NotNil(t, o)
}
func Test_options_apply(t *testing.T) {
opt := WithEnableRetry()
o := new(options)
o.apply(opt)
assert.Equal(t, true, o.enableRetry)
}
func Test_options_isSecure(t *testing.T) {
o := new(options)
secure := o.isSecure()
assert.Equal(t, false, secure)
o.secureType = secureOneWay
secure = o.isSecure()
assert.Equal(t, true, secure)
}
func TestWithSecure(t *testing.T) {
o := new(options)
opt := WithSecure("foo", "", "", "", "")
o.apply(opt)
assert.Equal(t, "foo", o.secureType)
opt = WithSecure(secureOneWay, "", "", "", "")
o.apply(opt)
assert.Equal(t, secureOneWay, o.secureType)
opt = WithSecure(secureTwoWay, "", "", "", "")
o.apply(opt)
assert.Equal(t, secureTwoWay, o.secureType)
}
#!/bin/bash
# As SAN certificates are required for GO version 1.15 and above, steps to generate a SAN certificate
# copy /etc/pki/tls/openssl.cnf to the current directory and modify openssl.cnf
# (1) uncomment copy_extensions = copy
# (2) uncomment req_extensions = v3_req
# (3) add v3_req
# [ v3_req ]
# subjectAltName = @alt_names
# (4) add alt_names
# [alt_names]
# DNS.1 = localhost
# openssl.cnf file
opensslCnfFile=./openssl.cnf
# generating one-way authentication certificates
mkdir one-way && cd one-way
openssl genrsa -out server.key 2048
openssl req -new -x509 -days 3650 -sha256 -key server.key -out server.crt -subj "/C=cn/OU=custer/O=custer/CN=localhost" -config ${opensslCnfFile} -extensions v3_req
echo "A one-way certificate has been generated and is stored in the directory 'one-way'"
cd ..
# generate two-way authentication certificates
mkdir two-way && cd two-way
# generate a ca certificate
openssl genrsa -out ca.key 2048
openssl req -new -x509 -days 3650 -key ca.key -out ca.pem -subj "/C=cn/OU=custer/O=custer/CN=localhost"
# server
openssl genpkey -algorithm RSA -out server.key
openssl req -new -nodes -key server.key -out server.csr -days 3650 -subj "/C=cn/OU=custer/O=custer/CN=localhost" -config ${opensslCnfFile} -extensions v3_req
openssl x509 -req -days 3650 -in server.csr -out server.pem -CA ca.pem -CAkey ca.key -CAcreateserial -extfile ${opensslCnfFile} -extensions v3_req
# client
openssl genpkey -algorithm RSA -out client.key
openssl req -new -nodes -key client.key -out client.csr -days 3650 -subj "/C=cn/OU=custer/O=custer/CN=localhost" -config ${opensslCnfFile} -extensions v3_req
openssl x509 -req -days 3650 -in client.csr -out client.pem -CA ca.pem -CAkey ca.key -CAcreateserial -extfile ${opensslCnfFile} -extensions v3_req
echo "A two-way certificate has been generated and is stored in the directory 'two-way'"
cd ..
// Package certfile is used to locate the certificate file.
package certfile
import (
"path/filepath"
"runtime"
)
var basepath string
func init() {
_, currentFile, _, _ := runtime.Caller(0) //nolint
basepath = filepath.Dir(currentFile)
}
// Path return absolute path
func Path(rel string) string {
if filepath.IsAbs(rel) {
return rel
}
return filepath.Join(basepath, rel)
}
package certfile
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestPath(t *testing.T) {
testData := "README.md"
file := Path(testData)
assert.Equal(t, true, strings.Contains(file, testData))
}
-----BEGIN CERTIFICATE-----
MIIDOTCCAiGgAwIBAgIJAJ4oghzJ3CFQMA0GCSqGSIb3DQEBCwUAMEMxCzAJBgNV
BAYTAmNuMQ8wDQYDVQQLDAZjdXN0ZXIxDzANBgNVBAoMBmN1c3RlcjESMBAGA1UE
AwwJbG9jYWxob3N0MB4XDTIyMDUwOTEyMzUzMVoXDTMyMDUwNjEyMzUzMVowQzEL
MAkGA1UEBhMCY24xDzANBgNVBAsMBmN1c3RlcjEPMA0GA1UECgwGY3VzdGVyMRIw
EAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
AQDeMaFH/Bp0bZdiWOKUYCr52TbZPXkXo//4I0urnH/PXdNrKhkiys4cP0byTY+Q
G7eVKY1qK70xnLxHzzjIcnZzWYUwrfBqopWmWKv3CFwjtv4/UXmnAI0uhx0pxXJD
ciX8hDo2opF0F/1WktVELYLnGqCWHgffYtshQDkVVR9zmBAjvSmS5G0J7sTtMIfs
P0Qvtoj9uJZLk+FjsUd3Rgy7/fooBYZp/m8rT2+HR0aIFRRDpTRKWhBO7RNC1s14
lBNp3pa9EnT0wu9egaRFzEhZsRiTbn8pMCYJVI71XG6ojp2cR9aWvyoTf72BOezF
fmQQuofBHbCkeFynreAgtyFNAgMBAAGjMDAuMBQGA1UdEQQNMAuCCWxvY2FsaG9z
dDAJBgNVHRMEAjAAMAsGA1UdDwQEAwIF4DANBgkqhkiG9w0BAQsFAAOCAQEAz47Q
+GL+dDhok98/vRCX4nUyCAJDwkWYroHk5pi9jMyB2XvGJiI8TMa7uS1zbCYqc9e7
oqhuquqscNnvLbESyLYPcOKfH9Lv671aMOKcSdLk9mDWOvWJXgjLwTMStOIr/maJ
2DfbRyXbeIuO3XfOJoBaUo0p/bT7pmshUpHtPLMvdTK0VN2pZ0UJzrEVcjFKLOrq
NzT+7df2pDXfOraSmgOX4qUOkRltNlEl1s2rxTyGchvQ4XX2V0BWTh1+eBmaA4y3
uneqvbiTgdBYUD55jI0lU4jPRqmJa/pn4A3yug9aZ5t04wU2Fts3N1iZZtAPy54V
wXRL+aYg5YlXNOvkYg==
-----END CERTIFICATE-----
-----BEGIN RSA PRIVATE KEY-----
MIIEogIBAAKCAQEA3jGhR/wadG2XYljilGAq+dk22T15F6P/+CNLq5x/z13TayoZ
IsrOHD9G8k2PkBu3lSmNaiu9MZy8R884yHJ2c1mFMK3waqKVplir9whcI7b+P1F5
pwCNLocdKcVyQ3Il/IQ6NqKRdBf9VpLVRC2C5xqglh4H32LbIUA5FVUfc5gQI70p
kuRtCe7E7TCH7D9EL7aI/biWS5PhY7FHd0YMu/36KAWGaf5vK09vh0dGiBUUQ6U0
SloQTu0TQtbNeJQTad6WvRJ09MLvXoGkRcxIWbEYk25/KTAmCVSO9VxuqI6dnEfW
lr8qE3+9gTnsxX5kELqHwR2wpHhcp63gILchTQIDAQABAoIBAARcKPQGqGY8eEn/
wIJ02KMKdh8RK70hBLbTynpVmdRx3OIvc0rRe/Xl7/h1OSn0wUd2B0ZcEVxV3QPz
twOH63cb/JcV8q/E/PbEqqswSM6Smq6XZLG4Owz8rb/SFgnoxYIM/i9wRTZn+hqm
yvSJiBYM2bXYZQMnJ3Ghlv0qLHHSMrG24sF6MaaXGhWNIWURP5cYFaKclDWxYnb4
NqvbHUsHq/dosqg707VFcPM+AJuKOUbUS2yn9SDz0dEZZ31RYaby0IbWJY5H7WIK
ff7zOTgJK5/L+14VGqz19xuqzgRYp98R4AKa8gZWgDSSDTmtMWcu3vB08M0RxeK2
4xYEQcECgYEA9XrEF/LW36x2a+xUyq/ocUShm/OjrUheeEb4dRffOQJD4BdTI94V
7lCo7ZTkIijH/a1ZHRHKbq5qRXfeDepY0NR61Dgp0a93y+9voumAV1LE4mj4QvfV
hA4Kem7gbkm1GLFIegfJ0UyJ0ezWg5nBsVekb1AI88teutbbMLCdY8UCgYEA57dj
rNM5H1L4A6w2U0rxKzxdo6Tpv41xGWo5F6JUrquZp5Xwewuc3phU2JqbtPxmAhKZ
rSDLytoNM4g9ghlnqHCrIVgvB4PnRhPsLN4+Xw80vVa7rT/9t3fHFJk2yf7D+gYe
/0ytjqr8lJK+n8TvccCYqLMx8v4DJYA/BHVoN+kCgYA/YWAp8sgp3iSBPvaxknOI
czjqxCA5iFrj4ScbTHuVA5G01TDhLOEqs+a52NyCOVdRlyVQDRzgMOY9Y3KQ0zX1
TTcdfhbGDfD3Va3UOUeqxDMTZhjbaZPWUa3A6MnHj/5TDsrwvvwLryBSdDz6o4NQ
H5nUJ6z4DUU7JmhXjPJGJQKBgCaFX0Wo67MgaOi7ZWCdcdBdPbfpv44/plCyTwF1
1BwhnO62R389I/wBWQGVWlNRLmgjzXZxoE99xnoNqSQKsyfWayyk61SVggotX7Lp
03acIYlkdNjNbZFlkSwEeI5GEzBqeha5GZVBKbJNXvFSnPfNK9PHzoL15XIDivZj
ykqZAoGAe5D1yHrSicR0vN8xRbVkdn0gAFJ7Dj/hBnYeFWWdpJ5yPj73z93/cZoR
tERIEjvza6ne7VOvVCw8qB0xVSBPqphaTM1dmcGMOCx6+wLlkABWiAWGx7m9obcd
HEfAbRLVcHOjik+tYyrQE54dck9C1N+0tlMMYZSAhD/cFb8/OJA=
-----END RSA PRIVATE KEY-----
差异被折叠。
-----BEGIN RSA PRIVATE KEY-----
MIIEogIBAAKCAQEArauYhYSNx9u6ErhA12/tw2tmHvi5uTNTjWPEpIyNl37K4ULX
alrteeS1cyu9sOE7LNTyMiHAPuIdFtyIm7i2JTGVe5+VB6bhMfqbB2G9jhGcUHNB
xcF8vIOg2AZ/6Z7pQ6npAt2DhP1SfadQ3CI+too3EGWdQaodTXVGpVbOlx9NhKah
KtfgZOjh2Y80enCC7+OF69qSUpXZcM4iFTpZIIBIx3aeoGdHEOU7m//v4D2Jokvb
zWhQYl95D8jYkucikHSdOpfklbGmgovXNjTrS6/Rtyr/EOTPY9TMZlervnlRCa9p
2p4HOTK5mIUkzhZ+WShbGnWQmUELvNGtEuZPiQIDAQABAoIBAClKqULll0gzh/Q3
dsNCS4exG2C1xoKwH2d4lyKAgJHKhbY1TD5vTBM1O6cceUd65bTtFICy4aCR5lSv
LpPHwRbqyR3RfX+KP/TAHugMZyNGMKI4JSU9scioiy+RrAwBynhaB5q0zDZsfJZ/
l4wfYEXKavktD8yzA7CM97UMBKaNWHzoEKWWtRVAYemA9JRhr5OO8aKMe+ug2qeu
ppJe8JzrBc6mlPs4hRXjIVE9ybKgDoMImGH1eP80Wrj5RQ1ZdyDypoPmzBac0yyQ
lB80DkT8rCTAsswvvdiPkaCNsbm12uHMMOlasrvVGHBih+Cz5G6nI5u4IfSePJ0h
L143iXkCgYEA1WKdgBFjwEQANH+xqF+W7mVCNTTkaQAL646T/97ZBsVvgJDxKvEO
fTLlX4+KSei8IULIfcZal5cqFNPA0UvpEVHc9+aq9oIZQ2a+RRx4UJAf+5gTVFjZ
k4jnece4XWX2qg67j1BLD+3SsxFpx6EXLvSTpoezpT9KDRwe//UGpEMCgYEA0FqL
BqdyC1zZ+GbMIYXl9/NfzxBNOoFqP8IR8ZCu4mBlBOhpEXlvMWW76+FdL6txOQGV
6La//vffsq24vZk6hN3T8ykOeoXtdguKyoWkEV4IA5L4ZJ4LRCFrAMzVFzgWAzg3
RHu9yYv6Qiyht78GIEPT/ITlP1urmOAmoB1VRkMCgYB22URMDmN2tOlAVFcJJqSU
B0YHCHynluUMwA7ilqZeRR1DiHcqqbSeOvjSbsphPAV8qQuuMgpHIGTJ0N82M4eO
o//k+08BmZikl9cl+yNwC7YklaE+e3ZD3B7BD2I6cw4dzbLdsaT9LEMMhYhbLfgR
qRuLx01hnoyKHL2PZlParQKBgCQ7qrO0iOOq+Qj2r4cg5vYwr7etqRCEkvqVgFNX
CuK5SrgIxsTQLmMTwxNpNLBmiyETwoMezNLFPnSvO1JVhFS40vQVbrwl8D64ESHZ
DcgrZw9gDqxIw8LMYPRZqrEIAuP6cboKHt4e5p19b34pzSHtSb8/STK0eWnziSQN
FdhLAoGADvOI+DM3VBP+Y/tMu0WfYgJVvfvWDoFrTPtf4otmqmlxoQ2cdgSx+WMv
Hm7B+x+8GhNZcqNmEYBEVTdyjoxmgCcTaNDzsCN2NNlUUOdFVoOztGXUnrswaRc5
qa8GdWYUIEdZTsKWAE0HPr0i4P+ck/F67mUfrdUrzTfMSp0nUhw=
-----END RSA PRIVATE KEY-----
-----BEGIN CERTIFICATE-----
MIIDWTCCAkGgAwIBAgIJALDASGB2020CMA0GCSqGSIb3DQEBCwUAMEMxCzAJBgNV
BAYTAmNuMQ8wDQYDVQQLDAZjdXN0ZXIxDzANBgNVBAoMBmN1c3RlcjESMBAGA1UE
AwwJbG9jYWxob3N0MB4XDTIyMDUwOTEyMzUzMVoXDTMyMDUwNjEyMzUzMVowQzEL
MAkGA1UEBhMCY24xDzANBgNVBAsMBmN1c3RlcjEPMA0GA1UECgwGY3VzdGVyMRIw
EAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
AQCtq5iFhI3H27oSuEDXb+3Da2Ye+Lm5M1ONY8SkjI2XfsrhQtdqWu155LVzK72w
4Tss1PIyIcA+4h0W3IibuLYlMZV7n5UHpuEx+psHYb2OEZxQc0HFwXy8g6DYBn/p
nulDqekC3YOE/VJ9p1DcIj62ijcQZZ1Bqh1NdUalVs6XH02EpqEq1+Bk6OHZjzR6
cILv44Xr2pJSldlwziIVOlkggEjHdp6gZ0cQ5Tub/+/gPYmiS9vNaFBiX3kPyNiS
5yKQdJ06l+SVsaaCi9c2NOtLr9G3Kv8Q5M9j1MxmV6u+eVEJr2nangc5MrmYhSTO
Fn5ZKFsadZCZQQu80a0S5k+JAgMBAAGjUDBOMB0GA1UdDgQWBBS5eLpBhPh0JtMI
8xBBb0VX+Q98zDAfBgNVHSMEGDAWgBS5eLpBhPh0JtMI8xBBb0VX+Q98zDAMBgNV
HRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAJN5lIRwOQIAsVR7TSipyCh7/L
oIdfWvH2wDi457UBvZdvWPI7o8pfMSO3ChBmoDPxo1sK0VejEnXnFLXIh7KHd9IG
BLeAxAXM0EZ8crTc1XItnVBvqWSDHHFBthPfFXNC0fDBfrkoHpiiS6LaVTYgDUV2
JK21cJ9zRF0r0VSGktPN+2bvVdozJj14jhK62ewdxEzOaJw19v9Y9T/834q0nhW9
5EFhSEQU4VBmoGyZGDiOnp+jk0/ffMu6AkQbu9xUgRGZ8acnYx/A7Bbv2hhSDyrZ
bPeFbJhxNzdRmryfffRYlepZCZ8fzgGynVFeXS4an/DZxVKPqSNwPzQPHoso
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE REQUEST-----
MIIBwjCCASsCAQAwQzELMAkGA1UEBhMCY24xDzANBgNVBAsMBmN1c3RlcjEPMA0G
A1UECgwGY3VzdGVyMRIwEAYDVQQDDAlsb2NhbGhvc3QwgZ8wDQYJKoZIhvcNAQEB
BQADgY0AMIGJAoGBAMVr7hmv1HQoipJa5ANdi2ipHtXZVzqPPWbZv30TKjAp1H7s
NxoDOpKuerNj9X6HwsPhbvykAfdlrNSw39Ied+0Z5gIIVZi3H/FEuT/cJeNEU51G
J3cu+BEtK5hs1gpv0AAsQKYNhzj/wU1rePUW7VPOYuj7cxnCUyxqa4EvjMRzAgMB
AAGgPzA9BgkqhkiG9w0BCQ4xMDAuMBQGA1UdEQQNMAuCCWxvY2FsaG9zdDAJBgNV
HRMEAjAAMAsGA1UdDwQEAwIF4DANBgkqhkiG9w0BAQsFAAOBgQBTr9Hv6p6eutgf
WBB8YsAEGwy2l1Zm9/EaR8TDxUwivSz+/O5v1bLk3KUSF13A765K8w4k+zs6ZjTE
Z7hTA66g+D+QUfNFLNtG2noZd6s9Z4pOQ5/0UswmE7N45rKjPJrasGaKn4pF6hbo
YNSrbIm/+C3WkSamv86YXSjI3CZZNw==
-----END CERTIFICATE REQUEST-----
-----BEGIN PRIVATE KEY-----
MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAMVr7hmv1HQoipJa
5ANdi2ipHtXZVzqPPWbZv30TKjAp1H7sNxoDOpKuerNj9X6HwsPhbvykAfdlrNSw
39Ied+0Z5gIIVZi3H/FEuT/cJeNEU51GJ3cu+BEtK5hs1gpv0AAsQKYNhzj/wU1r
ePUW7VPOYuj7cxnCUyxqa4EvjMRzAgMBAAECgYACzrogG2QGEt2Fn01GzvnAj0ck
+2ZGKutQnyAeAzvCW5XuCCXwdMNMera7/lvrZLrcVkRhy2NLxWJj0/Aa6NwDuMbi
pUgNIXvtn+2du3FtDMuyAJeSXPmyDO9MAfwceshCSULqwmr8G2MPfycmFwliGf5W
E2RbdmHoq/tlaoEYQQJBAO5J5gmZwvN7eCTe5s24kuKr6QBTo/Vnfd0YZq65H0eZ
+VKO7ABkxaOoSTw2PCENAnNCTNmGAaarYJLOmd+v03sCQQDUGGwnvnfzsecI/Qz0
SgUPDARmW0DyId+ADFfYcwo5lmuDqgexdluipNwF4bHElDGqHUtzqUnlcR7TMwsM
JuVpAkBlF/z8PbuzyMIkAl0xEglfGUf014dL6ehAEMYfgnJ+0hgwqmn9kMM4t4C6
htfjvb04YPxxnKS+rR5/qh8mA1ZZAkA1fTNJkq+NtgAeNDNgKIq+ELnLVpg6eHB1
Sqec3uZlP5o9ylPGXaMekZUrpo++k+EyldDBiqAoTS8I9IaEugqBAkEAgUtHWQqh
tHZqAJf9Trp6vrCue3YySmv7Z2RnUAjbEiCF2arQBurawmH4MRDozk+28UL9DUsl
NRlhp8+OEuhoQQ==
-----END PRIVATE KEY-----
-----BEGIN CERTIFICATE-----
MIICtTCCAZ2gAwIBAgIJAJRGbxsxpd2HMA0GCSqGSIb3DQEBCwUAMEMxCzAJBgNV
BAYTAmNuMQ8wDQYDVQQLDAZjdXN0ZXIxDzANBgNVBAoMBmN1c3RlcjESMBAGA1UE
AwwJbG9jYWxob3N0MB4XDTIyMDUwOTEyMzUzMVoXDTMyMDUwNjEyMzUzMVowQzEL
MAkGA1UEBhMCY24xDzANBgNVBAsMBmN1c3RlcjEPMA0GA1UECgwGY3VzdGVyMRIw
EAYDVQQDDAlsb2NhbGhvc3QwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMVr
7hmv1HQoipJa5ANdi2ipHtXZVzqPPWbZv30TKjAp1H7sNxoDOpKuerNj9X6HwsPh
bvykAfdlrNSw39Ied+0Z5gIIVZi3H/FEuT/cJeNEU51GJ3cu+BEtK5hs1gpv0AAs
QKYNhzj/wU1rePUW7VPOYuj7cxnCUyxqa4EvjMRzAgMBAAGjMDAuMBQGA1UdEQQN
MAuCCWxvY2FsaG9zdDAJBgNVHRMEAjAAMAsGA1UdDwQEAwIF4DANBgkqhkiG9w0B
AQsFAAOCAQEArVAzVK2GAlWRYNdPOBp097136xrWw932Yn2MbB+Bv+QQmGdhQrxJ
jhbaPUw+as9LkLAeohxx9hpNO/VkMxWC1sWEyU8xYcCR1ym3Z4+qNMiMnBEUPHCs
8Ox8IqRKcmOrHfM7UT/TjjIbkh2Sc/mXZThjSEfFWBNqz1DnW8/FrJGVkY4kZT6u
wPgDanCjPc7udFy86HCP4dVl3YFeoIV2qSzQWnVqDSGbtYgyL+iFJVPlctmHl5q+
37cp8hxN6PaANUQNX0GvGrKNr5eURxdc5c6djx5O8+j8ouwjsWOoZxptiFkA34et
mmbOsHt7/fqL2T3EiUuNBIS/um6FnM0lIw==
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE REQUEST-----
MIIBwjCCASsCAQAwQzELMAkGA1UEBhMCY24xDzANBgNVBAsMBmN1c3RlcjEPMA0G
A1UECgwGY3VzdGVyMRIwEAYDVQQDDAlsb2NhbGhvc3QwgZ8wDQYJKoZIhvcNAQEB
BQADgY0AMIGJAoGBAKyyEDoTJ2kVVupWFdrrILH4DdLprYt/jzpp6qgGuyD5xysV
mrqmspKkh6GUEKwnBdAi7qR3OCPmaUBQUI4R++rKbnZLkynb4Aq2S7rEtRglhz8g
ThM3H90V7nwdj0EzkIXaSfcX6PAOEDK1tDw75krIzXFj7qW/xyPYwcff42L3AgMB
AAGgPzA9BgkqhkiG9w0BCQ4xMDAuMBQGA1UdEQQNMAuCCWxvY2FsaG9zdDAJBgNV
HRMEAjAAMAsGA1UdDwQEAwIF4DANBgkqhkiG9w0BAQsFAAOBgQBrE6nQ0WVad0Zh
bpC3J/DtW6kECeTMK+oWA7TpRAKs6fhCfimhpJlp5JyaLeOXx1CXEI7OEpH1VPMr
2bHdSFs5PpadRUJlLNUtcA+gheCOqJWmx6xtFu/Jc+8jYaNj6nkMRQrjAsQomvRB
Xtz308RZhEPk8U7ZcNZKM0NMJM+v2A==
-----END CERTIFICATE REQUEST-----
-----BEGIN PRIVATE KEY-----
MIICdQIBADANBgkqhkiG9w0BAQEFAASCAl8wggJbAgEAAoGBAKyyEDoTJ2kVVupW
FdrrILH4DdLprYt/jzpp6qgGuyD5xysVmrqmspKkh6GUEKwnBdAi7qR3OCPmaUBQ
UI4R++rKbnZLkynb4Aq2S7rEtRglhz8gThM3H90V7nwdj0EzkIXaSfcX6PAOEDK1
tDw75krIzXFj7qW/xyPYwcff42L3AgMBAAECgYABJBgJWjELd7GgULtKO/12T44/
031rC4e1uhdrzseTuzK+rSDdlNZfM6kVvzWw/X0DWAe9nHAJhK8zVpSBq8q/Sphi
ie2hvGDBzbXiDcm/bgRDwSdaHzGEd+G13KKOjbL6azgZ60rd4WcxpNshNPYsDzUM
LFfE9Fs0N3tFMe3U8QJBAOAUMGUtYU5pgR5NH+TGCGz0hmEsYRGhuL0AS7T2iAKE
3KN9uqH9YMsDO7kvAxR0VDzLpeb+pAvF0ej26KY3xAkCQQDFTAEiAqgHjhRBRRmX
VJ+HF0qIFbVfDgJpYQnGWoiPU7h4Hb5MEeMOY0FzfYAMBlkOmrJtAiimUDSPueru
ka7/AkBJz9yxN2WaQr65kIY1Ada8rT+musuu1yrXd0V48sySp9lWMZBM0/4SYZpG
wemtzzQAYsTfdnnrNtqSduLj/fKhAkBE6yjWJZFmmjXvGuE2oKPdP8CUSukBXFZx
Uylj9YoQbxFYPCOWVQU4qGlbm3JYQPtpA5biR0fF2OyTbEFpttAnAkASJ7+Gle61
b7F4mzbMpuEjsMmf7negz6WCmTLGElOEFFUgJvODTDI02wd/LYSfoPbkdK3OFq8i
+KNNEswq3B8m
-----END PRIVATE KEY-----
-----BEGIN CERTIFICATE-----
MIICtTCCAZ2gAwIBAgIJAJRGbxsxpd2GMA0GCSqGSIb3DQEBCwUAMEMxCzAJBgNV
BAYTAmNuMQ8wDQYDVQQLDAZjdXN0ZXIxDzANBgNVBAoMBmN1c3RlcjESMBAGA1UE
AwwJbG9jYWxob3N0MB4XDTIyMDUwOTEyMzUzMVoXDTMyMDUwNjEyMzUzMVowQzEL
MAkGA1UEBhMCY24xDzANBgNVBAsMBmN1c3RlcjEPMA0GA1UECgwGY3VzdGVyMRIw
EAYDVQQDDAlsb2NhbGhvc3QwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAKyy
EDoTJ2kVVupWFdrrILH4DdLprYt/jzpp6qgGuyD5xysVmrqmspKkh6GUEKwnBdAi
7qR3OCPmaUBQUI4R++rKbnZLkynb4Aq2S7rEtRglhz8gThM3H90V7nwdj0EzkIXa
SfcX6PAOEDK1tDw75krIzXFj7qW/xyPYwcff42L3AgMBAAGjMDAuMBQGA1UdEQQN
MAuCCWxvY2FsaG9zdDAJBgNVHRMEAjAAMAsGA1UdDwQEAwIF4DANBgkqhkiG9w0B
AQsFAAOCAQEAnd2KBxIO96Xf8DGcuaCcjD4MKs5+ZREZ8HGNSq/aORR3Mk1pEc/W
zJntbgQbjV42kjW8aBpKDVZ0uZA1tTAlqbOJNEKn/f4s9JJ86gCb52Uwp+bE/a74
7snaALEAXaS/j8Ag7cge+ssq/o4cD9you/TMBjvs4xNRPMaQ1CxhJ1hr1BndSR7u
oUVWkTf9QfLMEB6OcLmgoH1PfDw7XUEws5GyHnaj72rI1qa6gLyyGJqiA4/Xp34G
AXn3jLELPeQoixIvuTBMgc6cF94BU7UbuSFtywj/B/u7hqhSLDbXtzSjCFJcAMVT
omPY5F+fFVt/Sfe1CZHvQ5eepo0Yjd2UPQ==
-----END CERTIFICATE-----
package gtls
import (
"crypto/tls"
"crypto/x509"
"os"
"google.golang.org/grpc/credentials"
)
// GetClientTLSCredentialsByCA two-way authentication via CA-issued root certificate
func GetClientTLSCredentialsByCA(serverName string, caFile string, certFile string, keyFile string) (credentials.TransportCredentials, error) {
// read and parse the information from the certificate file to obtain the certificate public key, key pair
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
if err != nil {
return nil, err
}
// create an empty CertPool
certPool := x509.NewCertPool()
ca, err := os.ReadFile(caFile)
if err != nil {
return nil, err
}
// attempts to parse the incoming PEM-encoded certificate. If the parsing is successful it will be added to the CertPool for later use
if ok := certPool.AppendCertsFromPEM(ca); !ok {
return nil, err
}
// building TLS-based TransportCredentials options
c := credentials.NewTLS(&tls.Config{
Certificates: []tls.Certificate{cert}, // set up a certificate chain that allows the inclusion of one or more
ServerName: serverName, // requirement to verify the client's certificate
RootCAs: certPool,
})
return c, err
}
// GetClientTLSCredentials TLS encryption
func GetClientTLSCredentials(serverName string, certFile string) (credentials.TransportCredentials, error) {
c, err := credentials.NewClientTLSFromFile(certFile, serverName)
if err != nil {
return nil, err
}
return c, err
}
package gtls
import (
"testing"
"gitlab.wanzhuangkj.com/tush/xpkg/grpc/gtls/certfile"
"github.com/stretchr/testify/assert"
)
func TestGetClientTLSCredentials(t *testing.T) {
credentials, err := GetClientTLSCredentials("localhost", certfile.Path("one-way/server.crt"))
assert.NoError(t, err)
assert.NotNil(t, credentials)
_, err = GetClientTLSCredentials("localhost", certfile.Path("one-way/notfound.crt"))
assert.Error(t, err)
}
func TestGetClientTLSCredentialsByCA(t *testing.T) {
credentials, err := GetClientTLSCredentialsByCA(
"localhost",
certfile.Path("two-way/ca.pem"),
certfile.Path("two-way/client/client.pem"),
certfile.Path("two-way/client/client.key"),
)
assert.NoError(t, err)
assert.NotNil(t, credentials)
_, err = GetClientTLSCredentialsByCA(
"localhost",
certfile.Path("two-way/ca.pem"),
certfile.Path("two-way/client/notfound.pem"),
certfile.Path("two-way/client/notfound.key"),
)
assert.Error(t, err)
_, err = GetClientTLSCredentialsByCA(
"localhost",
certfile.Path("two-way/notfound.pem"),
certfile.Path("two-way/client/client.pem"),
certfile.Path("two-way/client/client.key"),
)
assert.Error(t, err)
}
// Package gtls provides grpc secure connectivity, supporting both server-only authentication and client-server authentication.
package gtls
import (
"crypto/tls"
"crypto/x509"
"errors"
"os"
"google.golang.org/grpc/credentials"
)
// GetServerTLSCredentialsByCA two-way authentication via CA-issued root certificate
func GetServerTLSCredentialsByCA(caFile string, certFile string, keyFile string) (credentials.TransportCredentials, error) {
//read and parse the information from the certificate file to obtain the certificate public key, key pair
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
if err != nil {
return nil, err
}
// create an empty CertPool
certPool := x509.NewCertPool()
ca, err := os.ReadFile(caFile)
if err != nil {
return nil, err
}
//attempts to parse the incoming PEM-encoded certificate. If the parsing is successful it will be added to the CertPool for later use
if ok := certPool.AppendCertsFromPEM(ca); !ok {
return nil, errors.New("certPool.AppendCertsFromPEM err")
}
//building TLS-based TransportCredentials options
c := credentials.NewTLS(&tls.Config{
Certificates: []tls.Certificate{cert}, // set up a certificate chain that allows the inclusion of one or more
ClientAuth: tls.RequireAndVerifyClientCert, // requirement to verify the client's certificate
ClientCAs: certPool, // set the set of root certificates and use the mode set in ClientAuth for verification
})
return c, err
}
// GetServerTLSCredentials server-side authentication
func GetServerTLSCredentials(certFile string, keyFile string) (credentials.TransportCredentials, error) {
c, err := credentials.NewServerTLSFromFile(certFile, keyFile)
if err != nil {
return nil, err
}
return c, err
}
package gtls
import (
"testing"
"github.com/stretchr/testify/assert"
"gitlab.wanzhuangkj.com/tush/xpkg/grpc/gtls/certfile"
)
func TestGetServerTLSCredentials(t *testing.T) {
credentials, err := GetServerTLSCredentials(certfile.Path("one-way/server.crt"), certfile.Path("one-way/server.key"))
assert.NoError(t, err)
assert.NotNil(t, credentials)
_, err = GetServerTLSCredentials(certfile.Path("one-way/notfound.crt"), certfile.Path("one-way/notfound.key"))
assert.Error(t, err)
}
func TestGetServerTLSCredentialsByCA(t *testing.T) {
credentials, err := GetServerTLSCredentialsByCA(
certfile.Path("two-way/ca.pem"),
certfile.Path("two-way/server/server.pem"),
certfile.Path("two-way/server/server.key"),
)
assert.NoError(t, err)
assert.NotNil(t, credentials)
_, err = GetServerTLSCredentialsByCA(
certfile.Path("two-way/ca.pem"),
certfile.Path("two-way/server/notfound.pem"),
certfile.Path("two-way/server/notfound.key"),
)
assert.Error(t, err)
_, err = GetServerTLSCredentialsByCA(
certfile.Path("two-way/notfound.pem"),
certfile.Path("two-way/server/server.pem"),
certfile.Path("two-way/server/server.key"),
)
assert.Error(t, err)
}
// Package interceptor provides commonly used grpc client-side and server-side interceptors.
package interceptor
import (
"context"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"gitlab.wanzhuangkj.com/tush/xpkg/container/group"
"gitlab.wanzhuangkj.com/tush/xpkg/errcode"
"gitlab.wanzhuangkj.com/tush/xpkg/shield/circuitbreaker"
)
// ErrNotAllowed error not allowed.
var ErrNotAllowed = circuitbreaker.ErrNotAllowed
// CircuitBreakerOption set the circuit breaker circuitBreakerOptions.
type CircuitBreakerOption func(*circuitBreakerOptions)
type circuitBreakerOptions struct {
group *group.Group
// rpc code for circuit breaker, default already includes codes.Internal and codes.Unavailable
validCodes map[codes.Code]struct{}
// degrade handler for unary server
unaryServerDegradeHandler func(ctx context.Context, req interface{}) (reply interface{}, error error)
}
func defaultCircuitBreakerOptions() *circuitBreakerOptions {
return &circuitBreakerOptions{
group: group.NewGroup(func() interface{} {
return circuitbreaker.NewBreaker()
}),
validCodes: map[codes.Code]struct{}{
codes.Internal: {},
codes.Unavailable: {},
},
}
}
func (o *circuitBreakerOptions) apply(opts ...CircuitBreakerOption) {
for _, opt := range opts {
opt(o)
}
}
// WithGroup with circuit breaker group.
// NOTE: implements generics circuitbreaker.CircuitBreaker
func WithGroup(g *group.Group) CircuitBreakerOption {
return func(o *circuitBreakerOptions) {
if g != nil {
o.group = g
}
}
}
// WithValidCode rpc code to mark failed
func WithValidCode(code ...codes.Code) CircuitBreakerOption {
return func(o *circuitBreakerOptions) {
for _, c := range code {
o.validCodes[c] = struct{}{}
}
}
}
// WithUnaryServerDegradeHandler unary server degrade handler function
func WithUnaryServerDegradeHandler(handler func(ctx context.Context, req interface{}) (reply interface{}, error error)) CircuitBreakerOption {
return func(o *circuitBreakerOptions) {
o.unaryServerDegradeHandler = handler
}
}
// UnaryClientCircuitBreaker client-side unary circuit breaker interceptor
func UnaryClientCircuitBreaker(opts ...CircuitBreakerOption) grpc.UnaryClientInterceptor {
o := defaultCircuitBreakerOptions()
o.apply(opts...)
return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
breaker := o.group.Get(method).(circuitbreaker.CircuitBreaker)
if err := breaker.Allow(); err != nil {
// NOTE: when client reject request locally, keep adding counter let the drop ratio higher.
breaker.MarkFailed()
return errcode.StatusServiceUnavailable.ToRPCErr(err.Error())
}
err := invoker(ctx, method, req, reply, cc, opts...)
if err != nil {
// NOTE: need to check internal and service unavailable error
s, ok := status.FromError(err)
_, isHit := o.validCodes[s.Code()]
if ok && isHit {
breaker.MarkFailed()
} else {
breaker.MarkSuccess()
}
}
return err
}
}
// StreamClientCircuitBreaker client-side stream circuit breaker interceptor
func StreamClientCircuitBreaker(opts ...CircuitBreakerOption) grpc.StreamClientInterceptor {
o := defaultCircuitBreakerOptions()
o.apply(opts...)
return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
breaker := o.group.Get(method).(circuitbreaker.CircuitBreaker)
if err := breaker.Allow(); err != nil {
// NOTE: when client reject request locally, keep adding counter let the drop ratio higher.
breaker.MarkFailed()
return nil, errcode.StatusServiceUnavailable.ToRPCErr(err.Error())
}
clientStream, err := streamer(ctx, desc, cc, method, opts...)
if err != nil {
// NOTE: need to check internal and service unavailable error
s, ok := status.FromError(err)
_, isHit := o.validCodes[s.Code()]
if ok && isHit {
breaker.MarkFailed()
} else {
breaker.MarkSuccess()
}
}
return clientStream, err
}
}
// UnaryServerCircuitBreaker server-side unary circuit breaker interceptor
func UnaryServerCircuitBreaker(opts ...CircuitBreakerOption) grpc.UnaryServerInterceptor {
o := defaultCircuitBreakerOptions()
o.apply(opts...)
return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
breaker := o.group.Get(info.FullMethod).(circuitbreaker.CircuitBreaker)
if err := breaker.Allow(); err != nil {
// NOTE: when client reject request locally, keep adding let the drop ratio higher.
breaker.MarkFailed()
if o.unaryServerDegradeHandler != nil {
return o.unaryServerDegradeHandler(ctx, req)
}
return nil, errcode.StatusServiceUnavailable.ToRPCErr(err.Error())
}
reply, err := handler(ctx, req)
if err != nil {
// NOTE: need to check internal and service unavailable error
s, ok := status.FromError(err)
_, isHit := o.validCodes[s.Code()]
if ok && isHit {
breaker.MarkFailed()
} else {
breaker.MarkSuccess()
}
}
return reply, err
}
}
// StreamServerCircuitBreaker server-side stream circuit breaker interceptor
func StreamServerCircuitBreaker(opts ...CircuitBreakerOption) grpc.StreamServerInterceptor {
o := defaultCircuitBreakerOptions()
o.apply(opts...)
return func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
breaker := o.group.Get(info.FullMethod).(circuitbreaker.CircuitBreaker)
if err := breaker.Allow(); err != nil {
// NOTE: when client reject request locally, keep adding counter let the drop ratio higher.
breaker.MarkFailed()
return errcode.StatusServiceUnavailable.ToRPCErr(err.Error())
}
err := handler(srv, ss)
if err != nil {
// NOTE: need to check internal and service unavailable error
s, ok := status.FromError(err)
_, isHit := o.validCodes[s.Code()]
if ok && isHit {
breaker.MarkFailed()
} else {
breaker.MarkSuccess()
}
}
return err
}
}
package interceptor
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
"google.golang.org/grpc"
"gitlab.wanzhuangkj.com/tush/xpkg/container/group"
"gitlab.wanzhuangkj.com/tush/xpkg/errcode"
"gitlab.wanzhuangkj.com/tush/xpkg/shield/circuitbreaker"
"google.golang.org/grpc/codes"
)
func TestUnaryClientCircuitBreaker(t *testing.T) {
interceptor := UnaryClientCircuitBreaker(
WithGroup(group.NewGroup(func() interface{} {
return circuitbreaker.NewBreaker()
})),
WithValidCode(codes.PermissionDenied),
)
assert.NotNil(t, interceptor)
ivoker := func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, opts ...grpc.CallOption) error {
return errcode.StatusInternalServerError.ToRPCErr()
}
for i := 0; i < 110; i++ {
err := interceptor(context.Background(), "/test", nil, nil, nil, ivoker)
assert.Error(t, err)
}
ivoker = func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, opts ...grpc.CallOption) error {
return errcode.StatusInvalidParams.Err()
}
err := interceptor(context.Background(), "/test", nil, nil, nil, ivoker)
assert.Error(t, err)
}
func TestSteamClientCircuitBreaker(t *testing.T) {
interceptor := StreamClientCircuitBreaker()
assert.NotNil(t, interceptor)
streamer := func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, opts ...grpc.CallOption) (grpc.ClientStream, error) {
return nil, errcode.StatusInternalServerError.ToRPCErr()
}
for i := 0; i < 110; i++ {
_, err := interceptor(context.Background(), nil, nil, "/test", streamer)
assert.Error(t, err)
}
streamer = func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, opts ...grpc.CallOption) (grpc.ClientStream, error) {
return nil, errcode.StatusInvalidParams.Err()
}
_, err := interceptor(context.Background(), nil, nil, "/test", streamer)
assert.Error(t, err)
}
func TestUnaryServerCircuitBreaker(t *testing.T) {
degradeHandler := func(ctx context.Context, req interface{}) (reply interface{}, error error) {
return "degrade", errcode.StatusSuccess.ToRPCErr()
}
interceptor := UnaryServerCircuitBreaker(WithUnaryServerDegradeHandler(degradeHandler))
assert.NotNil(t, interceptor)
count := 0
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
count++
if count%2 == 0 {
return nil, errcode.StatusSuccess.ToRPCErr()
}
return nil, errcode.StatusInternalServerError.ToRPCErr()
}
successCount, failCount, degradeCount := 0, 0, 0
for i := 0; i < 1000; i++ {
reply, err := interceptor(context.Background(), nil, &grpc.UnaryServerInfo{FullMethod: "/test"}, handler)
if err != nil {
failCount++
continue
}
if reply == "degrade" {
degradeCount++
} else {
successCount++
}
}
t.Logf("successCount: %d, failCount: %d, degradeCount: %d", successCount, failCount, degradeCount)
handler = func(ctx context.Context, req interface{}) (interface{}, error) {
return nil, errcode.StatusInvalidParams.Err()
}
_, err := interceptor(context.Background(), nil, &grpc.UnaryServerInfo{FullMethod: "/test"}, handler)
t.Log(err)
}
func TestSteamServerCircuitBreaker(t *testing.T) {
interceptor := StreamServerCircuitBreaker()
assert.NotNil(t, interceptor)
handler := func(srv interface{}, stream grpc.ServerStream) error {
return errcode.StatusInternalServerError.ToRPCErr()
}
for i := 0; i < 110; i++ {
err := interceptor(nil, nil, &grpc.StreamServerInfo{FullMethod: "/test"}, handler)
assert.Error(t, err)
}
handler = func(srv interface{}, stream grpc.ServerStream) error {
return errcode.StatusInvalidParams.Err()
}
err := interceptor(nil, nil, &grpc.StreamServerInfo{FullMethod: "/test"}, handler)
assert.Error(t, err)
}
package interceptor
import (
"context"
"google.golang.org/grpc"
"google.golang.org/grpc/metadata"
)
var unaryClientInvoker = func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, opts ...grpc.CallOption) error {
return nil
}
type streamClient struct {
}
func (s streamClient) Header() (metadata.MD, error) {
return metadata.MD{}, nil
}
func (s streamClient) Trailer() metadata.MD {
return metadata.MD{}
}
func (s streamClient) CloseSend() error {
return nil
}
func (s streamClient) Context() context.Context {
return context.Background()
}
func (s streamClient) SendMsg(m interface{}) error {
return nil
}
func (s streamClient) RecvMsg(m interface{}) error {
return nil
}
var streamClientFunc = func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, opts ...grpc.CallOption) (grpc.ClientStream, error) {
return &streamClient{}, nil
}
// -----------------------------------------------------------------------------------------
var unaryServerInfo = &grpc.UnaryServerInfo{
Server: nil,
FullMethod: "/ping",
}
var unaryServerHandler = func(ctx context.Context, req interface{}) (interface{}, error) {
return nil, nil
}
func newStreamServer(ctx context.Context) *streamServer {
return &streamServer{
ctx: ctx,
}
}
type streamServer struct {
ctx context.Context
}
func (s streamServer) SetHeader(md metadata.MD) error {
return nil
}
func (s streamServer) SendHeader(md metadata.MD) error {
return nil
}
func (s streamServer) SetTrailer(md metadata.MD) {}
func (s streamServer) Context() context.Context {
return s.ctx
}
func (s streamServer) SendMsg(m interface{}) error {
return nil
}
func (s streamServer) RecvMsg(m interface{}) error {
return nil
}
var streamServerInfo = &grpc.StreamServerInfo{
FullMethod: "/test",
IsClientStream: false,
IsServerStream: false,
}
var streamServerHandler = func(srv interface{}, stream grpc.ServerStream) error {
return nil
}
package interceptor
import (
"context"
grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware/v2"
grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/auth"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
"gitlab.wanzhuangkj.com/tush/xpkg/jwt"
)
// ---------------------------------- client ----------------------------------
// SetJwtTokenToCtx set the token (excluding prefix Bearer) to the context in grpc client side
// Example:
//
// authorization := "Bearer jwt-token"
//
// ctx := SetJwtTokenToCtx(ctx, authorization)
// cli.GetByID(ctx, req)
func SetJwtTokenToCtx(ctx context.Context, token string) context.Context {
md, ok := metadata.FromOutgoingContext(ctx)
if ok {
md.Set(headerAuthorize, authScheme+" "+token)
} else {
md = metadata.Pairs(headerAuthorize, authScheme+" "+token)
}
return metadata.NewOutgoingContext(ctx, md)
}
// SetAuthToCtx set the authorization (including prefix Bearer) to the context in grpc client side
// Example:
//
// ctx := SetAuthToCtx(ctx, authorization)
// cli.GetByID(ctx, req)
func SetAuthToCtx(ctx context.Context, authorization string) context.Context {
md, ok := metadata.FromOutgoingContext(ctx)
if ok {
md.Set(headerAuthorize, authorization)
} else {
md = metadata.Pairs(headerAuthorize, authorization)
}
return metadata.NewOutgoingContext(ctx, md)
}
// ---------------------------------- server interceptor ----------------------------------
var (
headerAuthorize = "authorization"
// auth Scheme
authScheme = "Bearer"
// authentication information in ctx key name
authCtxClaimsName = "tokenInfo"
// collection of skip authentication methods
authIgnoreMethods = map[string]struct{}{}
)
// GetAuthorization combining tokens into authentication information
func GetAuthorization(token string) string {
return authScheme + " " + token
}
// GetAuthCtxKey get the name of Claims
func GetAuthCtxKey() string {
return authCtxClaimsName
}
// StandardVerifyFn verify function, tokenTail32 is the last 32 characters of the token.
type StandardVerifyFn = func(claims *jwt.Claims, tokenTail32 string) error
// CustomVerifyFn verify custom function, tokenTail32 is the last 32 characters of the token.
type CustomVerifyFn = func(claims *jwt.CustomClaims, tokenTail32 string) error
type verifyOptions struct {
verifyType int // 1: use StandardVerifyFn, 2:use CustomVerifyFn
standardVerifyFn StandardVerifyFn
customVerifyFn CustomVerifyFn
}
func defaultVerifyOptions() *verifyOptions {
return &verifyOptions{
verifyType: 1,
}
}
// AuthOption setting the Authentication Field
type AuthOption func(*authOptions)
// authOptions settings
type authOptions struct {
authScheme string
ctxClaimsName string
ignoreMethods map[string]struct{}
verifyOpts *verifyOptions
}
func defaultAuthOptions() *authOptions {
return &authOptions{
authScheme: authScheme,
ctxClaimsName: authCtxClaimsName,
ignoreMethods: make(map[string]struct{}), // ways to ignore forensics
verifyOpts: defaultVerifyOptions(),
}
}
func (o *authOptions) apply(opts ...AuthOption) {
for _, opt := range opts {
opt(o)
}
}
// WithAuthScheme set the message prefix for authentication
func WithAuthScheme(scheme string) AuthOption {
return func(o *authOptions) {
o.authScheme = scheme
}
}
// WithAuthClaimsName set the key name of the information in ctx for authentication
func WithAuthClaimsName(claimsName string) AuthOption {
return func(o *authOptions) {
o.ctxClaimsName = claimsName
}
}
// WithAuthIgnoreMethods ways to ignore forensics
// fullMethodName format: /packageName.serviceName/methodName,
// example /api.userExample.v1.userExampleService/GetByID
func WithAuthIgnoreMethods(fullMethodNames ...string) AuthOption {
return func(o *authOptions) {
for _, method := range fullMethodNames {
o.ignoreMethods[method] = struct{}{}
}
}
}
// WithStandardVerify set the standard verify function for authentication
func WithStandardVerify(verify StandardVerifyFn) AuthOption {
return func(o *authOptions) {
if o.verifyOpts == nil {
o.verifyOpts = defaultVerifyOptions()
}
o.verifyOpts.verifyType = 1
o.verifyOpts.standardVerifyFn = verify
}
}
// WithCustomVerify set the custom verify function for authentication
func WithCustomVerify(verify CustomVerifyFn) AuthOption {
return func(o *authOptions) {
if o.verifyOpts == nil {
o.verifyOpts = defaultVerifyOptions()
}
o.verifyOpts.verifyType = 2
o.verifyOpts.customVerifyFn = verify
}
}
// -------------------------------------------------------------------------------------------
// verify authorization from context, support standard and custom verify processing
func jwtVerify(ctx context.Context, opt *verifyOptions) (context.Context, error) {
if opt == nil {
opt = &verifyOptions{
verifyType: 1, // default use VerifyGeneralFn
}
}
token, err := grpc_auth.AuthFromMD(ctx, authScheme) // key is authScheme
if err != nil {
return ctx, status.Errorf(codes.Unauthenticated, "%v", err)
}
if len(token) <= 100 {
return ctx, status.Errorf(codes.Unauthenticated, "authorization is illegal")
}
// custom claims
if opt.verifyType == 2 {
var claims *jwt.CustomClaims
claims, err = jwt.ParseCustomToken(token)
if err != nil {
return ctx, status.Errorf(codes.Unauthenticated, "%v", err)
}
if opt.customVerifyFn != nil {
tokenTail32 := token[len(token)-16:]
err = opt.customVerifyFn(claims, tokenTail32)
if err != nil {
return ctx, status.Errorf(codes.Unauthenticated, "%v", err)
}
}
newCtx := context.WithValue(ctx, authCtxClaimsName, claims) //nolint
return newCtx, nil
}
// standard claims
claims, err := jwt.ParseToken(token)
if err != nil {
return ctx, status.Errorf(codes.Unauthenticated, "%v", err)
}
if opt.standardVerifyFn != nil {
tokenTail32 := token[len(token)-16:]
err = opt.standardVerifyFn(claims, tokenTail32)
if err != nil {
return ctx, status.Errorf(codes.Unauthenticated, "%v", err)
}
}
newCtx := context.WithValue(ctx, authCtxClaimsName, claims) //nolint
return newCtx, nil
}
// GetJwtClaims get the jwt standard claims from context, contains fixed fields uid and name
func GetJwtClaims(ctx context.Context) (*jwt.Claims, bool) {
v, ok := ctx.Value(authCtxClaimsName).(*jwt.Claims)
return v, ok
}
// GetJwtCustomClaims get the jwt custom claims from context, contains custom fields
func GetJwtCustomClaims(ctx context.Context) (*jwt.CustomClaims, bool) {
v, ok := ctx.Value(authCtxClaimsName).(*jwt.CustomClaims)
return v, ok
}
// UnaryServerJwtAuth jwt unary interceptor
func UnaryServerJwtAuth(opts ...AuthOption) grpc.UnaryServerInterceptor {
o := defaultAuthOptions()
o.apply(opts...)
authScheme = o.authScheme
authCtxClaimsName = o.ctxClaimsName
authIgnoreMethods = o.ignoreMethods
verifyOpt := o.verifyOpts
return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
var newCtx context.Context
var err error
if _, ok := authIgnoreMethods[info.FullMethod]; ok {
newCtx = ctx
} else {
newCtx, err = jwtVerify(ctx, verifyOpt)
if err != nil {
return nil, err
}
}
return handler(newCtx, req)
}
}
// StreamServerJwtAuth jwt stream interceptor
func StreamServerJwtAuth(opts ...AuthOption) grpc.StreamServerInterceptor {
o := defaultAuthOptions()
o.apply(opts...)
authScheme = o.authScheme
authCtxClaimsName = o.ctxClaimsName
authIgnoreMethods = o.ignoreMethods
verifyOpt := o.verifyOpts
return func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
var newCtx context.Context
var err error
if _, ok := authIgnoreMethods[info.FullMethod]; ok {
newCtx = stream.Context()
} else {
newCtx, err = jwtVerify(stream.Context(), verifyOpt)
if err != nil {
return err
}
}
wrapped := grpc_middleware.WrapServerStream(stream)
wrapped.WrappedContext = newCtx
return handler(srv, wrapped)
}
}
package interceptor
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
"gitlab.wanzhuangkj.com/tush/xpkg/jwt"
"gitlab.wanzhuangkj.com/tush/xpkg/utils"
)
var (
expectedUid = "100"
expectedName = "tom"
expectedFields = jwt.KV{"id": utils.StrToUint64(expectedUid), "name": expectedName, "age": 10}
)
func standardVerifyHandler(claims *jwt.Claims, tokenTail32 string) error {
// token := getToken(claims.UID)
// if token[len(token)-32:] != tokenTail32 { return err }
if claims.UID != expectedUid || claims.Name != expectedName {
return status.Error(codes.Unauthenticated, "id or name not match")
}
return nil
}
func customVerifyHandler(claims *jwt.CustomClaims, tokenTail32 string) error {
err := status.Error(codes.Unauthenticated, "custom verify failed")
//token, fields := getToken(id)
// if token[len(token)-32:] != tokenTail32 { return err }
id, exist := claims.GetUint64("id")
if !exist || id != expectedFields["id"] {
return err
}
name, exist := claims.GetString("name")
if !exist || name != expectedFields["name"] {
return err
}
age, exist := claims.GetInt("age")
if !exist || age != expectedFields["age"] {
return err
}
return nil
}
func TestJwtVerify(t *testing.T) {
jwt.Init()
ctx := context.Background()
token, _, _ := jwt.GenerateToken(expectedUid, expectedName)
// success test
ctx = metadata.NewIncomingContext(ctx, metadata.MD{headerAuthorize: []string{GetAuthorization(token)}})
newCtx, err := jwtVerify(ctx, nil)
assert.NoError(t, err)
claims, ok := GetJwtClaims(newCtx)
assert.True(t, ok)
assert.Equal(t, expectedUid, claims.UID)
// success test
ctx = metadata.NewIncomingContext(ctx, metadata.MD{headerAuthorize: []string{GetAuthorization(token)}})
newCtx, err = jwtVerify(ctx, &verifyOptions{verifyType: 1, standardVerifyFn: standardVerifyHandler})
assert.NoError(t, err)
claims, ok = GetJwtClaims(newCtx)
assert.True(t, ok)
assert.Equal(t, expectedUid, claims.UID)
authorization := []string{GetAuthorization("error token......")}
// authorization format error, missing token
ctx = metadata.NewIncomingContext(context.Background(), metadata.MD{headerAuthorize: authorization})
_, err = jwtVerify(ctx, nil)
assert.Error(t, err)
// authorization format error, missing Bearer
ctx = context.WithValue(context.Background(), headerAuthorize, authorization)
_, err = jwtVerify(ctx, nil)
assert.Error(t, err)
}
func TestJwtCustomVerify(t *testing.T) {
jwt.Init()
ctx := context.Background()
token, _ := jwt.GenerateCustomToken(expectedFields)
verifyOpt := &verifyOptions{verifyType: 2}
// success test
ctx = metadata.NewIncomingContext(ctx, metadata.MD{headerAuthorize: []string{GetAuthorization(token)}})
newCtx, err := jwtVerify(ctx, verifyOpt)
assert.NoError(t, err)
claims, ok := GetJwtCustomClaims(newCtx)
assert.True(t, ok)
assert.Equal(t, expectedName, claims.Fields["name"])
// success test
ctx = metadata.NewIncomingContext(ctx, metadata.MD{headerAuthorize: []string{GetAuthorization(token)}})
verifyOpt.customVerifyFn = customVerifyHandler
newCtx, err = jwtVerify(ctx, verifyOpt)
assert.NoError(t, err)
claims, ok = GetJwtCustomClaims(newCtx)
assert.True(t, ok)
assert.Equal(t, expectedName, claims.Fields["name"])
authorization := []string{GetAuthorization("mock token......")}
// authorization format error, missing token
ctx = metadata.NewIncomingContext(context.Background(), metadata.MD{headerAuthorize: authorization})
_, err = jwtVerify(ctx, verifyOpt)
assert.Error(t, err)
// authorization format error, missing Bearer
ctx = context.WithValue(context.Background(), headerAuthorize, authorization)
_, err = jwtVerify(ctx, verifyOpt)
assert.Error(t, err)
}
func TestUnaryServerJwtAuth(t *testing.T) {
interceptor := UnaryServerJwtAuth(WithStandardVerify(standardVerifyHandler))
assert.NotNil(t, interceptor)
// mock client ctx
jwt.Init()
token, _, _ := jwt.GenerateToken(expectedUid, expectedName)
ctx := metadata.NewIncomingContext(context.Background(), metadata.MD{headerAuthorize: []string{GetAuthorization(token)}})
_, err := interceptor(ctx, nil, unaryServerInfo, unaryServerHandler)
assert.NoError(t, err)
ctx = metadata.NewIncomingContext(context.Background(), metadata.MD{headerAuthorize: []string{GetAuthorization("error token......")}})
_, err = interceptor(ctx, nil, unaryServerInfo, unaryServerHandler)
assert.Error(t, err)
}
func TestUnaryServerJwtCustomAuth(t *testing.T) {
interceptor := UnaryServerJwtAuth(WithCustomVerify(customVerifyHandler))
assert.NotNil(t, interceptor)
// mock client ctx
jwt.Init()
token, _ := jwt.GenerateCustomToken(expectedFields)
ctx := metadata.NewIncomingContext(context.Background(), metadata.MD{headerAuthorize: []string{GetAuthorization(token)}})
_, err := interceptor(ctx, nil, unaryServerInfo, unaryServerHandler)
assert.NoError(t, err)
ctx = metadata.NewIncomingContext(context.Background(), metadata.MD{headerAuthorize: []string{GetAuthorization("error token......")}})
_, err = interceptor(ctx, nil, unaryServerInfo, unaryServerHandler)
assert.Error(t, err)
}
func TestStreamServerJwtAuth(t *testing.T) {
interceptor := StreamServerJwtAuth()
assert.NotNil(t, interceptor)
jwt.Init()
token, _, _ := jwt.GenerateToken(expectedUid, expectedName)
ctx := metadata.NewIncomingContext(context.Background(), metadata.MD{headerAuthorize: []string{authScheme + " " + token}})
err := interceptor(nil, newStreamServer(ctx), streamServerInfo, streamServerHandler)
assert.NoError(t, err)
err = interceptor(nil, newStreamServer(context.Background()), streamServerInfo, streamServerHandler)
assert.Error(t, err)
}
func TestStreamServerJwtCustomAuth(t *testing.T) {
interceptor := StreamServerJwtAuth(WithCustomVerify(nil))
assert.NotNil(t, interceptor)
jwt.Init()
token, _ := jwt.GenerateCustomToken(expectedFields)
ctx := metadata.NewIncomingContext(context.Background(), metadata.MD{headerAuthorize: []string{authScheme + " " + token}})
err := interceptor(nil, newStreamServer(ctx), streamServerInfo, streamServerHandler)
assert.NoError(t, err)
err = interceptor(nil, newStreamServer(context.Background()), streamServerInfo, streamServerHandler)
assert.Error(t, err)
}
func TestGetAuthCtxKey(t *testing.T) {
key := GetAuthCtxKey()
assert.Equal(t, authCtxClaimsName, key)
}
func TestGetAuthorization(t *testing.T) {
testData := "token"
authorization := GetAuthorization(testData)
assert.Equal(t, authScheme+" "+testData, authorization)
}
func TestAuthOptions(t *testing.T) {
o := defaultAuthOptions()
o.apply(WithAuthScheme(authScheme))
assert.Equal(t, authScheme, o.authScheme)
o.apply(WithAuthClaimsName(authCtxClaimsName))
assert.Equal(t, authCtxClaimsName, o.ctxClaimsName)
o.apply(WithAuthIgnoreMethods("/metrics"))
assert.Equal(t, struct{}{}, o.ignoreMethods["/metrics"])
o.apply(WithStandardVerify(nil))
assert.Equal(t, 1, o.verifyOpts.verifyType)
o.apply(WithStandardVerify(standardVerifyHandler))
assert.Equal(t, 1, o.verifyOpts.verifyType)
o.apply(WithCustomVerify(nil))
assert.Equal(t, 2, o.verifyOpts.verifyType)
o.apply(WithCustomVerify(customVerifyHandler))
assert.Equal(t, 2, o.verifyOpts.verifyType)
}
func TestSetJWTTokenToCtx(t *testing.T) {
jwt.Init()
ctx := context.Background()
token, _, _ := jwt.GenerateToken(expectedUid, expectedName)
expected := []string{GetAuthorization(token)}
ctx = SetJwtTokenToCtx(ctx, token)
md, _ := metadata.FromOutgoingContext(ctx)
assert.Equal(t, expected, md.Get(headerAuthorize))
}
func TestSetAuthToCtx(t *testing.T) {
jwt.Init()
ctx := context.Background()
token, _, _ := jwt.GenerateToken(expectedUid, expectedName)
authorization := GetAuthorization(token)
expected := []string{authorization}
ctx = SetAuthToCtx(ctx, authorization)
md, _ := metadata.FromOutgoingContext(ctx)
assert.Equal(t, expected, md.Get(headerAuthorize))
}
package interceptor
import (
"context"
"encoding/json"
"time"
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/status"
zapLog "gitlab.wanzhuangkj.com/tush/xpkg/logger"
)
var contentMark = []byte(" ...... ")
// ---------------------------------- client interceptor ----------------------------------
// UnaryClientLog client log unary interceptor
func UnaryClientLog(logger *zap.Logger, opts ...LogOption) grpc.UnaryClientInterceptor {
o := defaultLogOptions()
o.apply(opts...)
if logger == nil {
logger, _ = zap.NewProduction()
}
if o.isReplaceGRPCLogger {
zapLog.ReplaceGRPCLoggerV2(logger)
}
return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
startTime := time.Now()
var reqIDField zap.Field
if requestID := ClientCtxRequestID(ctx); requestID != "" {
reqIDField = zap.String(ContextRequestIDKey, requestID)
} else {
reqIDField = zap.Skip()
}
err := invoker(ctx, method, req, reply, cc, opts...)
fields := []zap.Field{
zap.String("code", status.Code(err).String()),
zap.Error(err),
zap.String("type", "unary"),
zap.String("method", method),
zap.String("cost", time.Since(startTime).String()),
reqIDField,
}
logger.Info("invoker result", fields...)
return err
}
}
// StreamClientLog client log stream interceptor
func StreamClientLog(logger *zap.Logger, opts ...LogOption) grpc.StreamClientInterceptor {
o := defaultLogOptions()
o.apply(opts...)
if logger == nil {
logger, _ = zap.NewProduction()
}
if o.isReplaceGRPCLogger {
zapLog.ReplaceGRPCLoggerV2(logger)
}
return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string,
streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
startTime := time.Now()
var reqIDField zap.Field
if requestID := ClientCtxRequestID(ctx); requestID != "" {
reqIDField = zap.String(ContextRequestIDKey, requestID)
} else {
reqIDField = zap.Skip()
}
clientStream, err := streamer(ctx, desc, cc, method, opts...)
fields := []zap.Field{
zap.String("code", status.Code(err).String()),
zap.Error(err),
zap.String("type", "stream"),
zap.String("method", method),
zap.String("cost", time.Since(startTime).String()),
reqIDField,
}
logger.Info("invoker result", fields...)
return clientStream, err
}
}
// ---------------------------------- server interceptor ----------------------------------
var defaultMaxLength = 300 // max length of response data to print
var ignoreLogMethods = map[string]struct{}{} // ignore printing methods
var defaultMarshalFn = func(reply interface{}) []byte {
data, _ := json.Marshal(reply)
return data
}
// LogOption log settings
type LogOption func(*logOptions)
type logOptions struct {
maxLength int
fields map[string]interface{}
ignoreMethods map[string]struct{}
isReplaceGRPCLogger bool
marshalFn func(reply interface{}) []byte // default json.Marshal
}
func defaultLogOptions() *logOptions {
return &logOptions{
maxLength: defaultMaxLength,
fields: make(map[string]interface{}),
ignoreMethods: make(map[string]struct{}),
marshalFn: defaultMarshalFn,
}
}
func (o *logOptions) apply(opts ...LogOption) {
for _, opt := range opts {
opt(o)
}
}
// WithMaxLen logger content max length
func WithMaxLen(maxLen int) LogOption {
return func(o *logOptions) {
if maxLen > 0 {
o.maxLength = maxLen
}
}
}
// WithReplaceGRPCLogger replace grpc logger v2
func WithReplaceGRPCLogger() LogOption {
return func(o *logOptions) {
o.isReplaceGRPCLogger = true
}
}
// WithLogFields adding a custom print field
func WithLogFields(kvs map[string]interface{}) LogOption {
return func(o *logOptions) {
if len(kvs) == 0 {
return
}
o.fields = kvs
}
}
// WithMarshalFn custom response data marshal function
func WithMarshalFn(fn func(reply interface{}) []byte) LogOption {
return func(o *logOptions) {
if fn != nil {
o.marshalFn = fn
}
}
}
// WithLogIgnoreMethods ignore printing methods
// fullMethodName format: /packageName.serviceName/methodName,
// example /api.userExample.v1.userExampleService/GetByID
func WithLogIgnoreMethods(fullMethodNames ...string) LogOption {
return func(o *logOptions) {
for _, method := range fullMethodNames {
o.ignoreMethods[method] = struct{}{}
}
}
}
// UnaryServerLog server-side log unary interceptor
func UnaryServerLog(logger *zap.Logger, opts ...LogOption) grpc.UnaryServerInterceptor {
o := defaultLogOptions()
o.apply(opts...)
ignoreLogMethods = o.ignoreMethods
if logger == nil {
logger, _ = zap.NewProduction()
}
if o.isReplaceGRPCLogger {
zapLog.ReplaceGRPCLoggerV2(logger)
}
return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
// ignore printing of the specified method
if _, ok := ignoreLogMethods[info.FullMethod]; ok {
return handler(ctx, req)
}
startTime := time.Now()
requestID := ServerCtxRequestID(ctx)
fields := []zap.Field{
zap.String("type", "unary"),
zap.String("method", info.FullMethod),
zap.Any("request", req),
}
if requestID != "" {
fields = append(fields, zap.String(ContextRequestIDKey, requestID))
}
logger.Info("<<<<<<<<<", fields...)
resp, err := handler(ctx, req)
data := o.marshalFn(resp)
if len(data) > o.maxLength {
data = append(data[:o.maxLength], contentMark...)
}
fields = []zap.Field{
zap.String("code", status.Code(err).String()),
zap.Error(err),
zap.String("type", "unary"),
zap.String("method", info.FullMethod),
zap.ByteString("data", data),
zap.String("cost", time.Since(startTime).String()),
}
if requestID != "" {
fields = append(fields, zap.String(ContextRequestIDKey, requestID))
}
logger.Info(">>>>>>>>>", fields...)
return resp, err
}
}
// UnaryServerSimpleLog server-side log unary interceptor, only print response
func UnaryServerSimpleLog(logger *zap.Logger, opts ...LogOption) grpc.UnaryServerInterceptor {
o := defaultLogOptions()
o.apply(opts...)
ignoreLogMethods = o.ignoreMethods
if logger == nil {
logger, _ = zap.NewProduction()
}
if o.isReplaceGRPCLogger {
zapLog.ReplaceGRPCLoggerV2(logger)
}
return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
// ignore printing of the specified method
if _, ok := ignoreLogMethods[info.FullMethod]; ok {
return handler(ctx, req)
}
startTime := time.Now()
requestID := ServerCtxRequestID(ctx)
resp, err := handler(ctx, req)
fields := []zap.Field{
zap.String("code", status.Code(err).String()),
zap.Error(err),
zap.String("type", "unary"),
zap.String("method", info.FullMethod),
zap.String("cost", time.Since(startTime).String()),
}
if requestID != "" {
fields = append(fields, zap.String(ContextRequestIDKey, requestID))
}
logger.Info("[GRPC] response", fields...)
return resp, err
}
}
// StreamServerLog Server-side log stream interceptor
func StreamServerLog(logger *zap.Logger, opts ...LogOption) grpc.StreamServerInterceptor {
o := defaultLogOptions()
o.apply(opts...)
ignoreLogMethods = o.ignoreMethods
if logger == nil {
logger, _ = zap.NewProduction()
}
if o.isReplaceGRPCLogger {
zapLog.ReplaceGRPCLoggerV2(logger)
}
return func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
// ignore printing of the specified method
if _, ok := ignoreLogMethods[info.FullMethod]; ok {
return handler(srv, stream)
}
startTime := time.Now()
requestID := ServerCtxRequestID(stream.Context())
fields := []zap.Field{
zap.String("type", "stream"),
zap.String("method", info.FullMethod),
}
if requestID != "" {
fields = append(fields, zap.String(ContextRequestIDKey, requestID))
}
logger.Info("<<<<<<<<<", fields...)
err := handler(srv, stream)
fields = []zap.Field{
zap.String("code", status.Code(err).String()),
zap.String("type", "stream"),
zap.String("method", info.FullMethod),
zap.String("cost", time.Since(startTime).String()),
}
if requestID != "" {
fields = append(fields, zap.String(ContextRequestIDKey, requestID))
}
logger.Info(">>>>>>>>>", fields...)
return err
}
}
// StreamServerSimpleLog Server-side log stream interceptor, only print response
func StreamServerSimpleLog(logger *zap.Logger, opts ...LogOption) grpc.StreamServerInterceptor {
o := defaultLogOptions()
o.apply(opts...)
ignoreLogMethods = o.ignoreMethods
if logger == nil {
logger, _ = zap.NewProduction()
}
if o.isReplaceGRPCLogger {
zapLog.ReplaceGRPCLoggerV2(logger)
}
return func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
// ignore printing of the specified method
if _, ok := ignoreLogMethods[info.FullMethod]; ok {
return handler(srv, stream)
}
startTime := time.Now()
requestID := ServerCtxRequestID(stream.Context())
err := handler(srv, stream)
fields := []zap.Field{
zap.String("code", status.Code(err).String()),
zap.String("type", "stream"),
zap.String("method", info.FullMethod),
zap.String("cost", time.Since(startTime).String()),
}
if requestID != "" {
fields = append(fields, zap.String(ContextRequestIDKey, requestID))
}
logger.Info("[GRPC] response", fields...)
return err
}
}
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论