biz生成测试

app/observe/models/ot_biz_edge.gen.go

@@ -4,20 +4,26 @@
 
 package models
 
+import (
+	"time"
+)
+
 const TableNameBizEdge = "ot_biz_edge"
 
 // BizEdge mapped from table <ot_biz_edge>
 type BizEdge struct {
-	ID         int64  `gorm:"column:id;type:bigint(20);primaryKey;autoIncrement:true" json:"id"`
-	Source     int64  `gorm:"column:source;type:bigint(20);not null;uniqueIndex:ot_biz_edge_pk,priority:2" json:"source"`
-	Target     int64  `gorm:"column:target;type:bigint(20);not null;uniqueIndex:ot_biz_edge_pk,priority:3" json:"target"`
-	SourceHash string `gorm:"column:source_hash;type:varchar(100);not null;comment:源节点hash" json:"source_hash"`  // 源节点hash
-	TargetHash string `gorm:"column:target_hash;type:varchar(100);not null;comment:目标节点hash" json:"target_hash"` // 目标节点hash
-	Type       string `gorm:"column:type;type:varchar(30);not null;default:0" json:"type"`
-	BizId      int64  `gorm:"column:biz_id;type:bigint(20);not null" json:"biz_id"`
-	BizHash    string `gorm:"column:biz_hash;type:varchar(100);not null;uniqueIndex:ot_biz_edge_pk,priority:1;comment:业务唯一哈希值" json:"biz_hash"` // 业务唯一哈希值
-	AppId      int64  `gorm:"column:app_id;type:bigint(20);not null" json:"app_id"`
-	AppAlias   string `gorm:"column:app_alias;type:varchar(30);not null;comment:应用别名" json:"app_alias"` // 应用别名
+	ID         int64     `gorm:"column:id;type:bigint(20);primaryKey;autoIncrement:true" json:"id"`
+	Source     int64     `gorm:"column:source;type:bigint(20);not null;uniqueIndex:ot_biz_edge_pk,priority:2" json:"source"`
+	Target     int64     `gorm:"column:target;type:bigint(20);not null;uniqueIndex:ot_biz_edge_pk,priority:3" json:"target"`
+	SourceHash string    `gorm:"column:source_hash;type:varchar(100);not null;comment:源节点hash" json:"source_hash"`  // 源节点hash
+	TargetHash string    `gorm:"column:target_hash;type:varchar(100);not null;comment:目标节点hash" json:"target_hash"` // 目标节点hash
+	Type       string    `gorm:"column:type;type:varchar(30);not null;default:0" json:"type"`
+	BizId      int64     `gorm:"column:biz_id;type:bigint(20);not null" json:"biz_id"`
+	BizHash    string    `gorm:"column:biz_hash;type:varchar(100);not null;uniqueIndex:ot_biz_edge_pk,priority:1;comment:业务唯一哈希值" json:"biz_hash"` // 业务唯一哈希值
+	AppId      int64     `gorm:"column:app_id;type:bigint(20);not null" json:"app_id"`
+	AppAlias   string    `gorm:"column:app_alias;type:varchar(30);not null;comment:应用别名" json:"app_alias"`                          // 应用别名
+	CreatedAt  time.Time `gorm:"column:created_at;type:datetime;not null;default:CURRENT_TIMESTAMP;comment:创建时间" json:"created_at"` // 创建时间
+	UpdatedAt  time.Time `gorm:"column:updated_at;type:datetime;not null;comment:更新时间" json:"updated_at"`                           // 更新时间
 }
 
 // TableName BizEdge's table name

cmd/consumer/biz/biz.go

@@ -162,13 +162,14 @@ func getSpanType(span tinySpan) string {
 }
 
 type BizNode struct {
-	Hash        string
-	AppAlias    string
-	ServiceName string
-	SpanName    string
-	SpanKind    string
-	IsRoot      bool
-	RootHash    string
+	Hash         string
+	AppAlias     string
+	ServiceName  string
+	SpanName     string
+	SpanKind     string
+	IsRoot       bool
+	RootHash     string
+	RootAppAlias string
 }
 
 type BizTree struct {
@@ -176,17 +177,23 @@ type BizTree struct {
 	Children []*BizTree
 }
 
-func genTree(root *BizTree, bizNodeMap map[string][]*BizNode) {
+func genTree(root *BizTree, bizNodeMap map[string][]*BizNode, visited map[string]struct{}) {
 	if len(bizNodeMap[root.Hash]) == 0 {
 		return
 	}
+	if _, ok := visited[root.Hash]; ok {
+		log.Infof("出现环路, 跳过... %s", root.RootHash)
+		return
+	}
+	visited[root.Hash] = struct{}{}
 	for _, bizNode := range bizNodeMap[root.Hash] {
-		bizNode.RootHash = root.Hash
+		bizNode.RootHash = root.RootHash
+		bizNode.RootAppAlias = root.RootAppAlias
 		child := &BizTree{
 			BizNode:  *bizNode,
 			Children: []*BizTree{},
 		}
-		genTree(child, bizNodeMap)
+		genTree(child, bizNodeMap, visited)
 		root.Children = append(root.Children, child)
 	}
 }
@@ -218,7 +225,8 @@ func getNodeAndEdge(root *BizTree) ([]*models.BizNode, []*models.BizEdge) {
 				TargetHash: child.Hash,
 				// Type:     getSpanType(item),
 				// BizId:   0,
-				BizHash: root.RootHash,
+				BizHash:  root.RootHash,
+				AppAlias: root.RootAppAlias,
 			}
 			dfs(child)
 		}
@@ -247,9 +255,10 @@ func GenBiz2() {
 						SpanName,
 						SpanKind
 					FROM otel_traces
-					WHERE Timestamp > (now() - toIntervalMinute(5))
+					WHERE Timestamp >= ? and Timestamp < ?
 				)
 				SELECT
+					distinct
 					cityHash64(concat(SourceAppAlias, '|', SourceServiceName, '|', SourceSpanName, '|', SourceSpanKind)) AS SourceHash,
 					cityHash64(concat(TargetAppAlias, '|', TargetServiceName, '|', TargetSpanName, '|', TargetSpanKind)) AS TargetHash,
 					SourceAppAlias, SourceServiceName, SourceSpanName, SourceSpanName, SourceSpanKind, TargetAppAlias, TargetServiceName, TargetSpanName, TargetSpanKind,
@@ -302,45 +311,49 @@ func GenBiz2() {
 			TargetSpanKind    string
 			IsRoot            bool
 		}{}
-		err := chdb.Raw(sql).Scan(&list).Error
+		start := time.Now().Add(-20 * time.Minute).Unix()
+		end := start + 600
+		err := chdb.Raw(sql, start, end).Scan(&list).Error
 		if err != nil {
 			panic(err)
 		}
-		bizList := []*models.Biz{}
-		// bizNodeList := []*models.BizNode{}
-		// bizEdgeList := []*models.BizEdge{}
 		bizNodeMap := map[string][]*BizNode{}
-		roots := []*BizTree{}
+		rootMap := map[string]*BizTree{}
 		for _, item := range list {
 			if item.IsRoot {
-				roots = append(roots, &BizTree{
+				rootMap[item.SourceHash] = &BizTree{
 					BizNode: BizNode{
-						AppAlias:    item.SourceAppAlias,
-						ServiceName: item.SourceServiceName,
-						SpanName:    item.SourceSpanName,
-						SpanKind:    item.SourceSpanKind,
-						Hash:        item.SourceHash,
-						IsRoot:      true,
-						RootHash:    item.SourceHash,
+						AppAlias:     item.SourceAppAlias,
+						ServiceName:  item.SourceServiceName,
+						SpanName:     item.SourceSpanName,
+						SpanKind:     item.SourceSpanKind,
+						Hash:         item.SourceHash,
+						IsRoot:       true,
+						RootHash:     item.SourceHash,
+						RootAppAlias: item.SourceAppAlias,
 					},
-				})
+				}
 			}
 			if _, ok := bizNodeMap[item.SourceHash]; !ok {
 				bizNodeMap[item.SourceHash] = []*BizNode{}
 			}
 			bizNodeMap[item.SourceHash] = append(bizNodeMap[item.SourceHash], &BizNode{
+				Hash:        item.TargetHash,
 				AppAlias:    item.TargetAppAlias,
 				ServiceName: item.TargetServiceName,
 				SpanName:    item.TargetSpanName,
 				SpanKind:    item.TargetSpanKind,
 			})
 		}
-		bizNodeList := []*models.BizNode{}
-		bizEdgeList := []*models.BizEdge{}
-		for _, root := range roots {
-			genTree(root, bizNodeMap)
+		bizMap := map[string]*models.Biz{}
+		nodeMap := map[string]*models.BizNode{}
+		// bizNodeList := []*models.BizNode{}
+		// bizEdgeList := []*models.BizEdge{}
+		edgeMap := map[string]*models.BizEdge{}
+		for _, root := range rootMap {
+			genTree(root, bizNodeMap, map[string]struct{}{})
 
-			bizList = append(bizList, &models.Biz{
+			bizMap[root.Hash] = &models.Biz{
 				Name:          root.SpanName,
 				Hash:          root.Hash,
 				AppAlias:      root.AppAlias,
@@ -349,37 +362,57 @@ func GenBiz2() {
 				SpanKind:      root.SpanKind,
 				IsAutoCreated: 1,
 				Favor:         0,
-			})
+			}
 			subNodes, subEdges := getNodeAndEdge(root)
-			bizNodeList = append(bizNodeList, subNodes...)
-			bizEdgeList = append(bizEdgeList, subEdges...)
+			for _, node := range subNodes {
+				nodeMap[node.Hash] = node
+			}
+			for _, edge := range subEdges {
+				edgeMap[fmt.Sprintf("%s-%s", edge.SourceHash, edge.TargetHash)] = edge
+			}
 		}
-		fmt.Println(len(bizEdgeList), len(bizEdgeList))
-		if len(bizList) > 0 {
+		// fmt.Println(len(bizMap), len(nodeMap), len(edgeMap))
+		if len(bizMap) > 0 {
+			bizList := []*models.Biz{}
+			for _, biz := range bizMap {
+				bizList = append(bizList, biz)
+			}
 			err := mydb.Model(&models.Biz{}).Clauses(clause.OnConflict{
-				Columns:   []clause.Column{{Name: "Hash"}},
+				Columns:   []clause.Column{{Name: "hash"}},
 				DoUpdates: clause.AssignmentColumns([]string{"updated_at"}),
-			}).CreateInBatches(bizList, 100)
+			}).CreateInBatches(bizList, 100).Error
 			if err != nil {
 				panic(err)
 			}
 		}
-		if len(bizNodeList) > 0 {
+		if len(nodeMap) > 0 {
+			bizNodeList := make([]*models.BizNode, 0, len(nodeMap))
+			for _, node := range nodeMap {
+				bizNodeList = append(bizNodeList, node)
+			}
 			err := mydb.Model(&models.BizNode{}).Clauses(clause.OnConflict{
-				DoNothing: true,
-			}).CreateInBatches(bizNodeList, 100)
+				Columns:   []clause.Column{{Name: "hash"}},
+				DoUpdates: clause.AssignmentColumns([]string{"updated_at"}),
+			}).CreateInBatches(bizNodeList, 100).Error
 			if err != nil {
 				panic(err)
 			}
 		}
-		if len(bizEdgeList) > 0 {
+		if len(edgeMap) > 0 {
+			bizEdgeList := make([]*models.BizEdge, 0, len(edgeMap))
+			for _, edge := range edgeMap {
+				bizEdgeList = append(bizEdgeList, edge)
+			}
 			err := mydb.Model(&models.BizEdge{}).Clauses(clause.OnConflict{
-				DoNothing: true,
-			}).CreateInBatches(bizEdgeList, 100)
+				Columns:   []clause.Column{{Name: "source_hash"}, {Name: "target_hash"}},
+				DoUpdates: clause.AssignmentColumns([]string{"updated_at"}),
+			}).CreateInBatches(bizEdgeList, 100).Error
 			if err != nil {
 				panic(err)
 			}
 		}
+		log.Infof("生成业务数据完成:%s-%s", time.Unix(start, 0), time.Unix(end, 0))
+		time.Sleep(time.Second * 10)
 	}
 }
 

common/database/initialize.go

@@ -1,18 +1,18 @@
 package database
 
 import (
-	"time"
-
 	log "github.com/go-admin-team/go-admin-core/logger"
 	"github.com/go-admin-team/go-admin-core/sdk"
 	toolsConfig "github.com/go-admin-team/go-admin-core/sdk/config"
 	"github.com/go-admin-team/go-admin-core/sdk/pkg"
 	mycasbin "github.com/go-admin-team/go-admin-core/sdk/pkg/casbin"
 	toolsDB "github.com/go-admin-team/go-admin-core/tools/database"
-	. "github.com/go-admin-team/go-admin-core/tools/gorm/logger"
+
+	// . "github.com/go-admin-team/go-admin-core/tools/gorm/logger"
 	"github.com/uptrace/opentelemetry-go-extra/otelgorm"
 	"gorm.io/gorm"
-	"gorm.io/gorm/logger"
+
+	// "gorm.io/gorm/logger"
 	"gorm.io/gorm/schema"
 
 	"go-admin/common/global"
@@ -49,14 +49,14 @@ func setupSimpleDatabase(host string, c *toolsConfig.Database, otel bool) {
 		NamingStrategy: schema.NamingStrategy{
 			SingularTable: true,
 		},
-		Logger: New(
-			logger.Config{
-				SlowThreshold: time.Second,
-				Colorful:      true,
-				LogLevel: logger.LogLevel(
-					log.DefaultLogger.Options().Level.LevelForGorm()),
-			},
-		),
+		// Logger: New(
+		// 	logger.Config{
+		// 		SlowThreshold: time.Second,
+		// 		Colorful:      true,
+		// 		LogLevel: logger.LogLevel(
+		// 			log.DefaultLogger.Options().Level.LevelForGorm()),
+		// 	},
+		// ),
 	}, opens[c.Driver])
 
 	if err != nil {