feat: support multiple trace agents (#1183)

* feat: support multiple trace agents

* feat: support multiple trace agents, let later calls run if error happens

* test: add more tests

core/trace/agent.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"sync"
 
+	"github.com/tal-tech/go-zero/core/lang"
 	"github.com/tal-tech/go-zero/core/logx"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/exporters/jaeger"
@@ -19,17 +20,31 @@ const (
 	kindZipkin = "zipkin"
 )
 
-var once sync.Once
+var (
+	agents = make(map[string]lang.PlaceholderType)
+	lock   sync.Mutex
+)
 
 // StartAgent starts a opentelemetry agent.
 func StartAgent(c Config) {
-	once.Do(func() {
-		startAgent(c)
-	})
+	lock.Lock()
+	defer lock.Unlock()
+
+	_, ok := agents[c.Endpoint]
+	if ok {
+		return
+	}
+
+	// if error happens, let later calls run.
+	if err := startAgent(c); err != nil {
+		return
+	}
+
+	agents[c.Endpoint] = lang.Placeholder
 }
 
 func createExporter(c Config) (sdktrace.SpanExporter, error) {
-	// Just support jaeger now, more for later
+	// Just support jaeger and zipkin now, more for later
 	switch c.Batcher {
 	case kindJaeger:
 		return jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(c.Endpoint)))
@@ -40,7 +55,7 @@ func createExporter(c Config) (sdktrace.SpanExporter, error) {
 	}
 }
 
-func startAgent(c Config) {
+func startAgent(c Config) error {
 	opts := []sdktrace.TracerProviderOption{
 		// Set the sampling rate based on the parent span to 100%
 		sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(c.Sampler))),
@@ -52,7 +67,7 @@ func startAgent(c Config) {
 		exp, err := createExporter(c)
 		if err != nil {
 			logx.Error(err)
-			return
+			return err
 		}
 
 		// Always be sure to batch in production.
@@ -66,4 +81,6 @@ func startAgent(c Config) {
 	otel.SetErrorHandler(otel.ErrorHandlerFunc(func(err error) {
 		logx.Errorf("[otel] error: %v", err)
 	}))
+
+	return nil
 }