proxy, metrics: add metrics on QPS distribution and connection lifetime (#1040)

Author: djshow832

pkg/metrics/metrics.go

@@ -107,6 +107,8 @@ func init() {
 		KeepAliveCounter,
 		QueryTotalCounter,
 		QueryDurationHistogram,
+		QueryTimeSinceConnCreationHistogram,
+		ConnLifetimeHistogram,
 		HandshakeDurationHistogram,
 		BackendStatusGauge,
 		GetBackendHistogram,

pkg/metrics/session.go

@@ -39,4 +39,22 @@ var (
 			Help:      "Bucketed histogram of processing time (s) of handshakes.",
 			Buckets:   prometheus.ExponentialBuckets(0.0005, 2, 29), // 0.5ms ~ 1.5days
 		}, []string{LblBackend})
+
+	QueryTimeSinceConnCreationHistogram = prometheus.NewHistogram(
+		prometheus.HistogramOpts{
+			Namespace: ModuleProxy,
+			Subsystem: LabelSession,
+			Name:      "query_time_since_conn_creation_seconds",
+			Help:      "Bucketed histogram of query start time (s) since connection creation.",
+			Buckets:   prometheus.ExponentialBuckets(1, 2, 21), // 1s ~ 24days
+		})
+
+	ConnLifetimeHistogram = prometheus.NewHistogram(
+		prometheus.HistogramOpts{
+			Namespace: ModuleProxy,
+			Subsystem: LabelSession,
+			Name:      "conn_lifetime_seconds",
+			Help:      "Bucketed histogram of connection lifetime (s).",
+			Buckets:   prometheus.ExponentialBuckets(0.1, 2, 25), // 1s ~ 38days
+		})
 )

pkg/proxy/backend/backend_conn_mgr.go

@@ -376,6 +376,7 @@ func (mgr *BackendConnManager) ExecuteCmd(ctx context.Context, request []byte) (
 		}
 		mgr.lastActiveTime = now
 		mgr.processLock.Unlock()
+		metrics.QueryTimeSinceConnCreationHistogram.Observe(startTime.Sub(mgr.createTime).Seconds())
 	}()
 	if len(request) < 1 {
 		err = mysql.ErrMalformPacket
@@ -835,6 +836,7 @@ func (mgr *BackendConnManager) Close() error {
 		mgr.cpt.Capture([]byte{pnet.ComQuit.Byte()}, time.Now(), mgr.connectionID, nil)
 	}
 	mgr.closeStatus.Store(statusClosed)
+	metrics.ConnLifetimeHistogram.Observe(time.Since(mgr.createTime).Seconds())
 	return errors.Collect(ErrCloseConnMgr, connErr, handErr)
 }