app-metrics/burrow_exporter: Apply patch to provide correct metrics

Package-Manager: Portage-2.3.56, Repoman-2.3.12
Signed-off-by: Manuel Rüger <mrueg@gentoo.org>

3 files changed, 329 insertions, 0 deletions

diff --git a/app-metrics/burrow_exporter/files/burrow_exporter-0.0.6-fix-metrics.patch b/app-metrics/burrow_exporter/files/burrow_exporter-0.0.6-fix-metrics.patch
new file mode 100644
index 000000000000..0d5a6f40dc70
--- /dev/null
+++ b/app-metrics/burrow_exporter/files/burrow_exporter-0.0.6-fix-metrics.patch
@@ -0,0 +1,49 @@
+From a40362c95ca5534040d8c29a23b40168a9d70015 Mon Sep 17 00:00:00 2001
+From: Guillaume Herail <guillaume@herail.net>
+Date: Wed, 16 Jan 2019 18:03:51 +0100
+Subject: [PATCH] fix(KafkaConsumerPartitionLag): Use proper current lag
+
+Before this fix, the latest committed lag was used for the metric, it
+now uses the current lag on the partition.
+
+Fixes #26
+---
+ burrow_exporter/client.go   | 11 ++++++-----
+ burrow_exporter/exporter.go |  2 +-
+ 2 files changed, 7 insertions(+), 6 deletions(-)
+
+diff --git a/burrow_exporter/client.go b/burrow_exporter/client.go
+index daaf84f..f4dc656 100644
+--- a/burrow_exporter/client.go
++++ b/burrow_exporter/client.go
+@@ -70,11 +70,12 @@ type ConsumerGroupStatus struct {
+ }
+ 
+ type Partition struct {
+-	Topic     string `json:"topic"`
+-	Partition int32  `json:"partition"`
+-	Status    string `json:"status"`
+-	Start     Offset `json:"start"`
+-	End       Offset `json:"end"`
++	Topic      string `json:"topic"`
++	Partition  int32  `json:"partition"`
++	Status     string `json:"status"`
++	Start      Offset `json:"start"`
++	End        Offset `json:"end"`
++	CurrentLag int64  `json:"current_lag"`
+ }
+ 
+ type ConsumerGroupStatusResp struct {
+diff --git a/burrow_exporter/exporter.go b/burrow_exporter/exporter.go
+index 8b48b35..153ccbb 100644
+--- a/burrow_exporter/exporter.go
++++ b/burrow_exporter/exporter.go
+@@ -45,7 +45,7 @@ func (be *BurrowExporter) processGroup(cluster, group string) {
+ 				"group":     status.Status.Group,
+ 				"topic":     partition.Topic,
+ 				"partition": strconv.Itoa(int(partition.Partition)),
+-			}).Set(float64(partition.End.Lag))
++			}).Set(float64(partition.CurrentLag))
+ 		}
+ 
+ 		if !be.skipPartitionCurrentOffset {
diff --git a/app-metrics/burrow_exporter/files/burrow_exporter-0.0.6-skippable-metrics.patch b/app-metrics/burrow_exporter/files/burrow_exporter-0.0.6-skippable-metrics.patch
new file mode 100644
index 000000000000..1b9c84599eb0
--- /dev/null
+++ b/app-metrics/burrow_exporter/files/burrow_exporter-0.0.6-skippable-metrics.patch
@@ -0,0 +1,273 @@
+From 50db40e49750378e22e2b638754ff07ff29f5362 Mon Sep 17 00:00:00 2001
+From: NovaPS6 <NovaPS6@ostmw0195464.bskyb.com>
+Date: Wed, 8 Aug 2018 14:43:33 +0100
+Subject: [PATCH] make all metrics skippable
+
+Change import back to jirwin
+---
+ .gitignore                  |   1 +
+ burrow-exporter.go          |  67 +++++++++++++++++------
+ burrow_exporter/exporter.go | 102 +++++++++++++++++++++---------------
+ 3 files changed, 113 insertions(+), 57 deletions(-)
+
+diff --git a/.gitignore b/.gitignore
+index 502219c..a8e0d62 100644
+--- a/.gitignore
++++ b/.gitignore
+@@ -18,3 +18,4 @@
+ burrow-exporter
+ dist/
+ vendor/
++.idea/
+diff --git a/burrow-exporter.go b/burrow-exporter.go
+index 1e60d67..b4e871b 100644
+--- a/burrow-exporter.go
++++ b/burrow-exporter.go
+@@ -14,7 +14,7 @@ import (
+ 	"github.com/jirwin/burrow_exporter/burrow_exporter"
+ )
+ 
+-var Version = "0.0.4"
++var Version = "0.0.5"
+ 
+ func main() {
+ 	app := cli.NewApp()
+@@ -22,36 +22,61 @@ func main() {
+ 	app.Name = "burrow-exporter"
+ 	app.Flags = []cli.Flag{
+ 		cli.StringFlag{
+-			Name:  "burrow-addr",
+-			Usage: "Address that burrow is listening on",
++			Name:   "burrow-addr",
++			Usage:  "Address that burrow is listening on",
+ 			EnvVar: "BURROW_ADDR",
+ 		},
+ 		cli.StringFlag{
+-			Name:  "metrics-addr",
+-			Usage: "Address to run prometheus on",
++			Name:   "metrics-addr",
++			Usage:  "Address to run prometheus on",
+ 			EnvVar: "METRICS_ADDR",
+ 		},
+ 		cli.IntFlag{
+-			Name:  "interval",
+-			Usage: "The interval(seconds) specifies how often to scrape burrow.",
++			Name:   "interval",
++			Usage:  "The interval(seconds) specifies how often to scrape burrow.",
+ 			EnvVar: "INTERVAL",
+ 		},
+ 		cli.IntFlag{
+-			Name:  "api-version",
+-			Usage: "Burrow API version to leverage",
+-			Value: 2,
++			Name:   "api-version",
++			Usage:  "Burrow API version to leverage",
++			Value:  2,
+ 			EnvVar: "API_VERSION",
+ 		},
+ 		cli.BoolFlag{
+-			Name: "skip-partition-status",
+-			Usage: "Skip exporting the per-partition status",
++			Name:   "skip-partition-status",
++			Usage:  "Skip exporting the per-partition status",
+ 			EnvVar: "SKIP_PARTITION_STATUS",
+ 		},
+ 		cli.BoolFlag{
+-			Name: "skip-group-status",
+-			Usage: "Skip exporting the per-group status",
++			Name:   "skip-group-status",
++			Usage:  "Skip exporting the per-group status",
+ 			EnvVar: "SKIP_GROUP_STATUS",
+ 		},
++		cli.BoolFlag{
++			Name:   "skip-partition-lag",
++			Usage:  "Skip exporting the partition lag",
++			EnvVar: "SKIP_PARTITION_LAG",
++		},
++		cli.BoolFlag{
++			Name:   "skip-partition-current-offset",
++			Usage:  "Skip exporting the current offset per partition",
++			EnvVar: "SKIP_PARTITION_CURRENT_OFFSET",
++		},
++		cli.BoolFlag{
++			Name:   "skip-partition-max-offset",
++			Usage:  "Skip exporting the partition max offset",
++			EnvVar: "SKIP_PARTITION_MAX_OFFSET",
++		},
++		cli.BoolFlag{
++			Name:   "skip-total-lag",
++			Usage:  "Skip exporting the total lag",
++			EnvVar: "SKIP_TOTAL_LAG",
++		},
++		cli.BoolFlag{
++			Name:   "skip-topic-partition-offset",
++			Usage:  "Skip exporting topic partition offset",
++			EnvVar: "SKIP_TOPIC_PARTITION_OFFSET",
++		},
+ 	}
+ 
+ 	app.Action = func(c *cli.Context) error {
+@@ -76,8 +101,18 @@ func main() {
+ 
+ 		ctx, cancel := context.WithCancel(context.Background())
+ 
+-		exporter := burrow_exporter.MakeBurrowExporter(c.String("burrow-addr"), c.Int("api-version"),
+-			c.String("metrics-addr"), c.Int("interval"), c.Bool("skip-partition-status"), c.Bool("skip-group-status"))
++		exporter := burrow_exporter.MakeBurrowExporter(
++			c.String("burrow-addr"),
++			c.Int("api-version"),
++			c.String("metrics-addr"),
++			c.Int("interval"),
++			c.Bool("skip-partition-status"),
++			c.Bool("skip-group-status"),
++			c.Bool("skip-partition-lag"),
++			c.Bool("skip-partition-current-offset"),
++			c.Bool("skip-partition-max-offset"),
++			c.Bool("skip-total-lag"),
++			c.Bool("skip-topic-partition-offset"))
+ 		go exporter.Start(ctx)
+ 
+ 		<-done
+diff --git a/burrow_exporter/exporter.go b/burrow_exporter/exporter.go
+index 7adc398..8b48b35 100644
+--- a/burrow_exporter/exporter.go
++++ b/burrow_exporter/exporter.go
+@@ -16,12 +16,17 @@ import (
+ )
+ 
+ type BurrowExporter struct {
+-	client              *BurrowClient
+-	metricsListenAddr   string
+-	interval            int
+-	wg                  sync.WaitGroup
+-	skipPartitionStatus bool
+-	skipConsumerStatus  bool
++	client                     *BurrowClient
++	metricsListenAddr          string
++	interval                   int
++	wg                         sync.WaitGroup
++	skipPartitionStatus        bool
++	skipConsumerStatus         bool
++	skipPartitionLag           bool
++	skipPartitionCurrentOffset bool
++	skipPartitionMaxOffset     bool
++	skipTotalLag               bool
++	skipTopicPartitionOffset   bool
+ }
+ 
+ func (be *BurrowExporter) processGroup(cluster, group string) {
+@@ -34,19 +39,23 @@ func (be *BurrowExporter) processGroup(cluster, group string) {
+ 	}
+ 
+ 	for _, partition := range status.Status.Partitions {
+-		KafkaConsumerPartitionLag.With(prometheus.Labels{
+-			"cluster":   status.Status.Cluster,
+-			"group":     status.Status.Group,
+-			"topic":     partition.Topic,
+-			"partition": strconv.Itoa(int(partition.Partition)),
+-		}).Set(float64(partition.End.Lag))
+-
+-		KafkaConsumerPartitionCurrentOffset.With(prometheus.Labels{
+-			"cluster":   status.Status.Cluster,
+-			"group":     status.Status.Group,
+-			"topic":     partition.Topic,
+-			"partition": strconv.Itoa(int(partition.Partition)),
+-		}).Set(float64(partition.End.Offset))
++		if !be.skipPartitionLag {
++			KafkaConsumerPartitionLag.With(prometheus.Labels{
++				"cluster":   status.Status.Cluster,
++				"group":     status.Status.Group,
++				"topic":     partition.Topic,
++				"partition": strconv.Itoa(int(partition.Partition)),
++			}).Set(float64(partition.End.Lag))
++		}
++
++		if !be.skipPartitionCurrentOffset {
++			KafkaConsumerPartitionCurrentOffset.With(prometheus.Labels{
++				"cluster":   status.Status.Cluster,
++				"group":     status.Status.Group,
++				"topic":     partition.Topic,
++				"partition": strconv.Itoa(int(partition.Partition)),
++			}).Set(float64(partition.End.Offset))
++		}
+ 
+ 		if !be.skipPartitionStatus {
+ 			KafkaConsumerPartitionCurrentStatus.With(prometheus.Labels{
+@@ -57,18 +66,22 @@ func (be *BurrowExporter) processGroup(cluster, group string) {
+ 			}).Set(float64(Status[partition.Status]))
+ 		}
+ 
+-		KafkaConsumerPartitionMaxOffset.With(prometheus.Labels{
+-			"cluster":   status.Status.Cluster,
+-			"group":     status.Status.Group,
+-			"topic":     partition.Topic,
+-			"partition": strconv.Itoa(int(partition.Partition)),
+-		}).Set(float64(partition.End.MaxOffset))
++		if !be.skipPartitionMaxOffset {
++			KafkaConsumerPartitionMaxOffset.With(prometheus.Labels{
++				"cluster":   status.Status.Cluster,
++				"group":     status.Status.Group,
++				"topic":     partition.Topic,
++				"partition": strconv.Itoa(int(partition.Partition)),
++			}).Set(float64(partition.End.MaxOffset))
++		}
+ 	}
+ 
+-	KafkaConsumerTotalLag.With(prometheus.Labels{
+-		"cluster": status.Status.Cluster,
+-		"group":   status.Status.Group,
+-	}).Set(float64(status.Status.TotalLag))
++	if !be.skipTotalLag {
++		KafkaConsumerTotalLag.With(prometheus.Labels{
++			"cluster": status.Status.Cluster,
++			"group":   status.Status.Group,
++		}).Set(float64(status.Status.TotalLag))
++	}
+ 
+ 	if !be.skipConsumerStatus {
+ 		KafkaConsumerStatus.With(prometheus.Labels{
+@@ -88,12 +101,14 @@ func (be *BurrowExporter) processTopic(cluster, topic string) {
+ 		return
+ 	}
+ 
+-	for i, offset := range details.Offsets {
+-		KafkaTopicPartitionOffset.With(prometheus.Labels{
+-			"cluster":   cluster,
+-			"topic":     topic,
+-			"partition": strconv.Itoa(i),
+-		}).Set(float64(offset))
++	if !be.skipTopicPartitionOffset {
++		for i, offset := range details.Offsets {
++			KafkaTopicPartitionOffset.With(prometheus.Labels{
++				"cluster":   cluster,
++				"topic":     topic,
++				"partition": strconv.Itoa(i),
++			}).Set(float64(offset))
++		}
+ 	}
+ }
+ 
+@@ -208,12 +223,17 @@ func (be *BurrowExporter) mainLoop(ctx context.Context) {
+ }
+ 
+ func MakeBurrowExporter(burrowUrl string, apiVersion int, metricsAddr string, interval int, skipPartitionStatus bool,
+-	skipConsumerStatus bool) *BurrowExporter {
++	skipConsumerStatus bool, skipPartitionLag bool, skipPartitionCurrentOffset bool, skipPartitionMaxOffset bool, skipTotalLag bool, skipTopicPartitionOffset bool) *BurrowExporter {
+ 	return &BurrowExporter{
+-		client:              MakeBurrowClient(burrowUrl, apiVersion),
+-		metricsListenAddr:   metricsAddr,
+-		interval:            interval,
+-		skipPartitionStatus: skipPartitionStatus,
+-		skipConsumerStatus:  skipConsumerStatus,
++		client:                     MakeBurrowClient(burrowUrl, apiVersion),
++		metricsListenAddr:          metricsAddr,
++		interval:                   interval,
++		skipPartitionStatus:        skipPartitionStatus,
++		skipConsumerStatus:         skipConsumerStatus,
++		skipPartitionLag:           skipPartitionLag,
++		skipPartitionCurrentOffset: skipPartitionCurrentOffset,
++		skipPartitionMaxOffset:     skipPartitionMaxOffset,
++		skipTotalLag:               skipTotalLag,
++		skipTopicPartitionOffset:   skipTopicPartitionOffset,
+ 	}
+ }
diff --git a/app-metrics/burrow_exporter/files/burrow_exporter.logrotated b/app-metrics/burrow_exporter/files/burrow_exporter.logrotated
new file mode 100644
index 000000000000..de78d102c0fc
--- /dev/null
+++ b/app-metrics/burrow_exporter/files/burrow_exporter.logrotated
@@ -0,0 +1,7 @@
+/var/log/burrow_exporter/burrow_exporter.log {
+	missingok
+	size 5M
+	rotate 3
+	compress
+	copytruncate
+}