~sircmpwn/chartsrv-devel

Refactor in preparation for histograms v2 PROPOSED

Noam Preil: 3
 Refactor in preparation for histograms
 Preserves tag list
 Add histogram support

 3 files changed, 256 insertions(+), 145 deletions(-)
Export patchset (mbox)
How do I use this?

Copy & paste the following snippet into your terminal to import this patchset into git:

curl -s https://lists.sr.ht/~sircmpwn/chartsrv-devel/patches/19730/mbox | git am -3
Learn more about email & git
View this thread in the archives

[PATCH v2 1/3] Refactor in preparation for histograms Export this patch

---
 main.go | 247 ++++++++++++++++++++++++++++++++------------------------
 1 file changed, 143 insertions(+), 104 deletions(-)

diff --git a/main.go b/main.go
index ab164e9..0f4b674 100644
--- a/main.go
+++ b/main.go
@@ -3,9 +3,11 @@ package main
import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"image/color"
	"log"
	"math"
	"net/http"
	"net/url"
	"os"
@@ -49,6 +51,18 @@ type PromResult struct {
	Values []Datapoint
}

type Args struct {
	start, end    time.Time
	width, height vg.Length
	data          []PromResult
	step          int
	title         string
	label         string
	query         string
	stacked       bool
	min, max      float64
}

func Query(q string, start time.Time, end time.Time, step int) ([]PromResult, error) {
	body := url.Values{}
	body.Set("query", q)
@@ -152,137 +166,162 @@ func handleLabel(p *plot.Plot, l *plotter.Line, label string, metric string) {
	}
}

func registerExtension(router chi.Router, extension string, mime string) {
	router.Get("/chart."+extension, func(w http.ResponseWriter, r *http.Request) {
		args := r.URL.Query()
		var query string
		if q, ok := args["query"]; !ok {
			w.WriteHeader(400)
			w.Write([]byte("Expected ?query=... parameter"))
			return
		} else {
			query = q[0]
		}
func extractArgs(in map[string][]string) (*Args, error) {
	args := Args{}
	if q, ok := in["query"]; !ok {
		return nil, errors.New("Expected ?query=... parameter")
	} else {
		args.query = q[0]
	}
	args.start = time.Now().Add(-24 * 60 * time.Minute)
	args.end = time.Now()
	if s, ok := in["since"]; ok {
		d, _ := time.ParseDuration(s[0])
		args.start = time.Now().Add(-d)
	}
	if u, ok := in["until"]; ok {
		d, _ := time.ParseDuration(u[0])
		args.end = time.Now().Add(-d)
	}

		start := time.Now().Add(-24 * 60 * time.Minute)
		end := time.Now()
		if s, ok := args["since"]; ok {
			d, _ := time.ParseDuration(s[0])
			start = time.Now().Add(-d)
		}
		if u, ok := args["until"]; ok {
			d, _ := time.ParseDuration(u[0])
			end = time.Now().Add(-d)
	args.width = 12 * vg.Inch
	args.height = 6 * vg.Inch
	if ws, ok := in["width"]; ok {
		w, _ := strconv.ParseFloat(ws[0], 32)
		args.width = vg.Length(w) * vg.Inch
	}
	if hs, ok := in["height"]; ok {
		h, _ := strconv.ParseFloat(hs[0], 32)
		args.height = vg.Length(h) * vg.Inch
	}
	// Label template
	if l, ok := in["label"]; ok {
		args.label = l[0]
	}

	// Set step so that there's approximately 25 data points per inch
	args.step = int(args.end.Sub(args.start).Seconds() / (25 * float64(args.width/vg.Inch)))
	if s, ok := in["step"]; ok {
		d, _ := strconv.ParseInt(s[0], 10, 32)
		args.step = int(d)
	}
	_, args.stacked = in["stacked"]
	args.max = math.Inf(+1)
	args.min = math.Inf(-1)
	if ms, ok := in["max"]; ok {
		m, _ := strconv.ParseFloat(ms[0], 64)
		args.max = m
	}

	if ms, ok := in["min"]; ok {
		m, _ := strconv.ParseFloat(ms[0], 64)
		args.min = m
	}
	args.title = ""
	if t, ok := in["title"]; ok {
		args.title = t[0]
	}
	return &args, nil
}

func plotChart(args Args, data []PromResult) (*plot.Plot, error) {

	p, err := plot.New()
	if err != nil {
		return nil, err
	}

	sums := make([]float64, len(data[0].Values))

	plotters := make([]plot.Plotter, len(data))
	var nextColor int
	colors := plotutil.SoftColors
	for i, res := range data {
		var points plotter.XYs
		for j, d := range res.Values {
			value := d.Value
			if args.stacked {
				value += sums[j]
			}
			points = append(points, plotter.XY{
				float64(d.Time.Unix()),
				value,
			})
			sums[j] += d.Value
		}

		width := 12 * vg.Inch
		height := 6 * vg.Inch
		if ws, ok := args["width"]; ok {
			w, _ := strconv.ParseFloat(ws[0], 32)
			width = vg.Length(w) * vg.Inch
		l, _, err := plotter.NewLinePoints(points)
		if err != nil {
			return nil, err
		}
		if hs, ok := args["height"]; ok {
			h, _ := strconv.ParseFloat(hs[0], 32)
			height = vg.Length(h) * vg.Inch
		if args.stacked {
			l.FillColor = colors[nextColor]
			if i != len(data)-1 {
				l.Color = color.RGBA{0, 0, 0, 0}
			}
		} else {
			l.Color = colors[nextColor]
		}

		// Label template
		var label string
		if l, ok := args["label"]; ok {
			label = l[0]
		nextColor += 1
		if nextColor >= len(colors) {
			nextColor = 0
		}

		// Set step so that there's approximately 25 data points per inch
		step := int(end.Sub(start).Seconds() / (25 * float64(width/vg.Inch)))
		if s, ok := args["step"]; ok {
			d, _ := strconv.ParseInt(s[0], 10, 32)
			step = int(d)
		plotters[i] = l
		if args.label != "" && len(res.Metric) > 2 && res.Metric[0] == '{' && res.Metric[len(res.Metric)-1] == '}' {
			handleLabel(p, l, args.label, res.Metric)
		} else {
			p.Legend.Add(res.Metric, l)
		}
		_, stacked := args["stacked"]
	}
	for i := len(plotters) - 1; i >= 0; i-- {
		p.Add(plotters[i])
	}
	return p, nil
}

		data, err := Query(query, start, end, step)
func registerExtension(router chi.Router, kind string, extension string, mime string) {
	if kind != "chart" {
		panic("Unknown kind: " + kind)
	}
	router.Get("/"+kind+"."+extension, func(w http.ResponseWriter, r *http.Request) {
		args, err := extractArgs(r.URL.Query())
		if err != nil {
			w.WriteHeader(400)
			w.Write([]byte(fmt.Sprintf("%v", err)))
			return
		}

		p, err := plot.New()
		data, err := Query(args.query, args.start, args.end, args.step)
		if err != nil {
			panic(err)
		}
		if t, ok := args["title"]; ok {
			p.Title.Text = t[0]
		}
		p.X.Label.Text = "Time"
		p.X.Tick.Marker = dateTicks{start, end}
		if ms, ok := args["max"]; ok {
			m, _ := strconv.ParseFloat(ms[0], 64)
			p.Y.Max = m
		}

		p.Y.Tick.Marker = humanTicks{}
		if ms, ok := args["min"]; ok {
			m, _ := strconv.ParseFloat(ms[0], 64)
			p.Y.Min = m
			w.WriteHeader(400)
			w.Write([]byte(fmt.Sprintf("%v", err)))
			return
		}
		p.Legend.Top = true

		sums := make([]float64, len(data[0].Values))

		plotters := make([]plot.Plotter, len(data))
		var nextColor int
		colors := plotutil.SoftColors
		for i, res := range data {
			var points plotter.XYs
			for j, d := range res.Values {
				value := d.Value
				if stacked {
					value += sums[j]
				}
				points = append(points, plotter.XY{
					float64(d.Time.Unix()),
					value,
				})
				sums[j] += d.Value
			}
		var p *plot.Plot

			l, _, err := plotter.NewLinePoints(points)
		if kind == "chart" {
			p, err = plotChart(*args, data)
			if err != nil {
				w.WriteHeader(400)
				w.Write([]byte(fmt.Sprintf("%v", err)))
				return
			}
			if stacked {
				l.FillColor = colors[nextColor]
				if i != len(data)-1 {
					l.Color = color.RGBA{0, 0, 0, 0}
				}
			} else {
				l.Color = colors[nextColor]
			}
			nextColor += 1
			if nextColor >= len(colors) {
				nextColor = 0
			}
			plotters[i] = l
			if label != "" && len(res.Metric) > 2 && res.Metric[0] == '{' && res.Metric[len(res.Metric)-1] == '}' {
				handleLabel(p, l, label, res.Metric)
			} else {
				p.Legend.Add(res.Metric, l)
			}
		}
		for i := len(plotters) - 1; i >= 0; i-- {
			p.Add(plotters[i])
		}
		p.X.Label.Text = "Time"
		p.X.Tick.Marker = dateTicks{args.start, args.end}
		p.Y.Tick.Marker = humanTicks{}
		p.Title.Text = args.title
		p.Legend.Top = true

		p.Y.Max = args.max
		p.Y.Min = args.min

		writer, err := p.WriterTo(width, height, extension)
		writer, err := p.WriterTo(args.width, args.height, extension)
		if err != nil {
			w.WriteHeader(400)
			w.Write([]byte(fmt.Sprintf("%v", err)))
			return
		}

		w.Header().Add("Content-Type", mime)
		writer.WriteTo(w)
	})
@@ -302,8 +341,8 @@ func main() {
	router.Use(middleware.RealIP)
	router.Use(middleware.Logger)

	registerExtension(router, "svg", "image/svg+xml")
	registerExtension(router, "png", "image/png")
	registerExtension(router, "chart", "svg", "image/svg+xml")
	registerExtension(router, "chart", "png", "image/png")

	addr := ":8142"
	if len(os.Args) > 2 {
-- 
2.30.0

[PATCH v2 2/3] Preserves tag list Export this patch

---
 main.go | 23 +++++------------------
 1 file changed, 5 insertions(+), 18 deletions(-)

diff --git a/main.go b/main.go
index 0f4b674..de770d6 100644
--- a/main.go
+++ b/main.go
@@ -47,7 +47,7 @@ type Datapoint struct {
}

type PromResult struct {
	Metric string
	Metric map[string]string
	Values []Datapoint
}

@@ -97,7 +97,7 @@ func Query(q string, start time.Time, end time.Time, step int) ([]PromResult, er
	var results []PromResult
	for _, res := range data.Data.Result {
		r := PromResult{}
		r.Metric = metricName(res.Metric)
		r.Metric = res.Metric

		var values []Datapoint
		for _, vals := range res.Values {
@@ -142,20 +142,7 @@ func metricName(metric map[string]string) string {
	return out + "{" + strings.Join(inner, ",") + "}"
}

func handleLabel(p *plot.Plot, l *plotter.Line, label string, metric string) {
	raw := metric[1 : len(metric)-1]
	raw_tags := strings.Split(raw, ",")
	tags := make(map[string]string)
	for _, v := range raw_tags {
		tag := strings.Split(v, "=")
		if len(tag) != 2 {
			log.Printf("Expected tag format: 'name=value'!")
			continue
		}
		if len(tag[1]) > 2 && tag[1][0] == '"' && tag[1][len(tag[1])-1] == '"' {
			tags[tag[0]] = tag[1][1 : len(tag[1])-1]
		}
	}
func handleLabel(p *plot.Plot, l *plotter.Line, label string, tags map[string]string) {
	tmpl, err := template.New("label").Parse(label)
	if err != nil {
		log.Printf("Failed to parse label template: ", err)
@@ -267,10 +254,10 @@ func plotChart(args Args, data []PromResult) (*plot.Plot, error) {
			nextColor = 0
		}
		plotters[i] = l
		if args.label != "" && len(res.Metric) > 2 && res.Metric[0] == '{' && res.Metric[len(res.Metric)-1] == '}' {
		if args.label != "" && len(res.Metric) > 0 {
			handleLabel(p, l, args.label, res.Metric)
		} else {
			p.Legend.Add(res.Metric, l)
			p.Legend.Add(metricName(res.Metric), l)
		}
	}
	for i := len(plotters) - 1; i >= 0; i-- {
-- 
2.30.0

[PATCH v2 3/3] Add histogram support Export this patch

---
 main.go | 131 ++++++++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 108 insertions(+), 23 deletions(-)

diff --git a/main.go b/main.go
index de770d6..c8af46b 100644
--- a/main.go
+++ b/main.go
@@ -142,7 +142,7 @@ func metricName(metric map[string]string) string {
	return out + "{" + strings.Join(inner, ",") + "}"
}

func handleLabel(p *plot.Plot, l *plotter.Line, label string, tags map[string]string) {
func handleLabel(p *plot.Plot, l plot.Thumbnailer, label string, tags map[string]string) {
	tmpl, err := template.New("label").Parse(label)
	if err != nil {
		log.Printf("Failed to parse label template: ", err)
@@ -211,12 +211,85 @@ func extractArgs(in map[string][]string) (*Args, error) {
	return &args, nil
}

func plotChart(args Args, data []PromResult) (*plot.Plot, error) {
func sortBuckets(data []PromResult) ([]PromResult, error) {
	numerics := []float64{}
	for _, res := range data {
		if le, ok := res.Metric["le"]; ok {
			d, _ := strconv.ParseFloat(le, 10)
			if le == "+Inf" {
				d = math.Inf(+1)
			}
			numerics = append(numerics, d)
		} else {
			return nil, errors.New("`le` tag missing")
		}
	}
	for i, n := range numerics {
		for j, n2 := range numerics {
			if i != j && n == n2 {
				return nil, errors.New("Multiple results in same bucket. Note that all tags other than bucket must match for now.")
			}
		}
	}
	snumerics := make([]float64, len(numerics))
	copy(snumerics, numerics)
	sort.Float64s(snumerics)
	sdata := make([]PromResult, len(data))
	for i, n := range snumerics {
		for j, n2 := range numerics {
			if n == n2 {
				sdata[i] = data[j]
			}
		}
	}
	return sdata, nil
}

	p, err := plot.New()
	if err != nil {
		return nil, err
func plotHistogram(args Args, data []PromResult, p *plot.Plot) error {
	// We only care about the buckets as they are now, not as they were in the past
	for i := len(data) - 1; i > 0; i -= 1 {
		l := len(data[i].Values)
		data[i].Values = data[i].Values[l-1:]
	}
	plotter := plotter.Histogram{
		Width:     1,
		FillColor: plotutil.SoftColors[0],
		LineStyle: plotter.DefaultLineStyle,
		Bins:      make([]plotter.HistogramBin, len(data)),
	}
	min := float64(0)
	for i := 0; i < len(data); i += 1 {
		// Prometheus buckets include all values of lower buckets, so remove those
		if i > 0 {
			data[i].Values[0].Value -= data[i-1].Values[0].Value
		}
		plotter.Bins[i].Min = min
		plotter.Bins[i].Max, _ = strconv.ParseFloat(data[i].Metric["le"], 10)
		plotter.Bins[i].Weight = data[i].Values[0].Value
		min = plotter.Bins[i].Max
	}
	metric := data[len(data)-1].Metric
	if args.label != "" && len(metric) > 0 {
		handleLabel(p, &plotter, args.label, metric)
	} else {
		p.Legend.Add(metricName(metric), &plotter)
	}
	p.Add(&plotter)

	for i := len(data) - 1; i > 0; i -= 1 {
		if data[i].Metric["le"] != "+Inf" {
			p.X.Max, _ = strconv.ParseFloat(data[i].Metric["le"], 10)
			break
		}
	}
	p.X.Label.Text = "Bucket"
	p.Y.Label.Text = "Count"
	p.X.Min = 0

	return nil
}

func plotChart(args Args, data []PromResult, p *plot.Plot) error {

	sums := make([]float64, len(data[0].Values))

@@ -239,7 +312,7 @@ func plotChart(args Args, data []PromResult) (*plot.Plot, error) {

		l, _, err := plotter.NewLinePoints(points)
		if err != nil {
			return nil, err
			return err
		}
		if args.stacked {
			l.FillColor = colors[nextColor]
@@ -260,16 +333,14 @@ func plotChart(args Args, data []PromResult) (*plot.Plot, error) {
			p.Legend.Add(metricName(res.Metric), l)
		}
	}
	for i := len(plotters) - 1; i >= 0; i-- {
		p.Add(plotters[i])
	}
	return p, nil
	p.X.Label.Text = "Time"
	p.X.Tick.Marker = dateTicks{args.start, args.end}
	p.Y.Max = args.max
	p.Y.Min = args.min
	return nil
}

func registerExtension(router chi.Router, kind string, extension string, mime string) {
	if kind != "chart" {
		panic("Unknown kind: " + kind)
	}
	router.Get("/"+kind+"."+extension, func(w http.ResponseWriter, r *http.Request) {
		args, err := extractArgs(r.URL.Query())
		if err != nil {
@@ -284,24 +355,35 @@ func registerExtension(router chi.Router, kind string, extension string, mime st
			return
		}

		var p *plot.Plot
		p, err := plot.New()
		if err != nil {
			w.WriteHeader(400)
			w.Write([]byte(fmt.Sprintf("%v", err)))
			return
		}

		p.Y.Tick.Marker = humanTicks{}
		p.Title.Text = args.title
		p.Legend.Top = true

		if kind == "chart" {
			p, err = plotChart(*args, data)
			err = plotChart(*args, data, p)
			if err != nil {
				w.WriteHeader(400)
				w.Write([]byte(fmt.Sprintf("%v", err)))
				return
			}
		} else if kind == "histogram" {
			data, err = sortBuckets(data)
			if err == nil {
				err = plotHistogram(*args, data, p)
			}
			if err != nil {
				w.WriteHeader(400)
				w.Write([]byte(fmt.Sprintf("%v", err)))
				return
			}
		}
		p.X.Label.Text = "Time"
		p.X.Tick.Marker = dateTicks{args.start, args.end}
		p.Y.Tick.Marker = humanTicks{}
		p.Title.Text = args.title
		p.Legend.Top = true

		p.Y.Max = args.max
		p.Y.Min = args.min

		writer, err := p.WriterTo(args.width, args.height, extension)
		if err != nil {
@@ -330,6 +412,9 @@ func main() {

	registerExtension(router, "chart", "svg", "image/svg+xml")
	registerExtension(router, "chart", "png", "image/png")
	registerExtension(router, "histogram", "svg", "image/svg+xml")
//  TODO: fix goplot. This currently hits an infinite loop in goplot somewhere.
//	registerExtension(router, "histogram", "png", "image/png")

	addr := ":8142"
	if len(os.Args) > 2 {
-- 
2.30.0