~sircmpwn/chartsrv-devel

This thread contains a patchset. You're looking at the original emails, but you may wish to use the patch review UI. Review patch
3

[PATCH 1/3] Refactor in preparation for histograms

Details
Message ID
<20210115060745.24831-1-noam@pixelhero.dev>
DKIM signature
pass
Download raw message
Patch: +143 -104
---
 main.go | 247 ++++++++++++++++++++++++++++++++------------------------
 1 file changed, 143 insertions(+), 104 deletions(-)

diff --git a/main.go b/main.go
index ab164e9..0f4b674 100644
--- a/main.go
+++ b/main.go
@@ -3,9 +3,11 @@ package main
import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"image/color"
	"log"
	"math"
	"net/http"
	"net/url"
	"os"
@@ -49,6 +51,18 @@ type PromResult struct {
	Values []Datapoint
}

type Args struct {
	start, end    time.Time
	width, height vg.Length
	data          []PromResult
	step          int
	title         string
	label         string
	query         string
	stacked       bool
	min, max      float64
}

func Query(q string, start time.Time, end time.Time, step int) ([]PromResult, error) {
	body := url.Values{}
	body.Set("query", q)
@@ -152,137 +166,162 @@ func handleLabel(p *plot.Plot, l *plotter.Line, label string, metric string) {
	}
}

func registerExtension(router chi.Router, extension string, mime string) {
	router.Get("/chart."+extension, func(w http.ResponseWriter, r *http.Request) {
		args := r.URL.Query()
		var query string
		if q, ok := args["query"]; !ok {
			w.WriteHeader(400)
			w.Write([]byte("Expected ?query=... parameter"))
			return
		} else {
			query = q[0]
		}
func extractArgs(in map[string][]string) (*Args, error) {
	args := Args{}
	if q, ok := in["query"]; !ok {
		return nil, errors.New("Expected ?query=... parameter")
	} else {
		args.query = q[0]
	}
	args.start = time.Now().Add(-24 * 60 * time.Minute)
	args.end = time.Now()
	if s, ok := in["since"]; ok {
		d, _ := time.ParseDuration(s[0])
		args.start = time.Now().Add(-d)
	}
	if u, ok := in["until"]; ok {
		d, _ := time.ParseDuration(u[0])
		args.end = time.Now().Add(-d)
	}

		start := time.Now().Add(-24 * 60 * time.Minute)
		end := time.Now()
		if s, ok := args["since"]; ok {
			d, _ := time.ParseDuration(s[0])
			start = time.Now().Add(-d)
		}
		if u, ok := args["until"]; ok {
			d, _ := time.ParseDuration(u[0])
			end = time.Now().Add(-d)
	args.width = 12 * vg.Inch
	args.height = 6 * vg.Inch
	if ws, ok := in["width"]; ok {
		w, _ := strconv.ParseFloat(ws[0], 32)
		args.width = vg.Length(w) * vg.Inch
	}
	if hs, ok := in["height"]; ok {
		h, _ := strconv.ParseFloat(hs[0], 32)
		args.height = vg.Length(h) * vg.Inch
	}
	// Label template
	if l, ok := in["label"]; ok {
		args.label = l[0]
	}

	// Set step so that there's approximately 25 data points per inch
	args.step = int(args.end.Sub(args.start).Seconds() / (25 * float64(args.width/vg.Inch)))
	if s, ok := in["step"]; ok {
		d, _ := strconv.ParseInt(s[0], 10, 32)
		args.step = int(d)
	}
	_, args.stacked = in["stacked"]
	args.max = math.Inf(+1)
	args.min = math.Inf(-1)
	if ms, ok := in["max"]; ok {
		m, _ := strconv.ParseFloat(ms[0], 64)
		args.max = m
	}

	if ms, ok := in["min"]; ok {
		m, _ := strconv.ParseFloat(ms[0], 64)
		args.min = m
	}
	args.title = ""
	if t, ok := in["title"]; ok {
		args.title = t[0]
	}
	return &args, nil
}

func plotChart(args Args, data []PromResult) (*plot.Plot, error) {

	p, err := plot.New()
	if err != nil {
		return nil, err
	}

	sums := make([]float64, len(data[0].Values))

	plotters := make([]plot.Plotter, len(data))
	var nextColor int
	colors := plotutil.SoftColors
	for i, res := range data {
		var points plotter.XYs
		for j, d := range res.Values {
			value := d.Value
			if args.stacked {
				value += sums[j]
			}
			points = append(points, plotter.XY{
				float64(d.Time.Unix()),
				value,
			})
			sums[j] += d.Value
		}

		width := 12 * vg.Inch
		height := 6 * vg.Inch
		if ws, ok := args["width"]; ok {
			w, _ := strconv.ParseFloat(ws[0], 32)
			width = vg.Length(w) * vg.Inch
		l, _, err := plotter.NewLinePoints(points)
		if err != nil {
			return nil, err
		}
		if hs, ok := args["height"]; ok {
			h, _ := strconv.ParseFloat(hs[0], 32)
			height = vg.Length(h) * vg.Inch
		if args.stacked {
			l.FillColor = colors[nextColor]
			if i != len(data)-1 {
				l.Color = color.RGBA{0, 0, 0, 0}
			}
		} else {
			l.Color = colors[nextColor]
		}

		// Label template
		var label string
		if l, ok := args["label"]; ok {
			label = l[0]
		nextColor += 1
		if nextColor >= len(colors) {
			nextColor = 0
		}

		// Set step so that there's approximately 25 data points per inch
		step := int(end.Sub(start).Seconds() / (25 * float64(width/vg.Inch)))
		if s, ok := args["step"]; ok {
			d, _ := strconv.ParseInt(s[0], 10, 32)
			step = int(d)
		plotters[i] = l
		if args.label != "" && len(res.Metric) > 2 && res.Metric[0] == '{' && res.Metric[len(res.Metric)-1] == '}' {
			handleLabel(p, l, args.label, res.Metric)
		} else {
			p.Legend.Add(res.Metric, l)
		}
		_, stacked := args["stacked"]
	}
	for i := len(plotters) - 1; i >= 0; i-- {
		p.Add(plotters[i])
	}
	return p, nil
}

		data, err := Query(query, start, end, step)
func registerExtension(router chi.Router, kind string, extension string, mime string) {
	if kind != "chart" {
		panic("Unknown kind: " + kind)
	}
	router.Get("/"+kind+"."+extension, func(w http.ResponseWriter, r *http.Request) {
		args, err := extractArgs(r.URL.Query())
		if err != nil {
			w.WriteHeader(400)
			w.Write([]byte(fmt.Sprintf("%v", err)))
			return
		}

		p, err := plot.New()
		data, err := Query(args.query, args.start, args.end, args.step)
		if err != nil {
			panic(err)
		}
		if t, ok := args["title"]; ok {
			p.Title.Text = t[0]
		}
		p.X.Label.Text = "Time"
		p.X.Tick.Marker = dateTicks{start, end}
		if ms, ok := args["max"]; ok {
			m, _ := strconv.ParseFloat(ms[0], 64)
			p.Y.Max = m
		}

		p.Y.Tick.Marker = humanTicks{}
		if ms, ok := args["min"]; ok {
			m, _ := strconv.ParseFloat(ms[0], 64)
			p.Y.Min = m
			w.WriteHeader(400)
			w.Write([]byte(fmt.Sprintf("%v", err)))
			return
		}
		p.Legend.Top = true

		sums := make([]float64, len(data[0].Values))

		plotters := make([]plot.Plotter, len(data))
		var nextColor int
		colors := plotutil.SoftColors
		for i, res := range data {
			var points plotter.XYs
			for j, d := range res.Values {
				value := d.Value
				if stacked {
					value += sums[j]
				}
				points = append(points, plotter.XY{
					float64(d.Time.Unix()),
					value,
				})
				sums[j] += d.Value
			}
		var p *plot.Plot

			l, _, err := plotter.NewLinePoints(points)
		if kind == "chart" {
			p, err = plotChart(*args, data)
			if err != nil {
				w.WriteHeader(400)
				w.Write([]byte(fmt.Sprintf("%v", err)))
				return
			}
			if stacked {
				l.FillColor = colors[nextColor]
				if i != len(data)-1 {
					l.Color = color.RGBA{0, 0, 0, 0}
				}
			} else {
				l.Color = colors[nextColor]
			}
			nextColor += 1
			if nextColor >= len(colors) {
				nextColor = 0
			}
			plotters[i] = l
			if label != "" && len(res.Metric) > 2 && res.Metric[0] == '{' && res.Metric[len(res.Metric)-1] == '}' {
				handleLabel(p, l, label, res.Metric)
			} else {
				p.Legend.Add(res.Metric, l)
			}
		}
		for i := len(plotters) - 1; i >= 0; i-- {
			p.Add(plotters[i])
		}
		p.X.Label.Text = "Time"
		p.X.Tick.Marker = dateTicks{args.start, args.end}
		p.Y.Tick.Marker = humanTicks{}
		p.Title.Text = args.title
		p.Legend.Top = true

		p.Y.Max = args.max
		p.Y.Min = args.min

		writer, err := p.WriterTo(width, height, extension)
		writer, err := p.WriterTo(args.width, args.height, extension)
		if err != nil {
			w.WriteHeader(400)
			w.Write([]byte(fmt.Sprintf("%v", err)))
			return
		}

		w.Header().Add("Content-Type", mime)
		writer.WriteTo(w)
	})
@@ -302,8 +341,8 @@ func main() {
	router.Use(middleware.RealIP)
	router.Use(middleware.Logger)

	registerExtension(router, "svg", "image/svg+xml")
	registerExtension(router, "png", "image/png")
	registerExtension(router, "chart", "svg", "image/svg+xml")
	registerExtension(router, "chart", "png", "image/png")

	addr := ":8142"
	if len(os.Args) > 2 {
-- 
2.30.0

[PATCH 2/3] Preserves tag list

Details
Message ID
<20210115060745.24831-2-noam@pixelhero.dev>
In-Reply-To
<20210115060745.24831-1-noam@pixelhero.dev> (view parent)
DKIM signature
pass
Download raw message
Patch: +5 -18
---
 main.go | 23 +++++------------------
 1 file changed, 5 insertions(+), 18 deletions(-)

diff --git a/main.go b/main.go
index 0f4b674..de770d6 100644
--- a/main.go
+++ b/main.go
@@ -47,7 +47,7 @@ type Datapoint struct {
}

type PromResult struct {
	Metric string
	Metric map[string]string
	Values []Datapoint
}

@@ -97,7 +97,7 @@ func Query(q string, start time.Time, end time.Time, step int) ([]PromResult, er
	var results []PromResult
	for _, res := range data.Data.Result {
		r := PromResult{}
		r.Metric = metricName(res.Metric)
		r.Metric = res.Metric

		var values []Datapoint
		for _, vals := range res.Values {
@@ -142,20 +142,7 @@ func metricName(metric map[string]string) string {
	return out + "{" + strings.Join(inner, ",") + "}"
}

func handleLabel(p *plot.Plot, l *plotter.Line, label string, metric string) {
	raw := metric[1 : len(metric)-1]
	raw_tags := strings.Split(raw, ",")
	tags := make(map[string]string)
	for _, v := range raw_tags {
		tag := strings.Split(v, "=")
		if len(tag) != 2 {
			log.Printf("Expected tag format: 'name=value'!")
			continue
		}
		if len(tag[1]) > 2 && tag[1][0] == '"' && tag[1][len(tag[1])-1] == '"' {
			tags[tag[0]] = tag[1][1 : len(tag[1])-1]
		}
	}
func handleLabel(p *plot.Plot, l *plotter.Line, label string, tags map[string]string) {
	tmpl, err := template.New("label").Parse(label)
	if err != nil {
		log.Printf("Failed to parse label template: ", err)
@@ -267,10 +254,10 @@ func plotChart(args Args, data []PromResult) (*plot.Plot, error) {
			nextColor = 0
		}
		plotters[i] = l
		if args.label != "" && len(res.Metric) > 2 && res.Metric[0] == '{' && res.Metric[len(res.Metric)-1] == '}' {
		if args.label != "" && len(res.Metric) > 0 {
			handleLabel(p, l, args.label, res.Metric)
		} else {
			p.Legend.Add(res.Metric, l)
			p.Legend.Add(metricName(res.Metric), l)
		}
	}
	for i := len(plotters) - 1; i >= 0; i-- {
-- 
2.30.0

[PATCH 3/3] Add histogram support

Details
Message ID
<20210115060745.24831-3-noam@pixelhero.dev>
In-Reply-To
<20210115060745.24831-1-noam@pixelhero.dev> (view parent)
DKIM signature
pass
Download raw message
Patch: +102 -7
---
 main.go | 109 ++++++++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 102 insertions(+), 7 deletions(-)

diff --git a/main.go b/main.go
index de770d6..2929767 100644
--- a/main.go
+++ b/main.go
@@ -142,7 +142,7 @@ func metricName(metric map[string]string) string {
	return out + "{" + strings.Join(inner, ",") + "}"
}

func handleLabel(p *plot.Plot, l *plotter.Line, label string, tags map[string]string) {
func handleLabel(p *plot.Plot, l plot.Thumbnailer, label string, tags map[string]string) {
	tmpl, err := template.New("label").Parse(label)
	if err != nil {
		log.Printf("Failed to parse label template: ", err)
@@ -211,6 +211,78 @@ func extractArgs(in map[string][]string) (*Args, error) {
	return &args, nil
}

func sortBuckets(data []PromResult) ([]PromResult, error) {
	numerics := []float64{}
	for _, res := range data {
		if le, ok := res.Metric["le"]; ok {
			d, _ := strconv.ParseFloat(le, 10)
			if le == "+Inf" {
				d = math.Inf(+1)
			}
			numerics = append(numerics, d)
		} else {
			return nil, errors.New("`le` tag missing")
		}
	}
	for i, n := range numerics {
		for j, n2 := range numerics {
			if i != j && n == n2 {
				return nil, errors.New("Multiple results in same bucket. Note that all tags other than bucket must match for now.")
			}
		}
	}
	snumerics := make([]float64, len(numerics))
	copy(snumerics, numerics)
	sort.Float64s(snumerics)
	sdata := make([]PromResult, len(data))
	for i, n := range snumerics {
		for j, n2 := range numerics {
			if n == n2 {
				sdata[i] = data[j]
			}
		}
	}
	return sdata, nil
}

func plotHistogram(args Args, data []PromResult) (*plot.Plot, error) {
	p, err := plot.New()
	if err != nil {
		return nil, err
	}
	// We only care about the buckets as they are now, not as they were in the past
	for i := len(data) - 1; i > 0; i -= 1 {
		l := len(data[i].Values)
		data[i].Values = data[i].Values[l-1:]
	}
	plotter := plotter.Histogram{
		Width:     1,
		FillColor: plotutil.SoftColors[0],
		LineStyle: plotter.DefaultLineStyle,
		Bins:      make([]plotter.HistogramBin, len(data)),
	}
	min := float64(0)
	for i := 0; i < len(data); i += 1 {
		// Prometheus buckets include all values of lower buckets, so remove those
		if i > 0 {
			data[i].Values[0].Value -= data[i-1].Values[0].Value
		}
		plotter.Bins[i].Min = min
		plotter.Bins[i].Max, _ = strconv.ParseFloat(data[i].Metric["le"], 10)
		plotter.Bins[i].Weight = data[i].Values[0].Value
		min = plotter.Bins[i].Max
	}
	metric := data[len(data)-1].Metric
	if args.label != "" && len(metric) > 0 {
		handleLabel(p, &plotter, args.label, metric)
	} else {
		p.Legend.Add(metricName(metric), &plotter)
	}
	p.Add(&plotter)

	return p, nil
}

func plotChart(args Args, data []PromResult) (*plot.Plot, error) {

	p, err := plot.New()
@@ -267,7 +339,7 @@ func plotChart(args Args, data []PromResult) (*plot.Plot, error) {
}

func registerExtension(router chi.Router, kind string, extension string, mime string) {
	if kind != "chart" {
	if kind != "chart" && kind != "histogram" {
		panic("Unknown kind: " + kind)
	}
	router.Get("/"+kind+"."+extension, func(w http.ResponseWriter, r *http.Request) {
@@ -293,16 +365,37 @@ func registerExtension(router chi.Router, kind string, extension string, mime st
				w.Write([]byte(fmt.Sprintf("%v", err)))
				return
			}
			p.X.Label.Text = "Time"
			p.X.Tick.Marker = dateTicks{args.start, args.end}
			p.Y.Max = args.max
			p.Y.Min = args.min
		} else if kind == "histogram" {
			data, err = sortBuckets(data)
			if err == nil {
				p, err = plotHistogram(*args, data)
			}
			if err != nil {
				w.WriteHeader(400)
				w.Write([]byte(fmt.Sprintf("%v", err)))
				return
			}
			p.X.Label.Text = "Bucket"
			p.Y.Label.Text = "Count"
			p.X.Min = 0
			for i := len(data) - 1; i > 0; i -= 1 {
				if data[i].Metric["le"] != "+Inf" {
					p.X.Max, _ = strconv.ParseFloat(data[i].Metric["le"], 10)
					break
				}
			}
		} else {
			panic("Unreachable")
		}
		p.X.Label.Text = "Time"
		p.X.Tick.Marker = dateTicks{args.start, args.end}

		p.Y.Tick.Marker = humanTicks{}
		p.Title.Text = args.title
		p.Legend.Top = true

		p.Y.Max = args.max
		p.Y.Min = args.min

		writer, err := p.WriterTo(args.width, args.height, extension)
		if err != nil {
			w.WriteHeader(400)
@@ -330,6 +423,8 @@ func main() {

	registerExtension(router, "chart", "svg", "image/svg+xml")
	registerExtension(router, "chart", "png", "image/png")
	registerExtension(router, "histogram", "svg", "image/svg+xml")
	registerExtension(router, "histogram", "png", "image/png")

	addr := ":8142"
	if len(os.Args) > 2 {
-- 
2.30.0

Re: [PATCH 2/3] Preserves tag list

Details
Message ID
<C8JIA7I1YLQ8.113HF0DZ6TZH7@pixelpc>
In-Reply-To
<20210115060745.24831-2-noam@pixelhero.dev> (view parent)
DKIM signature
pass
Download raw message
Rationale for this one:

The metric name was the only thing needed before I added label support, but the
label templating uses the raw tags, so it makes more sense now to preserve the
tags and render the name as needed instead of merging into into a string and
then splitting back into tags.
Reply to thread Export thread (mbox)