~eliasnaur/gio-patches

gio: gpu: support YUV/YCbCr image v1 PROPOSED

~dejadeja9
~dejadeja9: 1
 gpu: support YUV/YCbCr image

 3 files changed, 158 insertions(+), 24 deletions(-)
Export patchset (mbox)
How do I use this?

Copy & paste the following snippet into your terminal to import this patchset into git:

curl -s https://lists.sr.ht/~eliasnaur/gio-patches/patches/11394/mbox | git am -3
Learn more about email & git
View this thread in the archives

[PATCH gio] gpu: support YUV/YCbCr image Export this patch

~dejadeja9
From: Deja Deja <dejadejade9@outlook.com>

---
 gpu/gpu.go        | 168 +++++++++++++++++++++++++++++++++++++++++-----
 gpu/path.go       |   8 ++-
 op/paint/paint.go |   6 +-
 3 files changed, 158 insertions(+), 24 deletions(-)

diff --git a/gpu/gpu.go b/gpu/gpu.go
index 321b2a8..76fe746 100644
--- a/gpu/gpu.go
+++ b/gpu/gpu.go
@@ -111,6 +111,9 @@ type material struct {
	color f32color.RGBA
	// For materialTypeTexture.
	texture *texture
	// For yuvTypeTexture
	yuvtexture *yuvtexture

	uvTrans f32.Affine2D
}

@@ -123,6 +126,7 @@ type clipOp struct {
type imageOpData struct {
	rect   image.Rectangle
	src    *image.RGBA
	yuv    *image.YCbCr
	handle interface{}
}

@@ -146,16 +150,29 @@ func (op *clipOp) decode(data []byte) {
	}
}

func decodeImageOp(data []byte, refs []interface{}) imageOpData {
func decodeImageOp(data []byte, refs []interface{}) (mtype materialType, imgData imageOpData) {
	if opconst.OpType(data[0]) != opconst.TypeImage {
		panic("invalid op")
	}
	handle := refs[1]
	if handle == nil {
		return imageOpData{}
		return materialColor, imageOpData{}
	}
	bo := binary.LittleEndian
	return imageOpData{

	var src *image.RGBA
	var yuv *image.YCbCr
	if s, ok := refs[0].(*image.RGBA); ok {
		src = s
		mtype = materialTexture
	} else if y, ok := refs[0].(*image.YCbCr); ok {
		yuv = y
		mtype = materialYCbCr
	} else {
		panic("invalid image type")
	}

	imgData = imageOpData{
		rect: image.Rectangle{
			Min: image.Point{
				X: int(bo.Uint32(data[1:])),
@@ -166,9 +183,11 @@ func decodeImageOp(data []byte, refs []interface{}) imageOpData {
				Y: int(bo.Uint32(data[13:])),
			},
		},
		src:    refs[0].(*image.RGBA),
		src:    src,
		yuv:    yuv,
		handle: handle,
	}
	return
}

func decodeColorOp(data []byte) color.RGBA {
@@ -214,13 +233,19 @@ type texture struct {
	tex backend.Texture
}

type yuvtexture struct {
	src              *image.YCbCr
	ytex, utex, vtex backend.Texture
}

type blitter struct {
	ctx         backend.Device
	viewport    image.Point
	prog        [2]*program
	prog        [3]*program
	layout      backend.InputLayout
	colUniforms *blitColUniforms
	texUniforms *blitTexUniforms
	yuvUniforms *blitTexUniforms
	quadVerts   backend.Buffer
}

@@ -274,6 +299,7 @@ const (
const (
	materialColor materialType = iota
	materialTexture
	materialYCbCr
)

func New(ctx backend.Device) (*GPU, error) {
@@ -396,12 +422,65 @@ func (r *renderer) texHandle(t *texture) backend.Texture {
	return t.tex
}

func (r *renderer) yuvHandles(t *yuvtexture) (backend.Texture, backend.Texture, backend.Texture) {
	if t.ytex != nil && t.utex != nil && t.vtex != nil {
		return t.ytex, t.utex, t.vtex
	}

	//use YStride instead of Bounds().Dx()
	w, h := t.src.YStride, t.src.Bounds().Dy()
	tex, err := r.ctx.NewTexture(backend.TextureFormatLuminance, w, h, backend.FilterLinear, backend.FilterLinear, backend.BufferBindingTexture)
	if err != nil {
		panic(err)
	}
	tex.UploadPixels(t.src.Y, w, h)
	t.ytex = tex

	wsub, hsub := 1, 1
	switch t.src.SubsampleRatio {
	case image.YCbCrSubsampleRatio420:
		wsub, hsub = 2, 2
	case image.YCbCrSubsampleRatio444:
		break
	case image.YCbCrSubsampleRatio422:
		wsub, hsub = 2, 1
	}

	tex, err = r.ctx.NewTexture(backend.TextureFormatLuminance, w/wsub, h/hsub, backend.FilterLinear, backend.FilterLinear, backend.BufferBindingTexture)
	if err != nil {
		panic(err)
	}
	tex.UploadPixels(t.src.Cb, w/wsub, h/hsub)
	t.utex = tex

	tex, err = r.ctx.NewTexture(backend.TextureFormatLuminance, w/wsub, h/hsub, backend.FilterLinear, backend.FilterLinear, backend.BufferBindingTexture)
	if err != nil {
		panic(err)
	}
	tex.UploadPixels(t.src.Cr, w/wsub, h/hsub)
	t.vtex = tex

	return t.ytex, t.utex, t.vtex
}

func (t *texture) release() {
	if t.tex != nil {
		t.tex.Release()
	}
}

func (t *yuvtexture) release() {
	if t.ytex != nil {
		t.ytex.Release()
	}
	if t.utex != nil {
		t.utex.Release()
	}
	if t.vtex != nil {
		t.vtex.Release()
	}
}

func newRenderer(ctx backend.Device) *renderer {
	r := &renderer{
		ctx:     ctx,
@@ -436,8 +515,10 @@ func newBlitter(ctx backend.Device) *blitter {
	}
	b.colUniforms = new(blitColUniforms)
	b.texUniforms = new(blitTexUniforms)
	b.yuvUniforms = new(blitTexUniforms)

	prog, layout, err := createColorPrograms(ctx, shader_blit_vert, shader_blit_frag,
		[2]interface{}{&b.colUniforms.vert, &b.texUniforms.vert}, [2]interface{}{&b.colUniforms.frag, nil})
		[3]interface{}{&b.colUniforms.vert, &b.texUniforms.vert, &b.yuvUniforms.vert}, [3]interface{}{&b.colUniforms.frag, nil, nil})
	if err != nil {
		panic(err)
	}
@@ -454,8 +535,8 @@ func (b *blitter) release() {
	b.layout.Release()
}

func createColorPrograms(b backend.Device, vsSrc backend.ShaderSources, fsSrc [2]backend.ShaderSources, vertUniforms, fragUniforms [2]interface{}) ([2]*program, backend.InputLayout, error) {
	var progs [2]*program
func createColorPrograms(b backend.Device, vsSrc backend.ShaderSources, fsSrc [3]backend.ShaderSources, vertUniforms, fragUniforms [3]interface{}) ([3]*program, backend.InputLayout, error) {
	var progs [3]*program
	prog, err := b.NewProgram(vsSrc, fsSrc[materialTexture])
	if err != nil {
		return progs, nil, err
@@ -471,6 +552,7 @@ func createColorPrograms(b backend.Device, vsSrc backend.ShaderSources, fsSrc [2
		prog.SetFragmentUniforms(fragBuffer.buf)
	}
	progs[materialTexture] = newProgram(prog, vertBuffer, fragBuffer)

	prog, err = b.NewProgram(vsSrc, fsSrc[materialColor])
	if err != nil {
		progs[materialTexture].Release()
@@ -485,6 +567,23 @@ func createColorPrograms(b backend.Device, vsSrc backend.ShaderSources, fsSrc [2
		prog.SetFragmentUniforms(fragBuffer.buf)
	}
	progs[materialColor] = newProgram(prog, vertBuffer, fragBuffer)

	prog, err = b.NewProgram(vsSrc, fsSrc[materialYCbCr])
	if err != nil {
		progs[materialTexture].Release()
		progs[materialColor].Release()
		return progs, nil, err
	}
	if u := vertUniforms[materialYCbCr]; u != nil {
		vertBuffer = newUniformBuffer(b, u)
		prog.SetVertexUniforms(vertBuffer.buf)
	}
	if u := fragUniforms[materialYCbCr]; u != nil {
		fragBuffer = newUniformBuffer(b, u)
		prog.SetFragmentUniforms(fragBuffer.buf)
	}
	progs[materialYCbCr] = newProgram(prog, vertBuffer, fragBuffer)

	layout, err := b.NewInputLayout(vsSrc, []backend.InputDesc{
		{Type: backend.DataTypeFloat, Size: 2, Offset: 0},
		{Type: backend.DataTypeFloat, Size: 2, Offset: 4 * 2},
@@ -492,6 +591,7 @@ func createColorPrograms(b backend.Device, vsSrc backend.ShaderSources, fsSrc [2
	if err != nil {
		progs[materialTexture].Release()
		progs[materialColor].Release()
		progs[materialYCbCr].Release()
		return progs, nil, err
	}
	return progs, layout, nil
@@ -759,8 +859,8 @@ loop:
			state.matType = materialColor
			state.color = decodeColorOp(encOp.Data)
		case opconst.TypeImage:
			state.matType = materialTexture
			state.image = decodeImageOp(encOp.Data, encOp.Refs)
			//			state.matType = materialTexture
			state.matType, state.image = decodeImageOp(encOp.Data, encOp.Refs)
		case opconst.TypePaint:
			op := decodePaintOp(encOp.Data)
			// Transform (if needed) the painting rectangle and if so generate a clip path,
@@ -847,10 +947,15 @@ func (d *drawState) materialFor(cache *resourceCache, rect f32.Rectangle, off f3
		m.material = materialColor
		m.color = f32color.RGBAFromSRGB(d.color)
		m.opaque = m.color.A == 1.0
	case materialTexture:
		m.material = materialTexture
	case materialTexture, materialYCbCr:
		m.material = d.matType
		dr := boundRectF(rect.Add(off))
		sz := d.image.src.Bounds().Size()
		var sz image.Point
		if d.image.src != nil {
			sz = d.image.src.Bounds().Size()
		} else {
			sz = d.image.yuv.Bounds().Size()
		}
		sr := layout.FRect(d.image.rect)
		if dx := float32(dr.Dx()); dx != 0 {
			// Don't clip 1 px width sources.
@@ -868,13 +973,25 @@ func (d *drawState) materialFor(cache *resourceCache, rect f32.Rectangle, off f3
		}
		tex, exists := cache.get(d.image.handle)
		if !exists {
			t := &texture{
				src: d.image.src,
			if d.matType == materialTexture {
				t := &texture{
					src: d.image.src,
				}
				cache.put(d.image.handle, t)
				tex = t
			} else {
				t := &yuvtexture{
					src: d.image.yuv,
				}
				cache.put(d.image.handle, t)
				tex = t
			}
			cache.put(d.image.handle, t)
			tex = t
		}
		m.texture = tex.(*texture)
		if d.matType == materialTexture {
			m.texture = tex.(*texture)
		} else {
			m.yuvtexture = tex.(*yuvtexture)
		}
		uvScale, uvOffset := texSpaceTransform(sr, sz)
		m.uvTrans = trans.Mul(f32.Affine2D{}.Scale(f32.Point{}, uvScale).Offset(uvOffset))
	}
@@ -892,6 +1009,11 @@ func (r *renderer) drawZOps(ops []imageOp) {
		switch m.material {
		case materialTexture:
			r.ctx.BindTexture(0, r.texHandle(m.texture))
		case materialYCbCr:
			ytex, utex, vtex := r.yuvHandles(m.yuvtexture)
			r.ctx.BindTexture(0, ytex)
			r.ctx.BindTexture(1, utex)
			r.ctx.BindTexture(2, vtex)
		}
		drc := img.clip
		scale, off := clipSpaceTransform(drc, r.blitter.viewport)
@@ -912,6 +1034,11 @@ func (r *renderer) drawOps(ops []imageOp) {
		switch m.material {
		case materialTexture:
			r.ctx.BindTexture(0, r.texHandle(m.texture))
		case materialYCbCr:
			ytex, utex, vtex := r.yuvHandles(m.yuvtexture)
			r.ctx.BindTexture(0, ytex)
			r.ctx.BindTexture(1, utex)
			r.ctx.BindTexture(2, vtex)
		}
		drc := img.clip

@@ -954,6 +1081,11 @@ func (b *blitter) blit(z float32, mat materialType, col f32color.RGBA, scale, of
		b.texUniforms.vert.blitUniforms.uvTransformR1 = [4]float32{t1, t2, t3, 0}
		b.texUniforms.vert.blitUniforms.uvTransformR2 = [4]float32{t4, t5, t6, 0}
		uniforms = &b.texUniforms.vert.blitUniforms
	case materialYCbCr:
		t1, t2, t3, t4, t5, t6 := uvTrans.Elems()
		b.yuvUniforms.vert.blitUniforms.uvTransformR1 = [4]float32{t1, t2, t3, 0}
		b.yuvUniforms.vert.blitUniforms.uvTransformR2 = [4]float32{t4, t5, t6, 0}
		uniforms = &b.yuvUniforms.vert.blitUniforms
	}
	uniforms.z = z
	uniforms.transform = [4]float32{scale.X, scale.Y, off.X, off.Y}
diff --git a/gpu/path.go b/gpu/path.go
index 2dd5875..5be6c0c 100644
--- a/gpu/path.go
+++ b/gpu/path.go
@@ -28,9 +28,10 @@ type pather struct {

type coverer struct {
	ctx         backend.Device
	prog        [2]*program
	prog        [3]*program
	texUniforms *coverTexUniforms
	colUniforms *coverColUniforms
	yuvUniforms *coverTexUniforms
	layout      backend.InputLayout
}

@@ -149,9 +150,10 @@ func newCoverer(ctx backend.Device) *coverer {
	}
	c.colUniforms = new(coverColUniforms)
	c.texUniforms = new(coverTexUniforms)
	c.yuvUniforms = new(coverTexUniforms)
	prog, layout, err := createColorPrograms(ctx, shader_cover_vert, shader_cover_frag,
		[2]interface{}{&c.colUniforms.vert, &c.texUniforms.vert},
		[2]interface{}{&c.colUniforms.frag, nil},
		[3]interface{}{&c.colUniforms.vert, &c.texUniforms.vert, &c.yuvUniforms.vert},
		[3]interface{}{&c.colUniforms.frag, nil, nil},
	)
	if err != nil {
		panic(err)
diff --git a/op/paint/paint.go b/op/paint/paint.go
index b4a9a56..31aae93 100644
--- a/op/paint/paint.go
+++ b/op/paint/paint.go
@@ -24,7 +24,7 @@ type ImageOp struct {

	uniform bool
	color   color.RGBA
	src     *image.RGBA
	src     image.Image

	// handle is a key to uniquely identify this ImageOp
	// in a map of cached textures.
@@ -60,9 +60,9 @@ func NewImageOp(src image.Image) ImageOp {
			uniform: true,
			color:   col,
		}
	case *image.RGBA:
	case *image.RGBA, *image.YCbCr:
		bounds := src.Bounds()
		if bounds.Min == (image.Point{}) && src.Stride == bounds.Dx()*4 {
		if bounds.Min == (image.Point{}) { // && src.Stride == bounds.Dx()*4 {
			return ImageOp{
				Rect:   src.Bounds(),
				src:    src,
-- 
2.26.2
Thanks. As you allude in an earlier message, the yuv and rgba paths are quite
similar. I think you can exploit that in the code below.

On Mon Jun 29, 2020 at 11:18, ~dejadeja9 wrote: