YUV2RGB conversion by using fragment shader

OpenGL Development JOGL Development
#1

Hello,
I’m trying to write a simple example where I try to make a conversion from YUV 4:4:4 into RGB by using a fragment shader to improve the conversion… Below is the code…still not running. Has someone have a clue why it is not processing? (the result is black screen)…

Java code :

package essai2;

import java.awt.Frame;
import java.awt.event.WindowAdapter;
import java.awt.event.WindowEvent;
import java.nio.ByteBuffer;
import java.nio.FloatBuffer;
import java.nio.IntBuffer;

import javax.media.opengl.GL;
import javax.media.opengl.GL2ES2;
import javax.media.opengl.GLAutoDrawable;
import javax.media.opengl.GLCapabilities;
import javax.media.opengl.GLEventListener;
import javax.media.opengl.GLException;
import javax.media.opengl.GLProfile;
import javax.media.opengl.awt.GLCanvas;

import com.sun.opengl.util.Animator;
import com.sun.opengl.util.BufferUtil;
import com.sun.opengl.util.GLArrayDataClient;
import com.sun.opengl.util.glsl.ShaderCode;
import com.sun.opengl.util.glsl.ShaderProgram;
import com.sun.opengl.util.glsl.ShaderState;

public class YUV2RGBSimpleShaderExample implements GLEventListener {

private   ShaderState   shaderState   = null;   ;
private   ShaderProgram shaderProgram = null;

public static void main(final String[] args) {

    final GLCapabilities capabilities = new GLCapabilities(GLProfile.getGL2ES2());

    final GLCanvas canvas = new GLCanvas(capabilities);
    final Animator animator = new Animator(canvas);
    final Frame frame = new Frame("Simple JOGL Application with shaders");

    canvas.addGLEventListener(new YUV2RGBSimpleShaderExample());

    frame.addWindowListener(new WindowAdapter() {

        @Override public void windowClosing(final WindowEvent e) {
            new Thread(new Runnable() {
                public void run() {
                    animator.stop();
                    System.exit(0);
                }
            }).start();

        }
    });

    frame.setSize(640, 480);
    frame.setLocationRelativeTo(null);
    frame.add(canvas);
    frame.setVisible(true);
    animator.start();

}

public void init(final GLAutoDrawable drawable) {
	shaderState   = new ShaderState();   ;
    shaderProgram = new ShaderProgram();

    GL2ES2 gl = drawable.getGL().getGL2ES2();


    final ShaderCode vertexShaderCode = ShaderCode.create(gl, GL2ES2.GL_VERTEX_SHADER, 1, YUV2RGBSimpleShaderExample.class, "shader", "shader", "vertex.glsl");
    final ShaderCode fragmentShaderCode = ShaderCode.create(gl, GL2ES2.GL_FRAGMENT_SHADER, 1, YUV2RGBSimpleShaderExample.class, "shader", "shader", "fragment.glsl");

    shaderProgram.add(vertexShaderCode);
    shaderProgram.add(fragmentShaderCode);
    if (!shaderProgram.link(gl, System.err)) throw new GLException("Couldn't link program: " + shaderProgram);

    shaderState.attachShaderProgram(gl, shaderProgram);
    shaderState.glUseProgram(gl, true);

    final GLArrayDataClient vertices = GLArrayDataClient.createGLSL(gl, "a_Vertex", 3, GL.GL_FLOAT, false, 4);
    final FloatBuffer verticeb = (FloatBuffer) vertices.getBuffer();
    verticeb.put(-0.5f);  verticeb.put(0.5f);   verticeb.put(0f);
    verticeb.put(0.5f);   verticeb.put(0.5f);   verticeb.put(0f);
    verticeb.put(-0.5f);  verticeb.put(-0.5f);  verticeb.put(0f);
    verticeb.put(0.5f);   verticeb.put(-0.5f);  verticeb.put(0f);
    vertices.seal(gl, true);

    final GLArrayDataClient texcords = GLArrayDataClient.createGLSL(gl, "a_TexCoord", 2, GL.GL_FLOAT, false, 4);
    final FloatBuffer texcordsb = (FloatBuffer) texcords.getBuffer();
    texcordsb.put(0f);  texcordsb.put(0f);
    texcordsb.put(1f);  texcordsb.put(0f);
    texcordsb.put(0f);  texcordsb.put(1f);
    texcordsb.put(1f);  texcordsb.put(1f);
    texcords.seal(gl, true);
    
    final IntBuffer buffer = BufferUtil.newIntBuffer(3);
    {
		ByteBuffer dataBuffer0 = ByteBuffer.allocate(4);
		dataBuffer0.put(0,(byte)0);
		dataBuffer0.put(1,(byte)255);
		dataBuffer0.put(2,(byte)0);
		dataBuffer0.put(3,(byte)255);

        gl.glPixelStorei(GL.GL_UNPACK_ALIGNMENT, 1);
        
        // Each texture is defined by GL_TEXTURE{n} in the order of definition of the texture
        gl.glActiveTexture(GL.GL_TEXTURE0);
        // we get the ID for the texture 0 and put it in the ids buffer
        gl.glGenTextures(0, buffer);
        // we tell opengl that the texture {n} is a 2D texture
        gl.glBindTexture(GL.GL_TEXTURE_2D, buffer.get(0));

        // we send the texture to the GPU; but we don't tell for the moment where to use the texture and how
        gl.glTexImage2D(GL2ES2.GL_TEXTURE_2D, 0, GL2ES2.GL_LUMINANCE, 2, 2, 0, GL2ES2.GL_LUMINANCE, GL2ES2.GL_UNSIGNED_BYTE, dataBuffer0);
    	gl.glTexParameteri(GL.GL_TEXTURE_2D,GL.GL_TEXTURE_MIN_FILTER, GL.GL_NEAREST );
		gl.glTexParameteri(GL.GL_TEXTURE_2D,GL.GL_TEXTURE_MAG_FILTER, GL.GL_NEAREST );
		
		final int samplerloc0 = gl.glGetUniformLocation(shaderProgram.id(), "s_Texture0");
        // once made, we  assign the texture 0 to the  s_texture constant
        gl.glUniform1i(samplerloc0, GL.GL_TEXTURE0);
    }
    
    {
    	ByteBuffer dataBuffer1 = ByteBuffer.allocate(4);
		dataBuffer1.put(0,(byte)255);
		dataBuffer1.put(1,(byte)0);
		dataBuffer1.put(2,(byte)0);
		dataBuffer1.put(3,(byte)128);

        gl.glPixelStorei(GL.GL_UNPACK_ALIGNMENT, 1);
        
        // Each texture is defined by GL_TEXTURE{n} in the order of definition of the texture
        gl.glActiveTexture(GL.GL_TEXTURE1);
        // we get the ID for the texture 1 and put it in the ids buffer
        gl.glGenTextures(1, buffer);
        // we tell opengl that the texture {n} is a 2D texture
        gl.glBindTexture(GL.GL_TEXTURE_2D, buffer.get(1));
		
        // we send the texture to the GPU; but we don't tell for the moment where to use the texture and how
        gl.glTexImage2D(GL2ES2.GL_TEXTURE_2D, 0, GL2ES2.GL_LUMINANCE, 2, 2, 0, GL2ES2.GL_LUMINANCE, GL2ES2.GL_UNSIGNED_BYTE, dataBuffer1);
    	gl.glTexParameteri(GL.GL_TEXTURE_2D,GL.GL_TEXTURE_MIN_FILTER, GL.GL_NEAREST );
		gl.glTexParameteri(GL.GL_TEXTURE_2D,GL.GL_TEXTURE_MAG_FILTER, GL.GL_NEAREST );
		
		final int samplerloc1 = gl.glGetUniformLocation(shaderProgram.id(), "s_Texture1");
        // once made, we  assign the texture 1 to the  s_texture constant
        gl.glUniform1i(samplerloc1, GL.GL_TEXTURE1);
    }
    
    {
    	ByteBuffer dataBuffer2 = ByteBuffer.allocate(4);
		dataBuffer2.put(0,(byte)255);
		dataBuffer2.put(1,(byte)0);
		dataBuffer2.put(2,(byte)0);
		dataBuffer2.put(3,(byte)128);

        gl.glPixelStorei(GL.GL_UNPACK_ALIGNMENT, 1);
        
        // Each texture is defined by GL_TEXTURE{n} in the order of definition of the texture
        gl.glActiveTexture(GL.GL_TEXTURE2);
        // we get the ID for the texture 2 and put it in the ids buffer
        gl.glGenTextures(2, buffer);
        // we tell opengl that the texture {n} is a 2D texture
        gl.glBindTexture(GL.GL_TEXTURE_2D, buffer.get(2));
		
        // we send the texture to the GPU; but we don't tell for the moment where to use the texture and how
        gl.glTexImage2D(GL2ES2.GL_TEXTURE_2D, 0, GL2ES2.GL_LUMINANCE, 2, 2, 0, GL2ES2.GL_LUMINANCE, GL2ES2.GL_UNSIGNED_BYTE, dataBuffer2);
    	gl.glTexParameteri(GL.GL_TEXTURE_2D,GL.GL_TEXTURE_MIN_FILTER, GL.GL_NEAREST );
		gl.glTexParameteri(GL.GL_TEXTURE_2D,GL.GL_TEXTURE_MAG_FILTER, GL.GL_NEAREST );
		
		final int samplerloc2 = gl.glGetUniformLocation(shaderProgram.id(), "s_Texture2");
        // once made, we  assign the texture 2 to the  s_texture constant
        gl.glUniform1i(samplerloc2, GL.GL_TEXTURE2);
    }

    shaderState.glUseProgram(gl, false);
}

public void display(final GLAutoDrawable drawable) {

    GL2ES2 gl = drawable.getGL().getGL2ES2();
    shaderState.glUseProgram(gl, true);

    gl.glClear(GL.GL_COLOR_BUFFER_BIT);

    // we display the vertexes ; here we have a rectangle
    gl.glDrawArrays(GL.GL_TRIANGLE_STRIP, 0, 4);

    shaderState.glUseProgram(gl, false);
}


public void reshape(final GLAutoDrawable drawable, final int x, final int y, final int width, final int height) {}

public void dispose(final GLAutoDrawable drawable) {}

public void displayChanged(final GLAutoDrawable drawable, final boolean modeChanged, final boolean deviceChanged) {}

}

Vertex code :

attribute vec4 a_Vertex;
attribute vec2 a_TexCoord;
varying vec2 v_TexCoord;

void main() {
gl_Position = a_Vertex;
v_TexCoord = a_TexCoord;
}

Fragment code :

varying vec2 v_TexCoord;
uniform sampler2D s_Texture0;
uniform sampler2D s_Texture1;
uniform sampler2D s_Texture2;

void main() {
vec3 pre;

pre.x = texture2D(s_Texture0, v_TexCoord).x - (16.0 / 256.0);
pre.y = texture2D(s_Texture1, v_TexCoord).x - (128.0 / 256.0);
pre.z = texture2D(s_Texture2, v_TexCoord).x - (128.0 / 256.0);

vec3 red = vec3 (0.00456621, 0.0, 0.00625893) * 255.0;
vec3 green = vec3 (0.00456621, -0.00153632, -0.00318811) * 255.0;
vec3 blue = vec3 (0.00456621, 0.00791071, 0.0) * 255.0;

gl_FragColor[0] = dot(red, pre);
gl_FragColor[1] = dot(green, pre);
gl_FragColor[2] = dot(blue, pre);

}

#2

There is an example which does this in o3d, which is a Google browser plugin for 3d graphics. It uses its own shader language, but it’s very similar to the others and should be easy to translate.
http://o3d.googlecode.com/svn/trunk/samples/yuv2rgb.html

#3

When I say ‘the others’, I must admit I was talking more about Cg and HLSL… I’ve never actually done any GLSL programming and it does look a bit different (I was literally able to copy and paste HLSL functions into O3d with no complications).

I think your problem is that you need to multiply input position by a WorldViewProjectionMatrix or similar in the vertex program. Whenever I have a shader that is just outputting black, the first thing I always try is to make it output something like pure red to verify that it’s behaving as expected up to that point.

#4

Your use of the buffer and the glGenTextures / glBindTexture are not solid correct. You only want 1 texture id (3 times), so it must be always glGenTextures(1, buffer) and glBindTexture(GL.GL_TEXTURE_2D, buffer.get(0)).
(The value of buffer.get(0) is 0/1/2 because glGenTextures request the first free texture id and save them at buffer.get(0))

So it works here:


        final IntBuffer buffer = BufferUtil.newIntBuffer(1); // <--
        gl.glPixelStorei(GL.GL_UNPACK_ALIGNMENT, 1);

        // texture 0
        ByteBuffer dataBuffer0 = ByteBuffer.allocate(4);
        dataBuffer0.put(0, (byte) 0);
        dataBuffer0.put(1, (byte) 255);
        dataBuffer0.put(2, (byte) 0);
        dataBuffer0.put(3, (byte) 255);

        gl.glActiveTexture(GL.GL_TEXTURE0);
        gl.glGenTextures(1, buffer); // <--
        gl.glBindTexture(GL.GL_TEXTURE_2D, buffer.get(0)); // <--

        gl.glTexImage2D(GL2ES2.GL_TEXTURE_2D, 0, GL2ES2.GL_LUMINANCE, 2, 2, 0, GL2ES2.GL_LUMINANCE, GL2ES2.GL_UNSIGNED_BYTE, dataBuffer0);
        gl.glTexParameteri(GL.GL_TEXTURE_2D, GL.GL_TEXTURE_MIN_FILTER, GL.GL_NEAREST);
        gl.glTexParameteri(GL.GL_TEXTURE_2D, GL.GL_TEXTURE_MAG_FILTER, GL.GL_NEAREST);

        gl.glUniform1i(gl.glGetUniformLocation(shaderProgram.id(), "s_Texture0"), 0); // <--


        // texture 1
        ByteBuffer dataBuffer1 = ByteBuffer.allocate(4);
        dataBuffer1.put(0, (byte) 255);
        dataBuffer1.put(1, (byte) 0);
        dataBuffer1.put(2, (byte) 0);
        dataBuffer1.put(3, (byte) 128);

        gl.glActiveTexture(GL.GL_TEXTURE1);
        gl.glGenTextures(1, buffer); // <--
        gl.glBindTexture(GL.GL_TEXTURE_2D, buffer.get(0)); // <--

        gl.glTexImage2D(GL2ES2.GL_TEXTURE_2D, 0, GL2ES2.GL_LUMINANCE, 2, 2, 0, GL2ES2.GL_LUMINANCE, GL2ES2.GL_UNSIGNED_BYTE, dataBuffer1);
        gl.glTexParameteri(GL.GL_TEXTURE_2D, GL.GL_TEXTURE_MIN_FILTER, GL.GL_NEAREST);
        gl.glTexParameteri(GL.GL_TEXTURE_2D, GL.GL_TEXTURE_MAG_FILTER, GL.GL_NEAREST);

        gl.glUniform1i(gl.glGetUniformLocation(shaderProgram.id(), "s_Texture1"), 1); // <--


        // texture 2
        ByteBuffer dataBuffer2 = ByteBuffer.allocate(4);
        dataBuffer2.put(0, (byte) 255);
        dataBuffer2.put(1, (byte) 0);
        dataBuffer2.put(2, (byte) 0);
        dataBuffer2.put(3, (byte) 128);

        gl.glActiveTexture(GL.GL_TEXTURE2);
        gl.glGenTextures(1, buffer); // <--
        gl.glBindTexture(GL.GL_TEXTURE_2D, buffer.get(0)); // <--

        gl.glTexImage2D(GL2ES2.GL_TEXTURE_2D, 0, GL2ES2.GL_LUMINANCE, 2, 2, 0, GL2ES2.GL_LUMINANCE, GL2ES2.GL_UNSIGNED_BYTE, dataBuffer2);
        gl.glTexParameteri(GL.GL_TEXTURE_2D, GL.GL_TEXTURE_MIN_FILTER, GL.GL_NEAREST);
        gl.glTexParameteri(GL.GL_TEXTURE_2D, GL.GL_TEXTURE_MAG_FILTER, GL.GL_NEAREST);

        gl.glUniform1i(gl.glGetUniformLocation(shaderProgram.id(), "s_Texture2"), 2); // <--

Regards,
Fancy

#5

Thanks a lot both of you, and more especially fancy :slight_smile: I still don’t completly understand the genTexture/bind texture stuff, but it works :slight_smile:
best regards
Xavier

#6

Hello,
To continue the example serie with Jogl with shaders, I would like now include the result of the previous example inside a JavaFX Scene…
Below is the code I’ve written, but the result is not at all what I expected… has someone a solution?

Thx in advance

The previous example modified :

public class YUV2RGBSimpleShaderExample implements GLEventListener {

private   ShaderState   shaderState   = null;   ;
private   ShaderProgram shaderProgram = null;
public Animator animator;

public GLCanvas canvas; 

public static void main(String[] args) {
	new YUV2RGBSimpleShaderExample();
}

public YUV2RGBSimpleShaderExample() {
	 final GLCapabilities capabilities = new GLCapabilities(GLProfile.getGL2ES2());
     canvas = new GLCanvas(capabilities);
     animator = new Animator(canvas);
     canvas.addGLEventListener(this); 

}

… (the code is the same)

SwingJComponent :

package essai2;

import javafx.stage.Stage;
import javafx.scene.Scene;
import javafx.ext.swing.SwingComponent;

import com.sun.opengl.util.Animator;
import java.lang.Exception;

import javax.media.opengl.awt.*;

public class MainCalcWrapper extends SwingComponent {
public override function createJComponent():GLJPanel {
var pnl:GLJPanel = new GLJPanel();
var yuver:YUV2RGBSimpleShaderExample = new YUV2RGBSimpleShaderExample();
try {
pnl.add(yuver.canvas);
yuver.animator.start();
} catch (e:Exception) {
e.printStackTrace();
}
return pnl;
}
}

Main.fx :
package essai2;

import javafx.stage.Stage;
import javafx.scene.Scene;
import javafx.ext.swing.SwingComponent;

import essai2.YUV2RGBSimpleShaderExample;

var example:YUV2RGBSimpleShaderExample = new YUV2RGBSimpleShaderExample();

Stage {
title: “Application title”
width: 720
height: 576
scene: Scene {
content: [
MainCalcWrapper {
width :720
height :576
}
]

}

}

The result is the following :

http://beurk.PNG