how to implement grayscale rendering in OpenGL?

Question

When rendering a scene of textured polygons, I'd like to be able to switch between rendering in the original colors and a "grayscale" mode. I've been trying to achieve this using blending and color matrix operations; none of it worked (with blending I couldn't find a glBlendFunc() that achieved something remotely resembling to what I wanted, and color matrix operations ...are discussed here).

A solution that comes to mind (but also is rather expensive) is to capture the screen every frame and convert the resulting texture to a grayscale one and display that instead... (Where I said grayscale I actually meant anything with a low saturation, but I'm guessing for most of the possible solutions it won't differ all that much from grayscale).

What other options do I have?

Answer 1

The default OpenGL framebuffer uses the RGB colour-space, which doesn't store an explicit saturation. You need an approach for extracting the saturation, modifying it, and change it back again.

My previous suggestion which simply used the RGB vector length to represent 0 in luminance was incorrect, as it didn't take scaling into account, I apologize.

Credit for the new short snippet goes to the regular user "RTFM_FTW" from ##opengl and ##opengl3 on FreeNode/IRC, and it lets you modify the saturation directly without computing the costly RGB->HSV->RGB conversion, which is exactly what you want. Though the HSV code is inferior with respect to your question, I let it stay.

void main( void ) 
{ 
    vec3 R0 = texture2DRect( S, gl_TexCoord[0].st ).rgb;
    gl_FragColor = vec4( mix( vec3( dot( R0, vec3( 0.2125, 0.7154, 0.0721 ) ) ),
        R0, T ), gl_Color.a ); 
}

If you want more control than just the saturation, you need to convert to HSL or HSV colour-space. As shown below by using a GLSL fragment shader.

Read the OpenGL 3.0 and GLSL 1.30 specification available on http://www.opengl.org/registry to learn how to use GLSL v1.30 functionality.

#version 130
#define RED 0
#define GREEN 1
#define BLUE 2

in vec4 vertexIn;
in vec4 colorIn;
in vec2 tcoordIn;
out vec4 pixel;
Sampler2D tex;
vec4 texel;
const float epsilon = 1e-6;

vec3 RGBtoHSV(vec3 color)
{
    /* hue, saturation and value are all in the range [0,1> here, as opposed to their
       normal ranges of: hue: [0,360>, sat: [0, 100] and value: [0, 256> */
    int sortindex[3] = {RED,GREEN,BLUE};
    float rgbArr[3] = float[3](color.r, color.g, color.b);

    float hue, saturation, value, diff;
    float minCol, maxCol;
    int minIndex, maxIndex;

    if(color.g < color.r)
        swap(sortindex[0], sortindex[1]);
    if(color.b < color.g)
        swap(sortindex[1], sortindex[2]);
    if(color.r < color.b)
        swap(sortindex[2], sortindex[0]);

    minIndex = sortindex[0];
    maxIndex = sortindex[2];
    minCol = rgbArr[minIndex];
    maxCol = rgbArr[maxIndex];

    diff = maxCol - minCol;

    /* Hue */
    if( diff < epsilon){
        hue = 0.0;
    }
    else if(maxIndex == RED){
        hue = ((1.0/6.0) * ( (color.g - color.b) / diff )) + 1.0;
        hue = fract(hue);
    }
    else if(maxIndex == GREEN){
        hue = ((1.0/6.0) * ( (color.b - color.r) / diff )) + (1.0/3.0);
    }
    else if(maxIndex == BLUE){
        hue = ((1.0/6.0) * ( (color.r - color.g) / diff )) + (2.0/3.0);        
    }

    /* Saturation */
    if(maxCol < epsilon)
        saturation = 0;
    else
        saturation = (maxCol - minCol) / maxCol;

    /* Value */
    value = maxCol;

    return vec3(hue, saturation, value);
}
vec3 HSVtoRGB(vec3 color)
{
    float f,p,q,t, hueRound;
    int hueIndex;
    float hue, saturation, value;
    vec3 result;

    /* just for clarity */
    hue = color.r;
    saturation = color.g;
    value = color.b;

    hueRound = floor(hue * 6.0);
    hueIndex = int(hueRound) % 6;
    f = (hue * 6.0) - hueRound;
    p = value * (1.0 - saturation);
    q = value * (1.0 - f*saturation);
    t = value * (1.0 - (1.0 - f)*saturation);

    switch(hueIndex)
    {
        case 0:
            result = vec3(value,t,p);
        break;
        case 1:
            result = vec3(q,value,p);
        break;
        case 2:
            result = vec3(p,value,t);
        break;
        case 3:
            result = vec3(p,q,value);
        break;
        case 4:
            result = vec3(t,p,value);
        break;
        default:
            result = vec3(value,p,q);
        break;
    }
    return result;
}
void main(void)
{
    vec4 srcColor;
    vec3 hsvColor;
    vec3 rgbColor;
    texel = Texture2D(tex, tcoordIn);
    srcColor = texel*colorIn;
    hsvColor = RGBtoHSV(srcColor.rgb);
    /* You can do further changes here, if you want. */
    hsvColor.g = 0; /* Set saturation to zero */
    rgbColor = HSVtoRGB(hsvColor);
    pixel = vec4(rgbColor.r, rgbColor.g, rgbColor.b, srcColor.a);
}