From 23044a1047162a9f103689b8c6fac6ff0e45e62f Mon Sep 17 00:00:00 2001 From: Neil Roberts Date: Tue, 21 Jan 2014 17:33:56 +0000 Subject: [PATCH] cogl-gst: video-sink: add NV12 support This adds a cogl-gst renderer to decode NV12 data. NV12 is split into two buffers, one for the luma component and another for the two chrominance components at a quarter of the resolution. The second buffer is uploaded to a two-component RG texture. RG-component textures are only supported if COGL_FEATURE_ID_TEXTURE_RG is advertised by Cogl so the NV12 cap is also only advertised when that is available. Based on a patch by Lionel Landwerlin which was in turn based on a patch from Edward Hervey and Matthieu Bouron for Clutter-Gst: https://bugzilla.gnome.org/show_bug.cgi?id=712832 Reviewed-by: Robert Bragg Reviewed-by: Lionel Landwerlin (cherry picked from commit 2c619216964b46aab313be3ef1c405dfc720d258) --- cogl-gst/cogl-gst-video-sink.c | 117 +++++++++++++++++++++++++++++++-- 1 file changed, 111 insertions(+), 6 deletions(-) diff --git a/cogl-gst/cogl-gst-video-sink.c b/cogl-gst/cogl-gst-video-sink.c index 684d8d27e..7e8963606 100644 --- a/cogl-gst/cogl-gst-video-sink.c +++ b/cogl-gst/cogl-gst-video-sink.c @@ -55,7 +55,8 @@ "RGBA," \ "BGRA," \ "RGB," \ - "BGR }" + "BGR," \ + "NV12 }" #define SINK_CAPS GST_VIDEO_CAPS_MAKE (BASE_SINK_CAPS) @@ -103,12 +104,14 @@ typedef enum COGL_GST_AYUV, COGL_GST_YV12, COGL_GST_SURFACE, - COGL_GST_I420 + COGL_GST_I420, + COGL_GST_NV12 } CoglGstVideoFormat; typedef enum { - COGL_GST_RENDERER_NEEDS_GLSL = (1 << 0) + COGL_GST_RENDERER_NEEDS_GLSL = (1 << 0), + COGL_GST_RENDERER_NEEDS_TEXTURE_RG = (1 << 1) } CoglGstRendererFlag; /* We want to cache the snippets instead of recreating a new one every @@ -912,11 +915,105 @@ static CoglGstRenderer ayuv_glsl_renderer = cogl_gst_ayuv_upload, }; +static void +cogl_gst_nv12_glsl_setup_pipeline (CoglGstVideoSink *sink, + CoglPipeline *pipeline) +{ + CoglGstVideoSinkPrivate *priv = sink->priv; + static SnippetCache snippet_cache; + SnippetCacheEntry *entry; + + entry = get_cache_entry (sink, &snippet_cache); + + if (entry == NULL) + { + char *source; + + source = + g_strdup_printf ("vec4\n" + "cogl_gst_sample_video%i (vec2 UV)\n" + "{\n" + " vec4 color;\n" + " float y = 1.1640625 *\n" + " (texture2D (cogl_sampler%i, UV).a -\n" + " 0.0625);\n" + " vec2 uv = texture2D (cogl_sampler%i, UV).rg;\n" + " uv -= 0.5;\n" + " float u = uv.x;\n" + " float v = uv.y;\n" + " color.r = y + 1.59765625 * v;\n" + " color.g = y - 0.390625 * u - 0.8125 * v;\n" + " color.b = y + 2.015625 * u;\n" + " color.a = 1.0;\n" + " return color;\n" + "}\n", + priv->custom_start, + priv->custom_start, + priv->custom_start + 1); + + entry = add_cache_entry (sink, &snippet_cache, source); + g_free (source); + } + + setup_pipeline_from_cache_entry (sink, pipeline, entry, 2); +} + +static CoglBool +cogl_gst_nv12_upload (CoglGstVideoSink *sink, + GstBuffer *buffer) +{ + CoglGstVideoSinkPrivate *priv = sink->priv; + GstVideoFrame frame; + + if (!gst_video_frame_map (&frame, &priv->info, buffer, GST_MAP_READ)) + goto map_fail; + + clear_frame_textures (sink); + + priv->frame[0] = + video_texture_new_from_data (priv->ctx, + GST_VIDEO_INFO_COMP_WIDTH (&priv->info, 0), + GST_VIDEO_INFO_COMP_HEIGHT (&priv->info, 0), + COGL_PIXEL_FORMAT_A_8, + priv->info.stride[0], + frame.data[0]); + + priv->frame[1] = + video_texture_new_from_data (priv->ctx, + GST_VIDEO_INFO_COMP_WIDTH (&priv->info, 1), + GST_VIDEO_INFO_COMP_HEIGHT (&priv->info, 1), + COGL_PIXEL_FORMAT_RG_88, + priv->info.stride[1], + frame.data[1]); + + gst_video_frame_unmap (&frame); + + return TRUE; + + map_fail: + { + GST_ERROR_OBJECT (sink, "Could not map incoming video frame"); + return FALSE; + } +} + +static CoglGstRenderer nv12_glsl_renderer = +{ + "NV12 glsl", + COGL_GST_NV12, + COGL_GST_RENDERER_NEEDS_GLSL | COGL_GST_RENDERER_NEEDS_TEXTURE_RG, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES ("memory:SystemMemory", + "NV12")), + 2, /* n_layers */ + cogl_gst_nv12_glsl_setup_pipeline, + cogl_gst_nv12_upload, +}; + static GSList* cogl_gst_build_renderers_list (CoglContext *ctx) { GSList *list = NULL; - CoglBool has_glsl; + CoglGstRendererFlag flags = 0; int i; static CoglGstRenderer *const renderers[] = { @@ -930,13 +1027,18 @@ cogl_gst_build_renderers_list (CoglContext *ctx) &yv12_glsl_renderer, &i420_glsl_renderer, &ayuv_glsl_renderer, + &nv12_glsl_renderer, NULL }; - has_glsl = cogl_has_feature (ctx, COGL_FEATURE_ID_GLSL); + if (cogl_has_feature (ctx, COGL_FEATURE_ID_GLSL)) + flags |= COGL_GST_RENDERER_NEEDS_GLSL; + + if (cogl_has_feature (ctx, COGL_FEATURE_ID_TEXTURE_RG)) + flags |= COGL_GST_RENDERER_NEEDS_TEXTURE_RG; for (i = 0; renderers[i]; i++) - if (has_glsl || !(renderers[i]->flags & COGL_GST_RENDERER_NEEDS_GLSL)) + if ((renderers[i]->flags & flags) == renderers[i]->flags) list = g_slist_prepend (list, renderers[i]); return list; @@ -1060,6 +1162,9 @@ cogl_gst_video_sink_parse_caps (GstCaps *caps, format = COGL_GST_AYUV; bgr = FALSE; break; + case GST_VIDEO_FORMAT_NV12: + format = COGL_GST_NV12; + break; case GST_VIDEO_FORMAT_RGB: format = COGL_GST_RGB24; bgr = FALSE;