diff --git a/cogl/cogl-bitmap-fallback.c b/cogl/cogl-bitmap-fallback.c
index 00428d57a..a1802980f 100644
--- a/cogl/cogl-bitmap-fallback.c
+++ b/cogl/cogl-bitmap-fallback.c
@@ -359,8 +359,12 @@ _cogl_bitmap_fallback_convert (const CoglBitmap *bmp,
   /* Initialize destination bitmap */
   *dst_bmp = *bmp;
   dst_bmp->rowstride = sizeof(guint8) * dst_bpp * dst_bmp->width;
-  dst_bmp->format = ((bmp->format & COGL_PREMULT_BIT) |
-		     (dst_format & COGL_UNPREMULT_MASK));
+  /* Copy the premult bit if the new format has an alpha channel */
+  if ((dst_format & COGL_A_BIT))
+    dst_bmp->format = ((bmp->format & COGL_PREMULT_BIT) |
+                       (dst_format & COGL_UNPREMULT_MASK));
+  else
+    dst_bmp->format = dst_format;
 
   /* Allocate a new buffer to hold converted data */
   dst_bmp->data = g_malloc (sizeof(guint8)
diff --git a/cogl/cogl-bitmap.c b/cogl/cogl-bitmap.c
index 3b00cb8c9..77a28c76a 100644
--- a/cogl/cogl-bitmap.c
+++ b/cogl/cogl-bitmap.c
@@ -107,7 +107,15 @@ _cogl_bitmap_convert_format_and_premult (const CoglBitmap *bmp,
       dst_bmp->data = g_memdup (bmp->data, bmp->rowstride * bmp->height);
     }
 
-  if (!_cogl_bitmap_convert_premult_status (dst_bmp, dst_format))
+  /* We only need to do a premult conversion if both formats have an
+     alpha channel. If we're converting from RGB to RGBA then the
+     alpha will have been filled with 255 so the premult won't do
+     anything or if we are converting from RGBA to RGB we're losing
+     information so either converting or not will be wrong for
+     transparent pixels */
+  if ((dst_bmp->format & COGL_A_BIT) == COGL_A_BIT &&
+      (dst_format & COGL_A_BIT) == COGL_A_BIT &&
+      !_cogl_bitmap_convert_premult_status (dst_bmp, dst_format))
     {
       g_free (dst_bmp->data);
       return FALSE;