Add support for RGBA formats with separate alpha plane

This patch adds support for the RGBX8888_A8, BGRX8888_A8, RGB888_A8, BGR888_A8,
RGB565_A8, and BGR565_A8 RGB formats with 8-bit alpha on a separate plane.

Signed-off-by: Philipp Zabel <p.zabel@pengutronix.de>

3 files changed, 143 insertions, 11 deletions

diff --git a/src/bgi.c b/src/bgi.c
index 242abce..d4fccbf 100644
--- a/src/bgi.c
+++ b/src/bgi.c
@@ -22,8 +22,8 @@
 #include <stdint.h>
 #include <time.h>
 #include <pthread.h>
-#include <drm_fourcc.h>
 
+#include "drm_fourcc_rgb_a.h"
 #include "kmsfb.h"
 
 static void fbdev_open(fuse_req_t req, struct fuse_file_info *fi)
@@ -398,11 +398,29 @@ int bgi_init(struct kms_fb *fb)
 {
 	int ret;
 	pthread_t threadid;
-
+	int yres;
 	fb->can_wait = 0;
 
+	switch (fb->format) {
+	case DRM_FORMAT_RGB565_A8:
+	case DRM_FORMAT_BGR565_A8:
+		yres = fb->yres + (fb->yres + 1) / 2;
+		break;
+	case DRM_FORMAT_RGB888_A8:
+	case DRM_FORMAT_BGR888_A8:
+		yres = fb->yres + (fb->yres + 2) / 3;
+		break;
+	case DRM_FORMAT_RGBX8888_A8:
+	case DRM_FORMAT_BGRX8888_A8:
+		yres = fb->yres + (fb->yres + 3) / 4;
+		break;
+	default:
+		yres = fb->yres;
+		break;
+	}
+
 	fb->fb_fix.smem_start = fb->phys;
-	fb->fb_fix.smem_len = fb->pitch * fb->yres * fb->num_slices;
+	fb->fb_fix.smem_len = fb->pitch * yres * fb->num_slices;
 	fb->fb_fix.line_length = fb->pitch;
 	fb->fb_fix.visual = 2;
 	fb->fb_fix.ypanstep = 1;
@@ -411,13 +429,14 @@ int bgi_init(struct kms_fb *fb)
 	fb->fb_var.height = 0xffffffff;
 	fb->fb_var.width = 0xffffffff;
 	fb->fb_var.xres = fb->xres;
-	fb->fb_var.yres = fb->yres;
+	fb->fb_var.yres = yres;
 	fb->fb_var.xres_virtual = fb->xres;
-	fb->fb_var.yres_virtual = fb->yres * fb->num_slices;
+	fb->fb_var.yres_virtual = yres * fb->num_slices;
 	fb->fb_var.bits_per_pixel = fb->bpp;
 
 	switch (fb->format) {
 	case DRM_FORMAT_RGB565:
+	case DRM_FORMAT_RGB565_A8:
 		fb->fb_var.transp.offset = 0;
 		fb->fb_var.transp.length = 0;
 		fb->fb_var.red.offset = 11;
@@ -428,6 +447,7 @@ int bgi_init(struct kms_fb *fb)
 		fb->fb_var.blue.length = 5;
 		break;
 	case DRM_FORMAT_BGR565:
+	case DRM_FORMAT_BGR565_A8:
 		fb->fb_var.transp.offset = 0;
 		fb->fb_var.transp.length = 0;
 		fb->fb_var.red.offset = 0;
@@ -448,6 +468,7 @@ int bgi_init(struct kms_fb *fb)
 		fb->fb_var.blue.length = 5;
 		break;
 	case DRM_FORMAT_RGB888:
+	case DRM_FORMAT_RGB888_A8:
 		fb->fb_var.transp.offset = 0;
 		fb->fb_var.transp.length = 0;
 		fb->fb_var.red.offset = 0;
@@ -458,6 +479,7 @@ int bgi_init(struct kms_fb *fb)
 		fb->fb_var.blue.length = 8;
 		break;
 	case DRM_FORMAT_BGR888:
+	case DRM_FORMAT_BGR888_A8:
 		fb->fb_var.transp.offset = 0;
 		fb->fb_var.transp.length = 0;
 		fb->fb_var.red.offset = 16;
@@ -491,6 +513,7 @@ int bgi_init(struct kms_fb *fb)
 		break;
 	case DRM_FORMAT_RGBA8888:
 	case DRM_FORMAT_RGBX8888:
+	case DRM_FORMAT_RGBX8888_A8:
 		fb->fb_var.red.offset = 24;
 		fb->fb_var.red.length = 8;
 		fb->fb_var.green.offset = 16;
@@ -502,6 +525,7 @@ int bgi_init(struct kms_fb *fb)
 		break;
 	case DRM_FORMAT_BGRA8888:
 	case DRM_FORMAT_BGRX8888:
+	case DRM_FORMAT_BGRX8888_A8:
 		fb->fb_var.red.offset = 8;
 		fb->fb_var.red.length = 8;
 		fb->fb_var.green.offset = 16;
diff --git a/src/drm_fourcc_rgb_a.h b/src/drm_fourcc_rgb_a.h
new file mode 100644
index 0000000..0308df5
--- /dev/null
+++ b/src/drm_fourcc_rgb_a.h
@@ -0,0 +1,25 @@
+#ifndef __DRM_FOURCC_RGB_A_H__
+#define __DRM_FOURCC_RGB_A_H__
+
+#include <drm_fourcc.h>
+
+/*
+ * If the kernel headers don't include 2 plane RGB + A formats yet,
+ * add them manually:
+ */
+
+#ifndef DRM_FORMAT_RGB888_A8
+/*
+ * 2 plane RGB + A
+ * index 0 = RGB plane
+ * index 1 = A plane
+ */
+#define DRM_FORMAT_RGBX8888_A8	fourcc_code('R', 'X', 'A', '8')
+#define DRM_FORMAT_BGRX8888_A8	fourcc_code('B', 'X', 'A', '8')
+#define DRM_FORMAT_RGB888_A8    fourcc_code('R', '8', 'A', '8')
+#define DRM_FORMAT_BGR888_A8    fourcc_code('B', '8', 'A', '8')
+#define DRM_FORMAT_RGB565_A8    fourcc_code('R', '5', 'A', '8')
+#define DRM_FORMAT_BGR565_A8    fourcc_code('B', '5', 'A', '8')
+#endif /* DRM_FORMAT_RGB888_A8 */
+
+#endif /* __DRM_FOURCC_RGB_A_H__ */
diff --git a/src/kmsfb-manage.c b/src/kmsfb-manage.c
index c03f0f1..5fd7c84 100644
--- a/src/kmsfb-manage.c
+++ b/src/kmsfb-manage.c
@@ -28,6 +28,7 @@
 
 #include <xf86drmMode.h>
 
+#include "drm_fourcc_rgb_a.h"
 #include "kmsfb.h"
 
 int debug;
@@ -174,14 +175,26 @@ static int kms_data_val_set(struct kms_data *kd, const char *key, const char *va
 			kd->format = DRM_FORMAT_BGRX8888;
 		else if (!strcasecmp(value, "bgra8888"))
 			kd->format = DRM_FORMAT_BGRA8888;
+		else if (!strcasecmp(value, "rgbx8888_a8"))
+			kd->format = DRM_FORMAT_RGBX8888_A8;
+		else if (!strcasecmp(value, "bgrx8888_a8"))
+			kd->format = DRM_FORMAT_BGRX8888_A8;
 		else if (!strcasecmp(value, "rgb888"))
 			kd->format = DRM_FORMAT_RGB888;
 		else if (!strcasecmp(value, "bgr888"))
 			kd->format = DRM_FORMAT_BGR888;
+		else if (!strcasecmp(value, "rgb888_a8"))
+			kd->format = DRM_FORMAT_RGB888_A8;
+		else if (!strcasecmp(value, "bgr888_a8"))
+			kd->format = DRM_FORMAT_BGR888_A8;
 		else if (!strcasecmp(value, "rgb565"))
 			kd->format = DRM_FORMAT_RGB565;
 		else if (!strcasecmp(value, "bgr565"))
 			kd->format = DRM_FORMAT_BGR565;
+		else if (!strcasecmp(value, "rgb565_a8"))
+			kd->format = DRM_FORMAT_RGB565_A8;
+		else if (!strcasecmp(value, "bgr565_a8"))
+			kd->format = DRM_FORMAT_BGR565_A8;
 		else if (!strcasecmp(value, "argb1555"))
 			kd->format = DRM_FORMAT_ARGB1555;
 		else {
@@ -682,8 +695,26 @@ static int drm_mode_get_dumb(int fd, struct kms_fb *fb, int create_splash)
 	}
 
 	dumb.width = fb->xres;
-	dumb.height = fb->yres * fb->num_slices;
-	dumb.bpp = fb->bpp;
+	switch (fb->format) {
+	case DRM_FORMAT_RGB565_A8:
+	case DRM_FORMAT_BGR565_A8:
+		dumb.height = fb->yres + (fb->yres + 1) / 2;
+		dumb.bpp = 16;
+		break;
+	case DRM_FORMAT_RGB888_A8:
+	case DRM_FORMAT_BGR888_A8:
+		dumb.height = fb->yres + (fb->yres + 2) / 3;
+		dumb.bpp = 24;
+		break;
+	case DRM_FORMAT_RGBX8888_A8:
+	case DRM_FORMAT_BGRX8888_A8:
+		dumb.height = fb->yres + (fb->yres + 3) / 4;
+		dumb.bpp = 32;
+		break;
+	default:
+		dumb.height = fb->yres * fb->num_slices;
+		dumb.bpp = fb->bpp;
+	}
 	dumb.flags = 0;
 
 	err = ioctl(fd, DRM_IOCTL_MODE_CREATE_DUMB, &dumb);
@@ -892,6 +923,24 @@ static int create_drm_fbs(struct kms_fb *kms_fb)
 	int err, i;
 
 	switch (kms_fb->format) {
+	case DRM_FORMAT_RGB565_A8:
+	case DRM_FORMAT_BGR565_A8:
+		bo_handles[1] = kms_fb->gem_name;
+		pitches[1] = (kms_fb->pitch + 1) / 2;
+		offsets[1] = kms_fb->pitch * kms_fb->yres;
+		break;
+	case DRM_FORMAT_RGB888_A8:
+	case DRM_FORMAT_BGR888_A8:
+		bo_handles[1] = kms_fb->gem_name;
+		pitches[1] = (kms_fb->pitch + 2) / 3;
+		offsets[1] = kms_fb->pitch * kms_fb->yres;
+		break;
+	case DRM_FORMAT_RGBX8888_A8:
+	case DRM_FORMAT_BGRX8888_A8:
+		bo_handles[1] = kms_fb->gem_name;
+		pitches[1] = (kms_fb->pitch + 3) / 4;
+		offsets[1] = kms_fb->pitch * kms_fb->yres;
+		break;
 	case DRM_FORMAT_ARGB1555:
 	case DRM_FORMAT_RGB565:
 	case DRM_FORMAT_BGR565:
@@ -1269,10 +1318,14 @@ static int fb_create(struct drm_resource *res, int fd, const char *cmd)
 	case DRM_FORMAT_RGB565:
 	case DRM_FORMAT_BGR565:
 	case DRM_FORMAT_ARGB1555:
+	case DRM_FORMAT_BGR565_A8:
+	case DRM_FORMAT_RGB565_A8:
 		kd.bpp = 16;
 		break;
 	case DRM_FORMAT_RGB888:
 	case DRM_FORMAT_BGR888:
+	case DRM_FORMAT_RGB888_A8:
+	case DRM_FORMAT_BGR888_A8:
 		kd.bpp = 24;
 		break;
 	case DRM_FORMAT_XRGB8888:
@@ -1283,6 +1336,8 @@ static int fb_create(struct drm_resource *res, int fd, const char *cmd)
 	case DRM_FORMAT_RGBA8888:
 	case DRM_FORMAT_BGRX8888:
 	case DRM_FORMAT_BGRA8888:
+	case DRM_FORMAT_RGBX8888_A8:
+	case DRM_FORMAT_BGRX8888_A8:
 		kd.bpp = 32;
 		break;
 	default:
@@ -1343,8 +1398,11 @@ static int fb_create(struct drm_resource *res, int fd, const char *cmd)
 		} else if (kd.format == DRM_FORMAT_RGBX8888 ||
 			   kd.format == DRM_FORMAT_RGBA8888 ||
 			   kd.format == DRM_FORMAT_BGRX8888 ||
-			   kd.format == DRM_FORMAT_BGRA8888) {
+			   kd.format == DRM_FORMAT_BGRA8888 ||
+			   kd.format == DRM_FORMAT_RGBX8888_A8 ||
+			   kd.format == DRM_FORMAT_BGRX8888_A8) {
 			uint32_t *pix;
+			uint8_t *alpha;
 
 			for (y = 0; y < fb->yres; y++) {
 				for (x = 0; x < fb->xres; x++) {
@@ -1353,11 +1411,21 @@ static int fb_create(struct drm_resource *res, int fd, const char *cmd)
 						((0xff - (y & 0xff)) << 8) |
 						(((0xff - x) & 0xff) << 24) |
 						(((x ^ y) & 1) ? 0x00 : 0xff);
+					if (kd.format == DRM_FORMAT_RGBX8888 ||
+					    kd.format == DRM_FORMAT_RGBA8888 ||
+					    kd.format == DRM_FORMAT_BGRX8888 ||
+					    kd.format == DRM_FORMAT_BGRA8888)
+						continue;
+					alpha = fb->fb + fb->yres * fb->pitch +
+						x + y * ((fb->pitch + 3) / 4);
+					*alpha = ((x ^ y) & 1) ? 0x00 : 0xff;
 				}
 			}
 		} else if (kd.format == DRM_FORMAT_BGR888 ||
-			   kd.format == DRM_FORMAT_RGB888) {
-			uint8_t *pix;
+			   kd.format == DRM_FORMAT_RGB888 ||
+			   kd.format == DRM_FORMAT_BGR888_A8 ||
+			   kd.format == DRM_FORMAT_RGB888_A8) {
+			uint8_t *pix, *alpha;
 
 			for (y = 0; y < fb->yres; y++) {
 				for (x = 0; x < fb->xres; x++) {
@@ -1366,11 +1434,20 @@ static int fb_create(struct drm_resource *res, int fd, const char *cmd)
 					pix[1] = x & 0xff;
 					pix[2] = 0xff - (y & 0xff);
 						(((x ^ y) & 1) ? 0x00 : 0xff);
+					if (kd.format == DRM_FORMAT_BGR888 ||
+					    kd.format == DRM_FORMAT_RGB888)
+						continue;
+					alpha = fb->fb + fb->yres * fb->pitch +
+						x + y * ((fb->pitch + 2) / 3);
+					*alpha = ((x ^ y) & 1) ? 0x00 : 0xff;
 				}
 			}
 		} else if (kd.format == DRM_FORMAT_RGB565 ||
-			   kd.format == DRM_FORMAT_BGR565) {
+			   kd.format == DRM_FORMAT_BGR565 ||
+			   kd.format == DRM_FORMAT_RGB565_A8 ||
+			   kd.format == DRM_FORMAT_BGR565_A8) {
 			uint16_t *pix;
+			uint8_t *alpha;
 
 			for (y = 0; y < fb->yres; y++) {
 				for (x = 0; x < fb->xres; x++) {
@@ -1378,6 +1455,12 @@ static int fb_create(struct drm_resource *res, int fd, const char *cmd)
 					*pix = ((x & 0x3f) << 5) |
 						(0x1f - (y & 0x1f)) |
 						(((0x1f - x) & 0x1f) << 11);
+					if (kd.format == DRM_FORMAT_RGB565 ||
+					    kd.format == DRM_FORMAT_BGR565)
+						continue;
+					alpha = fb->fb + fb->yres * fb->pitch +
+						x + y * ((fb->pitch + 2) / 3);
+					*alpha = ((x ^ y) & 1) ? 0x00 : 0xff;
 				}
 			}
 		} else if (kd.format == DRM_FORMAT_ARGB1555) {