rg16 is actually rgb565

drm.c

@@ -32,8 +32,57 @@
 
 extern struct kmsvnc_data *kmsvnc;
 
+static int _rg16_clamp_flip(int x, int to) {
+    if (x < 0) return -x;
+    else if (x >= to) return to + to - x - 2;
+    else return x;
+}
+static void convert_bggr16le_to_rgba(const char *in, int width, int height, char *buff) {
+    if (unlikely(in == buff)) return;
+    #define RG16_BPP 2
+    #define RG16_GET_NBR_PIX(_x,_y) ( __builtin_bswap16(htons(*(uint16_t*)(in + RG16_BPP*(_rg16_clamp_flip((_x),width) + _rg16_clamp_flip((_y),height) * width)))) )
+    for (int y = 0; y < height; y++) {
+        for (int x = 0; x < width; x++) {
+            int i = x + y * width;
+            uint16_t v = __builtin_bswap16(htons(*(uint16_t*)(in + i * RG16_BPP)));
+            if ((x % 2) ^ (y % 2)) {
+                // g1,g2
+                *(uint8_t*)(buff + i * BYTES_PER_PIXEL + 1) = 0xff & (v >> 8);
+                uint32_t rb2 = RG16_GET_NBR_PIX(x-1, y) + RG16_GET_NBR_PIX(x+1, y);
+                uint32_t br2 = RG16_GET_NBR_PIX(x, y-1) + RG16_GET_NBR_PIX(x, y+1);
+                if (x % 2) {
+                    *(uint8_t*)(buff + i * BYTES_PER_PIXEL + 0) = 0xff & (rb2 >> 9);
+                    *(uint8_t*)(buff + i * BYTES_PER_PIXEL + 2) = 0xff & (br2 >> 9);
+                }
+                else {
+                    *(uint8_t*)(buff + i * BYTES_PER_PIXEL + 2) = 0xff & (rb2 >> 9);
+                    *(uint8_t*)(buff + i * BYTES_PER_PIXEL + 0) = 0xff & (br2 >> 9);
+                }
+            }
+            else {
+                // r,b
+                uint32_t g4 = RG16_GET_NBR_PIX(x-1, y) + RG16_GET_NBR_PIX(x+1, y) + RG16_GET_NBR_PIX(x, y-1) + RG16_GET_NBR_PIX(x, y+1);
+                *(uint8_t*)(buff + i * BYTES_PER_PIXEL + 1) = 0xff & (g4 >> 10);
+                uint32_t br4 = RG16_GET_NBR_PIX(x-1, y-1) + RG16_GET_NBR_PIX(x-1, y+1) + RG16_GET_NBR_PIX(x+1, y-1) + RG16_GET_NBR_PIX(x+1, y+1);
+                if (x % 2) {
+                    *(uint8_t*)(buff + i * BYTES_PER_PIXEL + 0) = 0xff & (v >> 8);
+                    *(uint8_t*)(buff + i * BYTES_PER_PIXEL + 2) = 0xff & (br4 >> 10);
+                }
+                else {
+                    *(uint8_t*)(buff + i * BYTES_PER_PIXEL + 2) = 0xff & (v >> 8);
+                    *(uint8_t*)(buff + i * BYTES_PER_PIXEL + 0) = 0xff & (br4 >> 10);
+                }
+            }
+        }
+    }
+}
+
 static int check_pixfmt_non_vaapi() {
-    if (
+    if (kmsvnc->drm->mfb->pixel_format == KMSVNC_FOURCC_TO_INT('R', 'G', '1', '6'))
+    {
+        kmsvnc->drm->funcs->convert = convert_bggr16le_to_rgba;
+    }
+    else if (
         kmsvnc->drm->mfb->pixel_format != KMSVNC_FOURCC_TO_INT('X', 'R', '2', '4') &&
         kmsvnc->drm->mfb->pixel_format != KMSVNC_FOURCC_TO_INT('A', 'R', '2', '4')
     )
@@ -706,7 +755,8 @@ int drm_vendors() {
     }
     else if (strcmp(driver_name, "vmwgfx") == 0 ||
              strcmp(driver_name, "vboxvideo") == 0 ||
-             strcmp(driver_name, "virtio_gpu") == 0
+             strcmp(driver_name, "virtio_gpu") == 0 ||
+             strcmp(driver_name, "vc4") == 0
     )
     {
         if (check_pixfmt_non_vaapi()) return 1;