switch to gitea actions

.drone.yml

@@ -1,21 +0,0 @@
-kind: pipeline
-type: docker
-name: default
-
-steps:
-- name: build
-  image: archlinux:latest
-  commands:
-  - pacman -Syu --noconfirm --needed base-devel libvncserver libxkbcommon libdrm libva git cmake
-  - mkdir build
-  - cd build
-  - cmake ..
-  - make
-
-trigger:
-  branch:
-  - master
-  - dev
-  event:
-    exclude:
-    - pull_request

.forgejo/workflows/default.yml

@@ -0,0 +1,27 @@
+name: Build with gcc + clang
+on:
+  push:
+    branches: [dev]
+jobs:
+  build:
+    if: "github.event_name != 'push' || !contains(github.event.head_commit.message, '[skip ci]')"
+    runs-on: docker
+    container:
+      image: archlinux:latest
+    env:
+      CFLAGS: "-pipe -fno-plt -fexceptions -fstack-clash-protection -fcf-protection -Wp,-D_FORTIFY_SOURCE=2 -Wformat -Werror=format-security"
+    steps:
+      - name: Prepare dependencies
+        run: |
+          pacman -Syu --noconfirm --needed nodejs git \
+            base-devel libvncserver libxkbcommon libdrm libva cmake clang
+      - name: Check out repository code
+        uses: actions/checkout@v4
+      - name: Build with gcc
+        run: |
+          CC=gcc cmake -B gcc-out
+          cmake --build gcc-out
+      - name: Build with clang
+        run: |
+          CC=clang cmake -B clang-out
+          cmake --build clang-out

CMakeLists.txt

@@ -13,13 +13,37 @@ pkg_search_module(XKBCOMMON REQUIRED xkbcommon)
 pkg_search_module(LIBVA REQUIRED libva)
 pkg_search_module(LIBVA_DRM REQUIRED libva-drm)
 
+add_executable(kmsvnc)
+set(kmsvnc_SOURCES kmsvnc.c drm.c input.c keymap.c va.c drm_master.c)
+
 include(CheckIncludeFiles)
 CHECK_INCLUDE_FILES("linux/uinput.h;linux/dma-buf.h" HAVE_LINUX_API_HEADERS)
 IF(NOT HAVE_LINUX_API_HEADERS)
   message(FATAL_ERROR "linux-api-headers not found")
 ENDIF()
 
-add_executable(kmsvnc kmsvnc.c drm.c input.c keymap.c va.c)
+include(CheckSymbolExists)
+check_symbol_exists(SYS_pidfd_getfd "sys/syscall.h" HAVE_LIBC_SYS_pidfd_getfd)
+IF(NOT HAVE_LIBC_SYS_pidfd_getfd)
+  message(WARNING "pidfd_getfd syscall not found, the --screen-blank options will be disabled")
+  target_compile_options(kmsvnc PUBLIC -DDISABLE_KMSVNC_SCREEN_BLANK)
+  list(REMOVE_ITEM kmsvnc_SOURCES drm_master.c)
+ENDIF()
+include(CMakePushCheckState)
+cmake_push_check_state()
+set(CMAKE_REQUIRED_INCLUDES ${LIBDRM_INCLUDEDIR}/libdrm) # can't do anything about that
+set(CMAKE_REQUIRED_LIBRARIES ${LIBDRM_LIBRARIES})
+check_symbol_exists(drmGetFormatName "xf86drm.h" HAVE_LIBDRM_drmGetFormatName)
+cmake_pop_check_state()
+IF(NOT HAVE_LIBDRM_drmGetFormatName)
+  message(WARNING "drmGetFormatName not found, format name printing will be disabled")
+  target_compile_options(kmsvnc PUBLIC -DDISABLE_KMSVNC_drmGetFormatName)
+ENDIF()
+
+
+target_sources(kmsvnc PUBLIC
+    ${kmsvnc_SOURCES}
+)
 target_include_directories(kmsvnc PUBLIC
   ${LIBDRM_INCLUDEDIR}
   ${LIBDRM_INCLUDEDIR}/libdrm

drm.c

@@ -11,6 +11,25 @@
 #include "drm.h"
 #include "va.h"
 
+#ifndef DISABLE_KMSVNC_SCREEN_BLANK
+    #include "drm_master.h"
+#endif
+
+#ifndef fourcc_mod_is_vendor
+    #define fourcc_mod_is_vendor(modifier, vendor) \
+            (fourcc_mod_get_vendor(modifier) == DRM_FORMAT_MOD_VENDOR_## vendor)
+#endif
+#ifdef DISABLE_KMSVNC_drmGetFormatName
+    static char* drmGetFormatName(uint32_t data) {
+        char *name = "missing drmGetFormatName";
+        char *out = malloc(strlen(name)+1);
+        if (out) {
+            memcpy(out, name, strlen(name)+1);
+        }
+        return out;
+    }
+#endif
+
 extern struct kmsvnc_data *kmsvnc;
 
 static int check_pixfmt_non_vaapi() {
@@ -26,14 +45,18 @@ static int check_pixfmt_non_vaapi() {
 
 static void convert_copy(const char *in, int width, int height, char *buff)
 {
-    memcpy(buff, in, width * height * BYTES_PER_PIXEL);
+    if (likely(in != buff)) {
+        memcpy(buff, in, width * height * BYTES_PER_PIXEL);
+    }
 }
 
 static void convert_bgra_to_rgba(const char *in, int width, int height, char *buff)
 {
-    memcpy(buff, in, width * height * BYTES_PER_PIXEL);
+    if (likely(in != buff)) {
+        memcpy(buff, in, width * height * BYTES_PER_PIXEL);
+    }
     for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
-        uint32_t pixdata = htonl(*((uint32_t*)(kmsvnc->drm->kms_convert_buf + i)));
+        uint32_t pixdata = htonl(*((uint32_t*)(buff + i)));
         buff[i+0] = (pixdata & 0x0000ff00) >> 8;
         buff[i+2] = (pixdata & 0xff000000) >> 24;
     }
@@ -96,39 +119,45 @@ void convert_intel_x_tiled_kmsbuf(const char *in, int width, int height, char *b
 }
 
 static void convert_vaapi(const char *in, int width, int height, char *buff) {
-    if (KMSVNC_FOURCC_TO_INT('R','G','B', 0) & kmsvnc->va->image->format.fourcc == KMSVNC_FOURCC_TO_INT('R','G','B', 0)) {
+    va_hwframe_to_vaapi(buff);
-        va_hwframe_to_vaapi(buff);
+    if (
-    }
+        (KMSVNC_FOURCC_TO_INT('R','G','B',0) & kmsvnc->va->selected_fmt->fourcc) == KMSVNC_FOURCC_TO_INT('R','G','B',0)
+    ) {}
     else {
-        if (convert_buf_allocate(width * height * BYTES_PER_PIXEL)) return;
-        va_hwframe_to_vaapi(kmsvnc->drm->kms_convert_buf);
         // is 30 depth?
-        if (kmsvnc->va->image->format.depth == 30) {
+        if (kmsvnc->va->selected_fmt->depth == 30) {
             for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
                 // ensure little endianess
-                uint32_t pixdata = __builtin_bswap32(htonl(*((uint32_t*)(kmsvnc->drm->kms_convert_buf + i))));
+                uint32_t pixdata = __builtin_bswap32(htonl(*((uint32_t*)(buff + i))));
-                kmsvnc->drm->kms_convert_buf[i] = (pixdata & 0x3ff00000) >> 20 >> 2;
+                buff[i] = (pixdata & 0x3ff00000) >> 20 >> 2;
-                kmsvnc->drm->kms_convert_buf[i+1] = (pixdata & 0xffc00) >> 10 >> 2;
+                buff[i+1] = (pixdata & 0xffc00) >> 10 >> 2;
-                kmsvnc->drm->kms_convert_buf[i+2] = (pixdata & 0x3ff) >> 2;
+                buff[i+2] = (pixdata & 0x3ff) >> 2;
+            }
+        }
+        else {
+            // actually, does anyone use this?
+            if (!kmsvnc->va->selected_fmt->byte_order) {
+                for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
+                    uint32_t *pixdata = (uint32_t*)(buff + i);
+                    *pixdata = __builtin_bswap32(*pixdata);
+                }
             }
         }
         // is xrgb?
-        if ((kmsvnc->va->image->format.blue_mask | kmsvnc->va->image->format.red_mask) < 0x1000000) {
+        if ((kmsvnc->va->selected_fmt->blue_mask | kmsvnc->va->selected_fmt->red_mask) < 0x1000000) {
             for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
-                uint32_t *pixdata = (uint32_t*)(kmsvnc->drm->kms_convert_buf + i);
+                uint32_t *pixdata = (uint32_t*)(buff + i);
                 *pixdata = ntohl(htonl(*pixdata) << 8);
             }
         }
         // is bgrx?
-        if (kmsvnc->va->image->format.blue_mask > kmsvnc->va->image->format.red_mask) {
+        if (kmsvnc->va->selected_fmt->blue_mask > kmsvnc->va->selected_fmt->red_mask) {
             for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
-                uint32_t pixdata = htonl(*((uint32_t*)(kmsvnc->drm->kms_convert_buf + i)));
+                uint32_t pixdata = htonl(*((uint32_t*)(buff + i)));
                 buff[i+0] = (pixdata & 0x0000ff00) >> 8;
                 buff[i+2] = (pixdata & 0xff000000) >> 24;
             }
         }
-        // rgbx now
-        memcpy(buff, kmsvnc->drm->kms_convert_buf, width * height * BYTES_PER_PIXEL);
     }
 }
 
@@ -154,6 +183,19 @@ void drm_sync_noop(int drmfd)
 
 void drm_cleanup() {
     if (kmsvnc->drm) {
+#ifndef DISABLE_KMSVNC_SCREEN_BLANK
+        if (kmsvnc->drm->gamma && kmsvnc->drm->gamma->size && kmsvnc->drm->gamma->red && kmsvnc->drm->gamma->green && kmsvnc->drm->gamma->blue) {
+            if (drmModeCrtcSetGamma(kmsvnc->drm->drm_master_fd ?: kmsvnc->drm->drm_fd, kmsvnc->drm->plane->crtc_id, kmsvnc->drm->gamma->size, kmsvnc->drm->gamma->red, kmsvnc->drm->gamma->green, kmsvnc->drm->gamma->blue)) perror("Failed to restore gamma");
+        }
+        if (kmsvnc->drm->gamma && kmsvnc->drm->gamma->red) {
+            free(kmsvnc->drm->gamma->red);
+            kmsvnc->drm->gamma->red = kmsvnc->drm->gamma->green = kmsvnc->drm->gamma->blue = NULL;
+        }
+        if (kmsvnc->drm->gamma) {
+            free(kmsvnc->drm->gamma);
+            kmsvnc->drm->gamma = NULL;
+        }
+#endif
         if (kmsvnc->drm->drm_ver) {
             drmFreeVersion(kmsvnc->drm->drm_ver);
             kmsvnc->drm->drm_ver = NULL;
@@ -202,6 +244,10 @@ void drm_cleanup() {
             close(kmsvnc->drm->drm_fd);
             kmsvnc->drm->drm_fd = 0;
         }
+        if (kmsvnc->drm->drm_master_fd > 0) {
+            close(kmsvnc->drm->drm_master_fd);
+            kmsvnc->drm->drm_master_fd = 0;
+        }
         if (kmsvnc->drm->plane_res) {
             drmModeFreePlaneResources(kmsvnc->drm->plane_res);
             kmsvnc->drm->plane_res = NULL;
@@ -458,6 +504,19 @@ int drm_open() {
     {
         KMSVNC_FATAL("card %s open failed: %s\n", kmsvnc->card, strerror(errno));
     }
+    if (!kmsvnc->screen_blank && drmIsMaster(drm->drm_fd)) {
+        if (drmDropMaster(drm->drm_fd)) fprintf(stderr, "Failed to drop master");
+    }
+#ifndef DISABLE_KMSVNC_SCREEN_BLANK
+    if (kmsvnc->screen_blank && !drmIsMaster(drm->drm_fd)) {
+        drm->drm_master_fd = drm_get_master_fd();
+        drm->drm_master_fd = drm->drm_master_fd > 0 ? drm->drm_master_fd : 0;
+        if (kmsvnc->debug_enabled) {
+            fprintf(stderr, "not master client, master fd %d\n", drm->drm_master_fd);
+        }
+    }
+#endif
+
     drm->drm_ver = drmGetVersion(drm->drm_fd);
     printf("drm driver is %s\n", drm->drm_ver->name);
 
@@ -469,6 +528,68 @@ int drm_open() {
 
     if (drm_refresh_planes(1)) return 1;
 
+#ifndef DISABLE_KMSVNC_SCREEN_BLANK
+    if (kmsvnc->screen_blank) {
+        drm->gamma = malloc(sizeof(struct kmsvnc_drm_gamma_data));
+        if (!drm->gamma) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
+        memset(drm->gamma, 0, sizeof(struct kmsvnc_drm_gamma_data));
+        drmModeCrtc *target_crtc = drmModeGetCrtc(drm->drm_fd, drm->plane->crtc_id);
+        if (target_crtc) {
+            drm->gamma->size = (uint32_t)target_crtc->gamma_size;
+            drm->gamma->red = malloc(drm->gamma->size*sizeof(uint16_t)*3);
+            if (!drm->gamma->size) {
+                fprintf(stderr, "drm->gamma->size = %u, not setting gamma.\n", drm->gamma->size);
+            }
+            else if (!drm->gamma->red) {
+                fprintf(stderr, "memory allocation error at %s:%d\n", __FILE__, __LINE__);
+                fprintf(stderr, "not setting gamma.\n");
+            }
+            else {
+                memset(drm->gamma->red, 0, drm->gamma->size*sizeof(uint16_t)*3);
+                drm->gamma->green = drm->gamma->red + drm->gamma->size;
+                drm->gamma->blue = drm->gamma->red + drm->gamma->size*2;
+                if (kmsvnc->screen_blank_restore) {
+                    int step = 0x10000 / drm->gamma->size;
+                    for (int i = 0; i < drm->gamma->size; i++) {
+                        drm->gamma->red[i] = drm->gamma->green[i] = drm->gamma->blue[i] = step * i;
+                    }
+                }
+                else {
+                    // legacy api, but weston also uses this, so whatever
+                    drmModeCrtcGetGamma(drm->drm_fd, drm->plane->crtc_id, drm->gamma->size, drm->gamma->red, drm->gamma->green, drm->gamma->blue);
+                }
+                if (kmsvnc->debug_enabled) {
+                    for (int i = 0; i < drm->gamma->size; i++) {
+                        fprintf(stderr, "gamma: %05d %05hu %05hu %05hu\n", i, drm->gamma->red[i], drm->gamma->green[i], drm->gamma->blue[i]);
+                    }
+                }
+                uint16_t *new_gamma_red = malloc(drm->gamma->size*sizeof(uint16_t)*3);
+                if (!new_gamma_red) {
+                    fprintf(stderr, "memory allocation error at %s:%d\n", __FILE__, __LINE__);
+                    fprintf(stderr, "not setting gamma.\n");
+                }
+                else {
+                    memset(new_gamma_red, 0, drm->gamma->size*sizeof(uint16_t)*3);
+                    uint16_t *new_gamma_green = new_gamma_red + drm->gamma->size;
+                    uint16_t *new_gamma_blue = new_gamma_red + drm->gamma->size*2;
+                    if (drmModeCrtcSetGamma(drm->drm_master_fd ?: drm->drm_fd, drm->plane->crtc_id, drm->gamma->size, new_gamma_red, new_gamma_green, new_gamma_blue)) perror("Failed to set gamma");
+                }
+                if (new_gamma_red) {
+                    free(new_gamma_red);
+                    new_gamma_red = NULL;
+                }
+            }
+        }
+        else {
+            fprintf(stderr, "Did not get a crtc structure, not setting gamma.\n");
+        }
+        if (target_crtc) {
+            drmModeFreeCrtc(target_crtc);
+            target_crtc = NULL;
+        }
+    }
+#endif
+
     drm->mfb = drmModeGetFB2(drm->drm_fd, drm->plane->fb_id);
     if (!drm->mfb) {
         KMSVNC_FATAL("Failed to get framebuffer %u: %s\n", drm->plane->fb_id, strerror(errno));

drm.h

@@ -2,9 +2,9 @@
 
 #include "kmsvnc.h"
 
-#define DRM_IOCTL_MUST(...) do{ int e; if (e = drmIoctl(__VA_ARGS__)) KMSVNC_FATAL("DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
+#define DRM_IOCTL_MUST(...) do{ int e; if ((e = drmIoctl(__VA_ARGS__))) KMSVNC_FATAL("DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
-#define DRM_IOCTL_MAY(...) do{ int e; if (e = drmIoctl(__VA_ARGS__)) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
+#define DRM_IOCTL_MAY(...) do{ int e; if ((e = drmIoctl(__VA_ARGS__))) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
-#define DRM_R_IOCTL_MAY(...) do{ int e; if (e = ioctl(__VA_ARGS__)) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
+#define DRM_R_IOCTL_MAY(...) do{ int e; if ((e = ioctl(__VA_ARGS__))) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
 
 
 void drm_cleanup();

drm_master.c

@@ -0,0 +1,110 @@
+#define _GNU_SOURCE
+
+#include <stdio.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+#include <dirent.h>
+#include <stdlib.h>
+
+#include "drm_master.h"
+
+extern struct kmsvnc_data *kmsvnc;
+
+
+static inline int clone_fd(pid_t pid, int target_fd) {
+    int pidfd = syscall(SYS_pidfd_open, pid, 0);
+    if (pidfd <= 0) {
+        perror("pidfd_open");
+        return -1;
+    }
+    int cloned = syscall(SYS_pidfd_getfd, pidfd, target_fd, 0);
+    if (cloned <= 0) {
+        perror("pidfd_getfd");
+    }
+    close(pidfd);
+    return cloned;
+}
+
+static inline int cmp_fds(pid_t pid, const char *drm_pth) {
+    char path[PATH_MAX+1];
+    snprintf(path, PATH_MAX+1, "/proc/%d/fd", pid);
+
+    struct dirent **fdlist;
+    int count = scandir(path, &fdlist, NULL, versionsort);
+    int ret = -1;
+    if (count >= 0) {
+        for (int n = 0; n < count; n++) {
+            if (ret == -1 && fdlist[n]->d_type == DT_LNK) {
+                char link_pth[PATH_MAX+1];
+                char real_pth[PATH_MAX+1];
+                #pragma GCC diagnostic push
+                #pragma GCC diagnostic ignored "-Wpragmas"
+                #pragma GCC diagnostic ignored "-Wunknown-warning-option"
+                #pragma GCC diagnostic ignored "-Wformat-truncation"
+                snprintf(link_pth, PATH_MAX+1, "%s/%s", path, fdlist[n]->d_name);
+                #pragma GCC diagnostic pop
+                memset(real_pth, 0, PATH_MAX+1);
+                #pragma GCC diagnostic push
+                #pragma GCC diagnostic ignored "-Wunused-result"
+                realpath(link_pth, real_pth);
+                #pragma GCC diagnostic pop
+                if (!strncmp(real_pth, drm_pth, PATH_MAX)) {
+                    int fd = atoi(fdlist[n]->d_name);
+                    if (fd > 0) {
+                        int cloned = clone_fd(pid, fd);
+                        if (cloned > 0 && drmIsMaster(cloned)) {
+                            ret = cloned;
+                            if (kmsvnc->debug_enabled) {
+                                fprintf(stderr, "found drm master pid=%d, fd=%d, cloned=%d\n", pid, fd, cloned);
+                            }
+                        }
+                        else {
+                            if (cloned > 0) close(cloned);
+                        }
+                    }
+                }
+            }
+            free(fdlist[n]);
+            fdlist[n] = NULL;
+        }
+        free(fdlist);
+        fdlist = NULL;
+    }
+    return ret;
+}
+
+int drm_get_master_fd() {
+    char drm_pth[PATH_MAX+1];
+    memset(drm_pth, 0, PATH_MAX+1);
+    #pragma GCC diagnostic push
+    #pragma GCC diagnostic ignored "-Wunused-result"
+    realpath(kmsvnc->card, drm_pth);
+    #pragma GCC diagnostic pop
+
+    struct dirent **proclist;
+    int count = scandir("/proc", &proclist, NULL, versionsort);
+    int ret = -1;
+    if (count >= 0) {
+        for (int n = 0; n < count; n++) {
+            if (ret == -1 && proclist[n]->d_type == DT_DIR) {
+                pid_t pid = (pid_t)atoi(proclist[n]->d_name);
+                if (pid > 0) {
+                    int cloned = cmp_fds(pid, drm_pth);
+                    if (cloned > 0) {
+                        ret = cloned;
+                    }
+                }
+            }
+            free(proclist[n]);
+            proclist[n] = NULL;
+        }
+        free(proclist);
+        proclist = NULL;
+    }
+    else {
+        perror("open /proc");
+    }
+    return ret;
+}

drm_master.h

@@ -0,0 +1,5 @@
+#pragma once
+
+#include "kmsvnc.h"
+
+int drm_get_master_fd();

input.h

@@ -7,8 +7,8 @@
 #define UINPUT_ABS_MAX INT16_MAX
 #define UINPUT_MAX_KEY 256
 
-#define INP_IOCTL_MUST(...) do{ int e; if (e = ioctl(__VA_ARGS__)) KMSVNC_FATAL("uinput ioctl error %d on line %d\n", e, __LINE__); } while(0)
+#define INP_IOCTL_MUST(...) do{ int e; if ((e = ioctl(__VA_ARGS__))) KMSVNC_FATAL("uinput ioctl error %d on line %d\n", e, __LINE__); } while(0)
-#define INP_IOCTL_MAY(...) do{ int e; if (e = ioctl(__VA_ARGS__)) fprintf(stderr, "uinput ioctl error %d on line %d\n", e, __LINE__); } while(0)
+#define INP_IOCTL_MAY(...) do{ int e; if ((e = ioctl(__VA_ARGS__))) fprintf(stderr, "uinput ioctl error %d on line %d\n", e, __LINE__); } while(0)
 
 void uinput_cleanup();
 int uinput_init();

kmsvnc.c

@@ -1,9 +1,12 @@
+#define _GNU_SOURCE
+
 #include <fcntl.h>
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
 #include <signal.h>
 #include <time.h>
+#include <unistd.h>
 #include <argp.h>
 #include <arpa/inet.h>
 
@@ -127,7 +130,7 @@ static inline void update_vnc_cursor(char *data, int width, int height) {
         if (!kmsvnc->cursor_bitmap) return;
         kmsvnc->cursor_bitmap_len = rwidth * rheight * BYTES_PER_PIXEL;
     }
-    char *rich_source = malloc(rwidth * rheight * BYTES_PER_PIXEL);
+    unsigned char *rich_source = malloc(rwidth * rheight * BYTES_PER_PIXEL);
     if (!rich_source) return;
     char *maskString = malloc(rwidth * rheight);
     if (!maskString) {
@@ -208,6 +211,7 @@ static void cleanup() {
     }
 }
 
+void signal_handler_noop(int signum){}
 void signal_handler(int signum){
     if (kmsvnc->shutdown) {
         return;
@@ -219,7 +223,7 @@ void signal_handler(int signum){
 }
 
 static struct argp_option kmsvnc_main_options[] = {
-    {"device", 'd', "/dev/dri/card0", 0, "DRM device"},
+    {"device", 'd', "/dev/dri/cardX", 0, "DRM device"},
     {"source-plane", 0xfefc, "0", 0, "Use specific plane"},
     {"source-crtc", 0xfefd, "0", 0, "Use specific crtc (to list all crtcs and planes, set this to -1)"},
     {"force-driver", 0xfefe, "i915", 0, "force a certain driver (for debugging)"},
@@ -238,6 +242,11 @@ static struct argp_option kmsvnc_main_options[] = {
     {"input-height", 0xff07, "0", 0, "Explicitly set input height"},
     {"input-offx", 0xff08, "0", 0, "Set input offset of x axis on a multi display system"},
     {"input-offy", 0xff09, "0", 0, "Set input offset of y axis on a multi display system"},
+#ifndef DISABLE_KMSVNC_SCREEN_BLANK
+    {"screen-blank", 0xff0a, 0, OPTION_ARG_OPTIONAL, "Blank screen with gamma set on crtc"},
+    {"screen-blank-restore-linear", 0xff0b, 0, OPTION_ARG_OPTIONAL, "Restore linear values on exit in case of messed up gamma"},
+#endif
+    {"va-byteorder-swap", 0xff0c, 0, OPTION_ARG_OPTIONAL, "Force swap vaapi image rgb byteorder"},
     {"wakeup", 'w', 0, OPTION_ARG_OPTIONAL, "Move mouse to wake the system up before start"},
     {"disable-input", 'i', 0, OPTION_ARG_OPTIONAL, "Disable uinput"},
     {"desktop-name", 'n', "kmsvnc", 0, "Specify vnc desktop name"},
@@ -269,24 +278,28 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
             kmsvnc->vnc_opt->bind6 = arg;
             break;
         case 'p':
-            int port = atoi(arg);
+            {
-            if (port > 0 && port < 65536) {
+                int port = atoi(arg);
-                kmsvnc->vnc_opt->port = port;
+                if (port > 0 && port < 65536) {
-            }
+                    kmsvnc->vnc_opt->port = port;
-            else {
+                }
-                argp_error(state, "invalid port %s", arg);
+                else {
+                    argp_error(state, "invalid port %s", arg);
+                }
             }
             break;
         case '4':
             kmsvnc->vnc_opt->disable_ipv6 = 1;
             break;
         case 0xff00:
-            int fps = atoi(arg);
+            {
-            if (fps > 0 && fps < 1000) {
+                int fps = atoi(arg);
-                kmsvnc->vnc_opt->sleep_ns = NS_IN_S / fps;
+                if (fps > 0 && fps < 1000) {
-            }
+                    kmsvnc->vnc_opt->sleep_ns = NS_IN_S / fps;
-            else {
+                }
-                argp_error(state, "invalid fps %s", arg);
+                else {
+                    argp_error(state, "invalid fps %s", arg);
+                }
             }
             break;
         case 0xff01:
@@ -314,29 +327,46 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
             kmsvnc->debug_enabled = 1;
             break;
         case 0xff06:
-            int width = atoi(arg);
+            {
-            if (width > 0) {
+                int width = atoi(arg);
-                kmsvnc->input_width = width;
+                if (width > 0) {
+                    kmsvnc->input_width = width;
+                }
             }
             break;
         case 0xff07:
-            int height = atoi(arg);
+            {
-            if (height > 0) {
+                int height = atoi(arg);
-                kmsvnc->input_height = height;
+                if (height > 0) {
+                    kmsvnc->input_height = height;
+                }
             }
             break;
         case 0xff08:
-            int offset_x = atoi(arg);
+            {
-            if (offset_x > 0) {
+                int offset_x = atoi(arg);
-                kmsvnc->input_offx = offset_x;
+                if (offset_x > 0) {
+                    kmsvnc->input_offx = offset_x;
+                }
             }
             break;
         case 0xff09:
-            int offset_y = atoi(arg);
+            {
-            if (offset_y > 0) {
+                int offset_y = atoi(arg);
-                kmsvnc->input_offy = offset_y;
+                if (offset_y > 0) {
+                    kmsvnc->input_offy = offset_y;
+                }
             }
             break;
+        case 0xff0a:
+            kmsvnc->screen_blank = 1;
+            break;
+        case 0xff0b:
+            kmsvnc->screen_blank_restore = 1;
+            break;
+        case 0xff0c:
+            kmsvnc->va_byteorder_swap = 1;
+            break;
         case 'w':
             kmsvnc->input_wakeup = 1;
             break;
@@ -369,7 +399,10 @@ int main(int argc, char **argv)
 
     kmsvnc->vnc_opt = vncopt;
 
-    kmsvnc->card = "/dev/dri/card0";
+    #define DEVICE_EXAMPLE_MAX_SIZE 15
+    #define DEVICE_EXAMPLE_FALLBACK "/dev/dri/card0"
+    static char device_example[DEVICE_EXAMPLE_MAX_SIZE] = DEVICE_EXAMPLE_FALLBACK;
+    kmsvnc->card = device_example;
     kmsvnc->va_derive_enabled = -1;
     kmsvnc->vnc_opt->bind = &(struct in_addr){0};
     kmsvnc->vnc_opt->always_shared = 1;
@@ -383,6 +416,18 @@ int main(int argc, char **argv)
     struct argp argp = {kmsvnc_main_options, parse_opt, args_doc, doc};
     argp_parse(&argp, argc, argv, 0, 0, NULL);
 
+    if (kmsvnc->card == device_example) {
+        for (int i = 0; i < 10; i++) {
+            snprintf(kmsvnc->card, DEVICE_EXAMPLE_MAX_SIZE, "/dev/dri/card%d", i);
+            if (!access(kmsvnc->card, F_OK)) {
+                break;
+            }
+            else {
+                snprintf(kmsvnc->card, DEVICE_EXAMPLE_MAX_SIZE, DEVICE_EXAMPLE_FALLBACK);
+            }
+        }
+    }
+
     if (!kmsvnc->disable_input) {
         const char* XKB_DEFAULT_LAYOUT = getenv("XKB_DEFAULT_LAYOUT");
         if (!XKB_DEFAULT_LAYOUT || strcmp(XKB_DEFAULT_LAYOUT, "") == 0) {
@@ -429,6 +474,20 @@ int main(int argc, char **argv)
         else {
             fprintf(stderr, "open file %s failed, %s\n", kmsvnc->debug_capture_fb, strerror(errno));
         }
+        if (kmsvnc->screen_blank) {
+            sigset_t signal_set;
+            int sig;
+            sigemptyset(&signal_set);
+            signal(SIGHUP, &signal_handler_noop);
+            signal(SIGINT, &signal_handler_noop);
+            signal(SIGTERM, &signal_handler_noop);
+            sigaddset(&signal_set, SIGHUP);
+            sigaddset(&signal_set, SIGINT);
+            sigaddset(&signal_set, SIGTERM);
+            fprintf(stderr, "blanking screen...\n");
+            sigwait(&signal_set, &sig);
+            fprintf(stderr, "got sig %d\n", sig);
+        }
         cleanup();
         return 0;
     }

kmsvnc.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <rfb/rfb.h>
+#include <stdint.h>
 #include <xkbcommon/xkbcommon.h>
 
 #include <xf86drm.h>
@@ -42,6 +43,9 @@ struct kmsvnc_data
     int input_height;
     int input_offx;
     int input_offy;
+    char screen_blank;
+    char screen_blank_restore;
+    char va_byteorder_swap;
     struct kmsvnc_drm_data *drm;
     struct kmsvnc_input_data *input;
     struct kmsvnc_keymap_data *keymap;
@@ -85,9 +89,18 @@ struct kmsvnc_drm_funcs
     void (*convert)(const char *, int, int, char *);
 };
 
+struct kmsvnc_drm_gamma_data
+{
+    uint32_t size;
+    uint16_t *red;
+    uint16_t *green;
+    uint16_t *blue;
+};
+
 struct kmsvnc_drm_data
 {
     int drm_fd;
+    int drm_master_fd;
     drmVersionPtr drm_ver;
     int prime_fd;
     drmModePlane *plane;
@@ -95,7 +108,7 @@ struct kmsvnc_drm_data
     drmModePlaneRes *plane_res;
     drmModeFB2 *mfb;
     drmModeFB2 *cursor_mfb;
-    u_int32_t plane_id;
+    uint32_t plane_id;
     int mmap_fd;
     size_t mmap_size;
     off_t mmap_offset;
@@ -113,6 +126,7 @@ struct kmsvnc_drm_data
     size_t kms_cpy_tmp_buf_len;
     char *kms_cursor_buf;
     size_t kms_cursor_buf_len;
+    struct kmsvnc_drm_gamma_data *gamma;
 };
 
 struct kmsvnc_va_data
@@ -125,6 +139,8 @@ struct kmsvnc_va_data
     char derive_enabled;
     VAImageFormat* img_fmts;
     int img_fmt_count;
+    VAImageFormat* selected_fmt;
+    const char *vendor_string;
 };
 
 #define KMSVNC_FATAL(...) do{ fprintf(stderr, __VA_ARGS__); return 1; } while(0)
@@ -133,3 +149,6 @@ struct kmsvnc_va_data
 #define KMSVNC_WRITE_MAY(fd,buf,count) do { ssize_t e = write((fd), (buf), (count)); if (e != (count)) fprintf(stderr, "should write %ld bytes, actually wrote %ld, on line %d\n", (count), e, __LINE__); } while (0)
 
 #define KMSVNC_DEBUG(...) do{ if (kmsvnc->debug_enabled) fprintf(stdout, __VA_ARGS__); } while(0)
+
+#define likely(x)   __builtin_expect(!!(x), 1)
+#define unlikely(x) __builtin_expect(!!(x), 0)

va.c

@@ -1,6 +1,12 @@
+#define _GNU_SOURCE
+
+#include <stdint.h>
+#include <stdio.h>
+#include <va/va.h>
 #include <va/va_drm.h>
 #include <va/va_drmcommon.h>
 #include <fcntl.h>
+#include <string.h>
 
 #include "va.h"
 #include "kmsvnc.h"
@@ -33,6 +39,9 @@ void va_cleanup() {
             VA_MAY(vaTerminate(kmsvnc->va->dpy));
             kmsvnc->va->dpy = NULL;
         }
+        if (kmsvnc->va->vendor_string) {
+            kmsvnc->va->vendor_string = NULL;
+        }
         free(kmsvnc->va);
         kmsvnc->va = NULL;
     }
@@ -69,20 +78,41 @@ static const struct {
     {KMSVNC_FOURCC_TO_INT('A', 'R', '3', '0'), KMSVNC_FOURCC_TO_INT('A', 'R', '3', '0'), VA_RT_FORMAT_RGB32_10, 1},
 };
 
+struct va_fmt_data {
+    uint32_t va_fourcc;
+    VAImageFormat *fmt;
+    char is_alpha;
+    uint32_t va_rt_format;
+    uint32_t depth;
+};
+
+static VAImageFormat* vaImgFmt_apply_quirks(struct va_fmt_data* data) {
+    static VAImageFormat ret = {0};
+    memcpy(&ret, data->fmt, sizeof(VAImageFormat));
+    if ((kmsvnc->va_byteorder_swap ^ !strncmp(kmsvnc->va->vendor_string, "Mesa", 4)) && data->depth != 30) {
+        printf("applying rgb mask byte order swap\n");
+        ret.blue_mask = __builtin_bswap32(data->fmt->blue_mask);
+        ret.green_mask = __builtin_bswap32(data->fmt->green_mask);
+        ret.red_mask = __builtin_bswap32(data->fmt->red_mask);
+    }
+    return &ret;
+}
+
 static void print_va_image_fmt(VAImageFormat *fmt) {
-        printf("image fmt: fourcc %d, %s, byte_order %s, bpp %d, depth %d, blue_mask %#x, green_mask %#x, red_mask %#x, reserved %#x %#x %#x %#x\n", fmt->fourcc,
+    printf("image fmt: fourcc %d, %s, byte_order %s, bpp %d, depth %d, blue_mask %#x, green_mask %#x, red_mask %#x, alpha_mask %#x, reserved %#x %#x %#x %#x\n", fmt->fourcc,
-            fourcc_to_str(fmt->fourcc),
+        fourcc_to_str(fmt->fourcc),
-            fmt->byte_order == 1 ? "VA_LSB_FIRST" : "VA_MSB_FIRST",
+        fmt->byte_order == 1 ? "VA_LSB_FIRST" : "VA_MSB_FIRST",
-            fmt->bits_per_pixel,
+        fmt->bits_per_pixel,
-            fmt->depth,
+        fmt->depth,
-            fmt->blue_mask,
+        fmt->blue_mask,
-            fmt->green_mask,
+        fmt->green_mask,
-            fmt->red_mask,
+        fmt->red_mask,
-            fmt->va_reserved[0],
+        fmt->alpha_mask,
-            fmt->va_reserved[1],
+        fmt->va_reserved[0],
-            fmt->va_reserved[2],
+        fmt->va_reserved[1],
-            fmt->va_reserved[3]
+        fmt->va_reserved[2],
-        );
+        fmt->va_reserved[3]
+    );
 }
 
 int va_init() {
@@ -91,6 +121,7 @@ int va_init() {
     }
 
     setenv("DISPLAY", "", 1);
+    setenv("WAYLAND_DISPLAY", "", 1);
 
     struct kmsvnc_va_data *va = malloc(sizeof(struct kmsvnc_va_data));
     if (!va) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
@@ -99,7 +130,7 @@ int va_init() {
 
     char* render_node;
     int effective_fd = 0;
-    if (render_node = drmGetRenderDeviceNameFromFd(kmsvnc->drm->drm_fd)) {
+    if ((render_node = drmGetRenderDeviceNameFromFd(kmsvnc->drm->drm_fd))) {
         va->render_node_fd = open(render_node, O_RDWR);
         free(render_node);
     }
@@ -126,8 +157,8 @@ int va_init() {
     VAStatus status;
     VA_MUST(vaInitialize(va->dpy, &major, &minor));
 
-    const char *vendor_string = vaQueryVendorString(va->dpy);
+    va->vendor_string = vaQueryVendorString(va->dpy);
-    printf("vaapi vendor %s\n", vendor_string);
+    printf("vaapi vendor %s\n", va->vendor_string);
 
     VADRMPRIMESurfaceDescriptor prime_desc;
     VASurfaceAttrib prime_attrs[2] = {
@@ -255,28 +286,26 @@ int va_init() {
         }
     }
 
-    struct fourcc_data {
+    struct va_fmt_data format_to_try[] = {
-        uint32_t va_fourcc;
+        {KMSVNC_FOURCC_TO_INT('R','G','B','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24},
-        VAImageFormat *fmt;
+        {KMSVNC_FOURCC_TO_INT('R','G','B','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32},
-        char is_alpha;
-        uint32_t va_rt_format;
-    };
-    struct fourcc_data format_to_try[] = {
-        {KMSVNC_FOURCC_TO_INT('R','G','B','X'), NULL, 0, VA_RT_FORMAT_RGB32},
-        {KMSVNC_FOURCC_TO_INT('R','G','B','A'), NULL, 1, VA_RT_FORMAT_RGB32},
-        {KMSVNC_FOURCC_TO_INT('X','R','G','B'), NULL, 0, VA_RT_FORMAT_RGB32},
-        {KMSVNC_FOURCC_TO_INT('A','R','G','B'), NULL, 1, VA_RT_FORMAT_RGB32},
 
-        {KMSVNC_FOURCC_TO_INT('B','G','R','X'), NULL, 0, VA_RT_FORMAT_RGB32},
+        {KMSVNC_FOURCC_TO_INT('X','B','G','R'), NULL, 0, VA_RT_FORMAT_RGB32, 24},
-        {KMSVNC_FOURCC_TO_INT('B','G','R','A'), NULL, 1, VA_RT_FORMAT_RGB32},
+        {KMSVNC_FOURCC_TO_INT('A','B','G','R'), NULL, 1, VA_RT_FORMAT_RGB32, 32},
-        {KMSVNC_FOURCC_TO_INT('X','B','G','R'), NULL, 0, VA_RT_FORMAT_RGB32},
-        {KMSVNC_FOURCC_TO_INT('A','B','G','R'), NULL, 1, VA_RT_FORMAT_RGB32},
 
-        {KMSVNC_FOURCC_TO_INT('X','R','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10},
+        {KMSVNC_FOURCC_TO_INT('X','R','G','B'), NULL, 0, VA_RT_FORMAT_RGB32, 24},
-        {KMSVNC_FOURCC_TO_INT('A','R','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10},
+        {KMSVNC_FOURCC_TO_INT('A','R','G','B'), NULL, 1, VA_RT_FORMAT_RGB32, 32},
-        {KMSVNC_FOURCC_TO_INT('X','B','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10},
+
-        {KMSVNC_FOURCC_TO_INT('A','B','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10},
+        {KMSVNC_FOURCC_TO_INT('B','G','R','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24},
+        {KMSVNC_FOURCC_TO_INT('B','G','R','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32},
+
+
+        {KMSVNC_FOURCC_TO_INT('X','R','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30},
+        {KMSVNC_FOURCC_TO_INT('A','R','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30},
+        {KMSVNC_FOURCC_TO_INT('X','B','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30},
+        {KMSVNC_FOURCC_TO_INT('A','B','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30},
     };
+
     for (int i = 0; i < va->img_fmt_count; i++) {
         for (int j = 0; j < KMSVNC_ARRAY_ELEMENTS(format_to_try); j++) {
             if (va->img_fmts[i].fourcc == format_to_try[j].va_fourcc) {
@@ -292,14 +321,14 @@ int va_init() {
     va->derive_enabled = kmsvnc->va_derive_enabled < 0 ? va->derive_enabled : kmsvnc->va_derive_enabled != 0;
     if (va->derive_enabled) {
         if ((s = vaDeriveImage(va->dpy, va->surface_id, va->image)) == VA_STATUS_SUCCESS) {
-            char found = 0;
             for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(format_to_try); i++) {
+                if (format_to_try[i].fmt == NULL) continue;
                 if (va->image->format.fourcc == format_to_try[i].fmt->fourcc) {
-                    found = 1;
+                    va->selected_fmt = vaImgFmt_apply_quirks(format_to_try + i);
                     break;
                 }
             }
-            if (!found) {
+            if (!va->selected_fmt) {
                 va->derive_enabled = 0;
                 printf("vaDeriveImage returned unknown fourcc %d %s\n", va->image->format.fourcc, fourcc_to_str(va->image->format.fourcc));
                 VA_MAY(vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id));
@@ -315,7 +344,6 @@ int va_init() {
         }
     }
     if (!va->derive_enabled) {
-        char success = 0;
         for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(format_to_try); i++) {
             if (format_to_try[i].fmt == NULL) continue;
             if (!kmsvnc->debug_enabled && rt_format != format_to_try[i].va_rt_format) continue;
@@ -341,17 +369,21 @@ int va_init() {
                 continue;
             }
             else {
-                success = 1;
+                va->selected_fmt = vaImgFmt_apply_quirks(format_to_try + i);
                 break;
             }
         }
-        if (!success) {
+        if (!va->selected_fmt) {
             va->imgbuf = NULL;
             KMSVNC_FATAL("failed to get vaapi image\n");
         }
     }
     printf("got vaapi %simage:\n", va->derive_enabled ? "derive " : "");
     print_va_image_fmt(&va->image->format);
+    if (kmsvnc->debug_enabled) {
+        fprintf(stderr, "selected image format:\n");
+        print_va_image_fmt(va->selected_fmt);
+    }
     return 0;
 }