kmsvnc/va.c

#include <va/va_drm.h>
#include <va/va_drmcommon.h>
#include <fcntl.h>

#include "va.h"
#include "kmsvnc.h"

extern struct kmsvnc_data *kmsvnc;

void va_cleanup () {

}

static void va_msg_callback(void *user_context, const char *message) {
    #ifdef KMSVNC_VA_DEBUG
    printf("va msg: %s");
    #endif
}

static void va_error_callback(void *user_context, const char *message) {
    printf("va error: %s");
}

int va_init() {
    if (!kmsvnc->drm || !kmsvnc->drm->drm_fd || !kmsvnc->drm->prime_fd) {
        KMSVNC_FATAL("drm is not initialized\n");
    }

    setenv("DISPLAY", "", 1);

    struct kmsvnc_va_data *va = malloc(sizeof(struct kmsvnc_va_data));
    memset(va, 0, sizeof(struct kmsvnc_va_data));
    kmsvnc->va = va;

    char* render_node;
    int drm_fd = 0;
    if (render_node = drmGetRenderDeviceNameFromFd(kmsvnc->drm->drm_fd)) {
        drm_fd = open(render_node, O_RDWR);
    }
    else {
        printf("Using non-render node because the device does not have an associated render node.\n");
    }
    if (drm_fd <= 0) {
        printf("Using non-render node because render node fails to open.\n");
        drm_fd = kmsvnc->drm->drm_fd;
    }

    va->dpy = vaGetDisplayDRM(drm_fd);
    if (!va->dpy) {
        KMSVNC_FATAL("vaGetDisplayDRM failed\n");
    }

    vaSetErrorCallback(va->dpy, &va_error_callback, NULL);
    vaSetInfoCallback(va->dpy, &va_msg_callback, NULL);

    int major, minor;
    VAStatus status;
    VA_CHECK(vaInitialize(va->dpy, &major, &minor));

    const char *vendor_string = vaQueryVendorString(va->dpy);
    printf("vaapi vendor %s\n", vendor_string);

    VADRMPRIMESurfaceDescriptor prime_desc;
    VASurfaceAttrib prime_attrs[2] = {
        {
            .type  = VASurfaceAttribMemoryType,
            .flags = VA_SURFACE_ATTRIB_SETTABLE,
            .value.type    = VAGenericValueTypeInteger,
            .value.value.i = VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2,
        },
        {
            .type  = VASurfaceAttribExternalBufferDescriptor,
            .flags = VA_SURFACE_ATTRIB_SETTABLE,
            .value.type    = VAGenericValueTypePointer,
            .value.value.p = &prime_desc,
        }
    };

    prime_desc.fourcc = kmsvnc->drm->mfb->pixel_format == KMSVNC_FOURCC_TO_INT('X', 'R', '2', '4') ?
        KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'X') :
        KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'A') ;
    prime_desc.width = kmsvnc->drm->mfb->width;
    prime_desc.height = kmsvnc->drm->mfb->height;

    int i;
    int max_size = 0;
    for (i = 0; i < 4; i++) {
        int size = kmsvnc->drm->mfb->offsets[i] + kmsvnc->drm->mfb->height * kmsvnc->drm->mfb->pitches[i];
        if (size > max_size) max_size = size;
    }
    for (i = 0; i < 4; i++) {
        prime_desc.objects[i].fd = kmsvnc->drm->prime_fd;
        prime_desc.objects[i].size = max_size;
        prime_desc.objects[i].drm_format_modifier = kmsvnc->drm->mfb->modifier;
    }

    prime_desc.num_layers = 1;
    prime_desc.layers[0].drm_format = kmsvnc->drm->mfb->pixel_format;
    for (i = 0; i < 4; i++) {
        prime_desc.layers[0].object_index[i] = 0;
        prime_desc.layers[0].offset[i] = kmsvnc->drm->mfb->offsets[i];
        prime_desc.layers[0].pitch[i] = kmsvnc->drm->mfb->pitches[i];
    }
    for (i = 0; i < 4; i++) {
        if (!kmsvnc->drm->mfb->handles[i]) {
            break;
        }
    }
    prime_desc.layers[0].num_planes = i;
    prime_desc.num_objects = 1;

    VAStatus s;
    if (s = vaCreateSurfaces(va->dpy, VA_RT_FORMAT_RGB32,
                            kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, &va->surface_id, 1,
                            prime_attrs, KMSVNC_ARRAY_ELEMENTS(prime_attrs)))
    {
        printf("vaCreateSurfaces prime2 error %#x %s, trying prime\n", s, vaErrorStr(s));
        
        VASurfaceAttribExternalBuffers buffer_desc;
        VASurfaceAttrib buffer_attrs[2] = {
            {
                .type  = VASurfaceAttribMemoryType,
                .flags = VA_SURFACE_ATTRIB_SETTABLE,
                .value.type    = VAGenericValueTypeInteger,
                .value.value.i = VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME,
            },
            {
                .type  = VASurfaceAttribExternalBufferDescriptor,
                .flags = VA_SURFACE_ATTRIB_SETTABLE,
                .value.type    = VAGenericValueTypePointer,
                .value.value.p = &buffer_desc,
            }
        };

        unsigned long fd = kmsvnc->drm->prime_fd;

        buffer_desc.pixel_format = prime_desc.fourcc;
        buffer_desc.width        = kmsvnc->drm->mfb->width;
        buffer_desc.height       = kmsvnc->drm->mfb->height;
        buffer_desc.data_size    = max_size;
        buffer_desc.buffers      = &fd;
        buffer_desc.num_buffers  = 1;
        buffer_desc.flags        = 0;

        for (i = 0; i < 4; i++) {
            buffer_desc.pitches[i] = kmsvnc->drm->mfb->pitches[i];
            buffer_desc.offsets[i] = kmsvnc->drm->mfb->offsets[i];
        }
        buffer_desc.num_planes = prime_desc.layers[0].num_planes;


        VA_CHECK(vaCreateSurfaces(va->dpy, VA_RT_FORMAT_RGB32,
                kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, &va->surface_id, 1,
                buffer_attrs, KMSVNC_ARRAY_ELEMENTS(buffer_attrs)));
    }

    #ifdef KMSVNC_VA_DEBUG
    int img_fmt_count = vaMaxNumImageFormats(va->dpy);
    VAImageFormat *img_fmts = malloc(sizeof(VAImageFormat) * img_fmt_count);
    {
        int got;
        vaQueryImageFormats(va->dpy, img_fmts, &got);
        if (got != img_fmt_count) {
            KMSVNC_FATAL("got less VAImageFormats, %d instead of %d\n", got, img_fmt_count);
        }
    }

    for (int i = 0; i < img_fmt_count; i++) {
        printf("fmt %d: fourcc %d, %c%c%c%c, byte_order %s, bpp %d, depth %d, blue_mask %#x, green_mask %#x, red_mask %#x, reserved %#x\n", i, img_fmts[i].fourcc,
            img_fmts[i].fourcc & 0xff,
            img_fmts[i].fourcc >> 8 & 0xff,
            img_fmts[i].fourcc >> 16 & 0xff,
            img_fmts[i].fourcc >> 24 & 0xff,
            img_fmts[i].byte_order - 1 ? "VA_LSB_FIRST" : "VA_MSB_FIRST",
            img_fmts[i].bits_per_pixel,
            img_fmts[i].depth,
            img_fmts[i].blue_mask,
            img_fmts[i].green_mask,
            img_fmts[i].red_mask,
            img_fmts[i].va_reserved
        );
    }
    #endif

    VAImageFormat format = {
        .fourcc = KMSVNC_FOURCC_TO_INT('R','G','B','X'),
        .byte_order = VA_LSB_FIRST,
        .bits_per_pixel = 32,
        .depth = 24,
        .blue_mask = 0x000000ff,
        .green_mask = 0x0000ff00,
        .red_mask = 0x00ff0000,
        .va_reserved = 0x00000000,
    };
    va->image = malloc(sizeof(VAImage));
    VA_CHECK(vaCreateImage(va->dpy, &format, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, va->image));
    //VA_CHECK(vaDeriveImage(va->dpy, va->surface_id, va->image));
    va->imgbuf = malloc(va->image->data_size);
    VA_CHECK(vaMapBuffer(va->dpy, va->image->buf, (void**)&va->imgbuf));
}

int va_hwframe_to_vaapi() {
    //VA_CHECK(vaSyncSurface(kmsvnc->va->dpy, kmsvnc->va->surface_id));
    VA_CHECK(vaGetImage(kmsvnc->va->dpy, kmsvnc->va->surface_id, 0, 0,
                        kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, kmsvnc->va->image->image_id));
    memcpy(kmsvnc->buf1, kmsvnc->va->imgbuf, kmsvnc->drm->mfb->width * kmsvnc->drm->mfb->height * BYTES_PER_PIXEL);
    //printf("image data size: %d\n", kmsvnc->va->image->data_size);
}