switch to gitea actions

and badge

.forgejo/workflows/default.yml

@@ -0,0 +1,27 @@
+name: Build with gcc + clang
+on:
+  push:
+    branches: [dev]
+jobs:
+  build:
+    if: "github.event_name != 'push' || !contains(github.event.head_commit.message, '[skip ci]')"
+    runs-on: docker
+    container:
+      image: archlinux:latest
+    env:
+      CFLAGS: "-pipe -fno-plt -fexceptions -fstack-clash-protection -fcf-protection -Wp,-D_FORTIFY_SOURCE=2 -Wformat -Werror=format-security"
+    steps:
+      - name: Prepare dependencies
+        run: |
+          pacman -Syu --noconfirm --needed nodejs git \
+            base-devel libvncserver libxkbcommon libdrm libva cmake clang
+      - name: Check out repository code
+        uses: actions/checkout@v4
+      - name: Build with gcc
+        run: |
+          CC=gcc cmake -B gcc-out
+          cmake --build gcc-out
+      - name: Build with clang
+        run: |
+          CC=clang cmake -B clang-out
+          cmake --build clang-out

CMakeLists.txt

@@ -10,6 +10,11 @@ find_package(PkgConfig REQUIRED)
 pkg_search_module(LIBDRM REQUIRED libdrm)
 pkg_search_module(LIBVNCSERVER REQUIRED libvncserver)
 pkg_search_module(XKBCOMMON REQUIRED xkbcommon)
+pkg_search_module(LIBVA REQUIRED libva)
+pkg_search_module(LIBVA_DRM REQUIRED libva-drm)
+
+add_executable(kmsvnc)
+set(kmsvnc_SOURCES kmsvnc.c drm.c input.c keymap.c va.c drm_master.c)
 
 include(CheckIncludeFiles)
 CHECK_INCLUDE_FILES("linux/uinput.h;linux/dma-buf.h" HAVE_LINUX_API_HEADERS)
@@ -17,17 +22,42 @@ IF(NOT HAVE_LINUX_API_HEADERS)
   message(FATAL_ERROR "linux-api-headers not found")
 ENDIF()
 
-add_executable(kmsvnc kmsvnc.c drm.c input.c keymap.c)
+include(CheckSymbolExists)
+check_symbol_exists(SYS_pidfd_getfd "sys/syscall.h" HAVE_LIBC_SYS_pidfd_getfd)
+IF(NOT HAVE_LIBC_SYS_pidfd_getfd)
+  message(WARNING "pidfd_getfd syscall not found, the --screen-blank options will be disabled")
+  target_compile_options(kmsvnc PUBLIC -DDISABLE_KMSVNC_SCREEN_BLANK)
+  list(REMOVE_ITEM kmsvnc_SOURCES drm_master.c)
+ENDIF()
+include(CMakePushCheckState)
+cmake_push_check_state()
+set(CMAKE_REQUIRED_INCLUDES ${LIBDRM_INCLUDEDIR}/libdrm) # can't do anything about that
+set(CMAKE_REQUIRED_LIBRARIES ${LIBDRM_LIBRARIES})
+check_symbol_exists(drmGetFormatName "xf86drm.h" HAVE_LIBDRM_drmGetFormatName)
+cmake_pop_check_state()
+IF(NOT HAVE_LIBDRM_drmGetFormatName)
+  message(WARNING "drmGetFormatName not found, format name printing will be disabled")
+  target_compile_options(kmsvnc PUBLIC -DDISABLE_KMSVNC_drmGetFormatName)
+ENDIF()
+
+
+target_sources(kmsvnc PUBLIC
+    ${kmsvnc_SOURCES}
+)
 target_include_directories(kmsvnc PUBLIC
   ${LIBDRM_INCLUDEDIR}
   ${LIBDRM_INCLUDEDIR}/libdrm
   ${LIBVNCSERVER_INCLUDEDIR}
   ${XKBCOMMON_INCLUDEDIR}
+  ${LIBVA_INCLUDEDIR}
+  ${LIBVA_DRM_INCLUDEDIR}
 )
 target_link_libraries(kmsvnc PUBLIC
   m
   ${LIBDRM_LIBRARIES}
   ${LIBVNCSERVER_LIBRARIES}
   ${XKBCOMMON_LIBRARIES}
+  ${LIBVA_LIBRARIES}
+  ${LIBVA_DRM_LIBRARIES}
 )
 install(TARGETS kmsvnc RUNTIME DESTINATION bin)

LICENSE

@@ -0,0 +1,674 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.  We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors.  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights.  Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received.  You must make sure that they, too, receive
+or can get the source code.  And you must show them these terms so they
+know their rights.
+
+  Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+  For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software.  For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+  Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so.  This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software.  The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable.  Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products.  If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+  Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary.  To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Use with the GNU Affero General Public License.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    {one line to give the program's name and a brief idea of what it does.}
+    Copyright (C) {year}  {name of author}
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+    {project}  Copyright (C) {year}  {fullname}
+    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+<http://www.gnu.org/licenses/>.
+
+  The GNU General Public License does not permit incorporating your program
+into proprietary programs.  If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.  But first, please read
+<http://www.gnu.org/philosophy/why-not-lgpl.html>.

README.md

@@ -1,15 +1,35 @@
 # kmsvnc
 
+[![Build Status](https://drone.jerryxiao.com/api/badges/Jerry/kmsvnc/status.svg)](https://drone.jerryxiao.com/Jerry/kmsvnc)
+
 ## Introduction
 A VNC server for DRM/KMS capable GNU/Linux devices.  
-The goal is to simply have a universally working vncserver on X, wayland and even something like kmscon.  
+The goal is to simply have a universally working vncserver on X, wayland and even something like your linux VT.  
-Currently in very early stage.
+Currently in very early development stage.
+
+## Notes
+Intel made a great thing called CCS (Color Control Surface), however that won't work with kmsvnc. Please set `INTEL_DEBUG=noccs` globally, ideally in /etc/systemd/system.conf.d. Manpage is at `man 5 systemd-system.conf`. For example:
+```
+# /etc/systemd/system.conf.d/intel-no-ccs.conf 
+[Manager]
+DefaultEnvironment=INTEL_DEBUG=noccs
+```
+NixOS:
+```
+systemd.extraConfig = ''
+  DefaultEnvironment=INTEL_DEBUG=noccs
+''
+```
+
+If you plan to use the default vaapi driver for Intel and AMD GPUs, please make sure your vaapi configuration is working.  
+Nvidia support is highly experimental (nvidia-legacy with drm enabled or nvidia-open). Only one X-TILED modifier is supported as of now.
 
 ## Dependencies
  * cmake
  * libvncserver
  * libxkbcommon
  * libdrm
+ * libva
 
 ## Building
 ```

drm.c

@@ -1,36 +1,78 @@
 #include <stdio.h>
+#include <assert.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <sys/mman.h>
 #include <string.h>
 #include <stdlib.h>
 #include <sys/ioctl.h>
+#include <libdrm/drm_fourcc.h>
 
 #include "drm.h"
+#include "va.h"
+
+#ifndef DISABLE_KMSVNC_SCREEN_BLANK
+    #include "drm_master.h"
+#endif
+
+#ifndef fourcc_mod_is_vendor
+    #define fourcc_mod_is_vendor(modifier, vendor) \
+            (fourcc_mod_get_vendor(modifier) == DRM_FORMAT_MOD_VENDOR_## vendor)
+#endif
+#ifdef DISABLE_KMSVNC_drmGetFormatName
+    static char* drmGetFormatName(uint32_t data) {
+        char *name = "missing drmGetFormatName";
+        char *out = malloc(strlen(name)+1);
+        if (out) {
+            memcpy(out, name, strlen(name)+1);
+        }
+        return out;
+    }
+#endif
 
 extern struct kmsvnc_data *kmsvnc;
 
-static void convert_copy(const char *in, int width, int height, char *buff) {
+static int check_pixfmt_non_vaapi() {
-    memcpy(buff, in, width * height * 4);
+    if (
+        kmsvnc->drm->mfb->pixel_format != KMSVNC_FOURCC_TO_INT('X', 'R', '2', '4') &&
+        kmsvnc->drm->mfb->pixel_format != KMSVNC_FOURCC_TO_INT('A', 'R', '2', '4')
+    )
+    {
+        KMSVNC_FATAL("Unsupported pixfmt %s, please create an issue with your pixfmt.\n", kmsvnc->drm->pixfmt_name);
+    }
+    return 0;
 }
 
-static void convert_bgrx_to_rgb(const char *in, int width, int height, char *buff)
+static void convert_copy(const char *in, int width, int height, char *buff)
 {
-    for (int y = 0; y < height; y++)
+    if (likely(in != buff)) {
-    {
+        memcpy(buff, in, width * height * BYTES_PER_PIXEL);
-        for (int x = 0; x < width; x++)
-        {
-            buff[(y * width + x) * 4] = in[(y * width + x) * 4 + 2];
-            buff[(y * width + x) * 4 + 1] = in[(y * width + x) * 4 + 1];
-            buff[(y * width + x) * 4 + 2] = in[(y * width + x) * 4];
-        }
     }
 }
 
-static char *kms_convert_buf = NULL;
+static void convert_bgra_to_rgba(const char *in, int width, int height, char *buff)
-static size_t kms_convert_buf_len = 0;
+{
-static char *kms_cpy_tmp_buf = NULL;
+    if (likely(in != buff)) {
-static size_t kms_cpy_tmp_buf_len = 0;
+        memcpy(buff, in, width * height * BYTES_PER_PIXEL);
+    }
+    for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
+        uint32_t pixdata = htonl(*((uint32_t*)(buff + i)));
+        buff[i+0] = (pixdata & 0x0000ff00) >> 8;
+        buff[i+2] = (pixdata & 0xff000000) >> 24;
+    }
+}
+
+static inline char convert_buf_allocate(size_t len) {
+    if (kmsvnc->drm->kms_convert_buf_len < len)
+    {
+        if (kmsvnc->drm->kms_convert_buf)
+            free(kmsvnc->drm->kms_convert_buf);
+        kmsvnc->drm->kms_convert_buf = malloc(len);
+        if (!kmsvnc->drm->kms_convert_buf) return 1;
+        kmsvnc->drm->kms_convert_buf_len = len;
+    }
+    return 0;
+}
 static inline void convert_x_tiled(const int tilex, const int tiley, const char *in, int width, int height, char *buff)
 {
     if (width % tilex)
@@ -42,23 +84,18 @@ static inline void convert_x_tiled(const int tilex, const int tiley, const char
         int sno = (width / tilex) + (height / tiley) * (width / tilex);
         int ord = (width % tilex) + (height % tiley) * tilex;
         int max_offset = sno * tilex * tiley + ord;
-        if (kms_cpy_tmp_buf_len < max_offset * 4 + 4)
+        if (kmsvnc->drm->kms_cpy_tmp_buf_len < max_offset * 4 + 4)
         {
-            if (kms_cpy_tmp_buf)
+            if (kmsvnc->drm->kms_cpy_tmp_buf)
-                free(kms_convert_buf);
+                free(kmsvnc->drm->kms_convert_buf);
-            kms_cpy_tmp_buf = malloc(max_offset * 4 + 4);
+            kmsvnc->drm->kms_cpy_tmp_buf = malloc(max_offset * 4 + 4);
-            kms_cpy_tmp_buf_len = max_offset * 4 + 4;
+            if (!kmsvnc->drm->kms_cpy_tmp_buf) return;
+            kmsvnc->drm->kms_cpy_tmp_buf_len = max_offset * 4 + 4;
         }
-        memcpy(kms_cpy_tmp_buf, in, max_offset * 4 + 4);
+        memcpy(kmsvnc->drm->kms_cpy_tmp_buf, in, max_offset * 4 + 4);
-        in = (const char *)kms_cpy_tmp_buf;
+        in = (const char *)kmsvnc->drm->kms_cpy_tmp_buf;
-    }
-    if (kms_convert_buf_len < width * height * 4)
-    {
-        if (kms_convert_buf)
-            free(kms_convert_buf);
-        kms_convert_buf = malloc(width * height * 4);
-        kms_convert_buf_len = width * height * 4;
     }
+    if (convert_buf_allocate(width * height * 4)) return;
     for (int y = 0; y < height; y++)
     {
         for (int x = 0; x < width; x++)
@@ -66,10 +103,10 @@ static inline void convert_x_tiled(const int tilex, const int tiley, const char
             int sno = (x / tilex) + (y / tiley) * (width / tilex);
             int ord = (x % tilex) + (y % tiley) * tilex;
             int offset = sno * tilex * tiley + ord;
-            memcpy(kms_convert_buf + (x + y * width) * 4, in + offset * 4, 4);
+            memcpy(kmsvnc->drm->kms_convert_buf + (x + y * width) * 4, in + offset * 4, 4);
         }
     }
-    convert_bgrx_to_rgb(kms_convert_buf, width, height, buff);
+    convert_bgra_to_rgba(kmsvnc->drm->kms_convert_buf, width, height, buff);
 }
 
 void convert_nvidia_x_tiled_kmsbuf(const char *in, int width, int height, char *buff)
@@ -81,15 +118,55 @@ void convert_intel_x_tiled_kmsbuf(const char *in, int width, int height, char *b
     convert_x_tiled(128, 8, in, width, height, buff);
 }
 
+static void convert_vaapi(const char *in, int width, int height, char *buff) {
+    va_hwframe_to_vaapi(buff);
+    if (
+        (KMSVNC_FOURCC_TO_INT('R','G','B',0) & kmsvnc->va->selected_fmt->fourcc) == KMSVNC_FOURCC_TO_INT('R','G','B',0)
+    ) {}
+    else {
+        // is 30 depth?
+        if (kmsvnc->va->selected_fmt->depth == 30) {
+            for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
+                // ensure little endianess
+                uint32_t pixdata = __builtin_bswap32(htonl(*((uint32_t*)(buff + i))));
+                buff[i] = (pixdata & 0x3ff00000) >> 20 >> 2;
+                buff[i+1] = (pixdata & 0xffc00) >> 10 >> 2;
+                buff[i+2] = (pixdata & 0x3ff) >> 2;
+            }
+        }
+        else {
+            // actually, does anyone use this?
+            if (!kmsvnc->va->selected_fmt->byte_order) {
+                for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
+                    uint32_t *pixdata = (uint32_t*)(buff + i);
+                    *pixdata = __builtin_bswap32(*pixdata);
+                }
+            }
+        }
+        // is xrgb?
+        if ((kmsvnc->va->selected_fmt->blue_mask | kmsvnc->va->selected_fmt->red_mask) < 0x1000000) {
+            for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
+                uint32_t *pixdata = (uint32_t*)(buff + i);
+                *pixdata = ntohl(htonl(*pixdata) << 8);
+            }
+        }
+        // is bgrx?
+        if (kmsvnc->va->selected_fmt->blue_mask > kmsvnc->va->selected_fmt->red_mask) {
+            for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
+                uint32_t pixdata = htonl(*((uint32_t*)(buff + i)));
+                buff[i+0] = (pixdata & 0x0000ff00) >> 8;
+                buff[i+2] = (pixdata & 0xff000000) >> 24;
+            }
+        }
+    }
+}
+
 static inline void drm_sync(int drmfd, uint64_t flags)
 {
-    int ioctl_err;
     struct dma_buf_sync sync = {
         .flags = flags,
     };
-    if (ioctl_err = ioctl(drmfd, DMA_BUF_IOCTL_SYNC, &sync)) {
+    DRM_R_IOCTL_MAY(drmfd, DMA_BUF_IOCTL_SYNC, &sync);
-        fprintf(stderr, "DRM ioctl error %d on line %d\n", ioctl_err, __LINE__);
-    }
 }
 
 void drm_sync_start(int drmfd)
@@ -106,22 +183,59 @@ void drm_sync_noop(int drmfd)
 
 void drm_cleanup() {
     if (kmsvnc->drm) {
+#ifndef DISABLE_KMSVNC_SCREEN_BLANK
+        if (kmsvnc->drm->gamma && kmsvnc->drm->gamma->size && kmsvnc->drm->gamma->red && kmsvnc->drm->gamma->green && kmsvnc->drm->gamma->blue) {
+            if (drmModeCrtcSetGamma(kmsvnc->drm->drm_master_fd ?: kmsvnc->drm->drm_fd, kmsvnc->drm->plane->crtc_id, kmsvnc->drm->gamma->size, kmsvnc->drm->gamma->red, kmsvnc->drm->gamma->green, kmsvnc->drm->gamma->blue)) perror("Failed to restore gamma");
+        }
+        if (kmsvnc->drm->gamma && kmsvnc->drm->gamma->red) {
+            free(kmsvnc->drm->gamma->red);
+            kmsvnc->drm->gamma->red = kmsvnc->drm->gamma->green = kmsvnc->drm->gamma->blue = NULL;
+        }
+        if (kmsvnc->drm->gamma) {
+            free(kmsvnc->drm->gamma);
+            kmsvnc->drm->gamma = NULL;
+        }
+#endif
         if (kmsvnc->drm->drm_ver) {
             drmFreeVersion(kmsvnc->drm->drm_ver);
             kmsvnc->drm->drm_ver = NULL;
         }
+        if (kmsvnc->drm->pixfmt_name) {
+            free(kmsvnc->drm->pixfmt_name);
+            kmsvnc->drm->pixfmt_name = NULL;
+        }
+        if (kmsvnc->drm->mod_vendor) {
+            free(kmsvnc->drm->mod_vendor);
+            kmsvnc->drm->mod_vendor = NULL;
+        }
+        if (kmsvnc->drm->mod_name) {
+            free(kmsvnc->drm->mod_name);
+            kmsvnc->drm->mod_name = NULL;
+        }
         if (kmsvnc->drm->plane) {
             drmModeFreePlane(kmsvnc->drm->plane);
             kmsvnc->drm->plane = NULL;
         }
+        if (kmsvnc->drm->cursor_plane) {
+            drmModeFreePlane(kmsvnc->drm->cursor_plane);
+            kmsvnc->drm->cursor_plane = NULL;
+        }
         if (kmsvnc->drm->mfb) {
             drmModeFreeFB2(kmsvnc->drm->mfb);
             kmsvnc->drm->mfb = NULL;
         }
-        if (kmsvnc->drm->mapped) {
+        if (kmsvnc->drm->cursor_mfb) {
+            drmModeFreeFB2(kmsvnc->drm->cursor_mfb);
+            kmsvnc->drm->cursor_mfb = NULL;
+        }
+        if (kmsvnc->drm->mapped && kmsvnc->drm->mapped != MAP_FAILED) {
             munmap(kmsvnc->drm->mapped, kmsvnc->drm->mmap_size);
             kmsvnc->drm->mapped = NULL;
         }
+        if (kmsvnc->drm->cursor_mapped && kmsvnc->drm->cursor_mapped != MAP_FAILED) {
+            munmap(kmsvnc->drm->cursor_mapped, kmsvnc->drm->cursor_mmap_size);
+            kmsvnc->drm->cursor_mapped = NULL;
+        }
         if (kmsvnc->drm->prime_fd > 0) {
             close(kmsvnc->drm->prime_fd);
             kmsvnc->drm->prime_fd = 0;
@@ -130,25 +244,279 @@ void drm_cleanup() {
             close(kmsvnc->drm->drm_fd);
             kmsvnc->drm->drm_fd = 0;
         }
+        if (kmsvnc->drm->drm_master_fd > 0) {
+            close(kmsvnc->drm->drm_master_fd);
+            kmsvnc->drm->drm_master_fd = 0;
+        }
         if (kmsvnc->drm->plane_res) {
             drmModeFreePlaneResources(kmsvnc->drm->plane_res);
             kmsvnc->drm->plane_res = NULL;
         }
+        if (kmsvnc->drm->kms_convert_buf) {
+            free(kmsvnc->drm->kms_convert_buf);
+            kmsvnc->drm->kms_convert_buf = NULL;
+        }
+        kmsvnc->drm->kms_convert_buf_len = 0;
+        if (kmsvnc->drm->kms_cpy_tmp_buf) {
+            free(kmsvnc->drm->kms_cpy_tmp_buf);
+            kmsvnc->drm->kms_cpy_tmp_buf = NULL;
+        }
+        kmsvnc->drm->kms_cpy_tmp_buf_len = 0;
+        if (kmsvnc->drm->kms_cursor_buf) {
+            free(kmsvnc->drm->kms_cursor_buf);
+            kmsvnc->drm->kms_cursor_buf = NULL;
+        }
+        kmsvnc->drm->kms_cursor_buf_len = 0;
         free(kmsvnc->drm);
         kmsvnc->drm = NULL;
     }
 }
 
+static const char* drm_get_plane_type_name(uint64_t plane_type) {
+    switch (plane_type) {
+        case DRM_PLANE_TYPE_OVERLAY:
+            return "overlay";
+        case DRM_PLANE_TYPE_PRIMARY:
+            return "primary";
+        case DRM_PLANE_TYPE_CURSOR:
+            return "cursor";
+        default:
+            return "unknown";
+    }
+};
+
+static int drm_refresh_planes(char first_time) {
+    struct kmsvnc_drm_data *drm = kmsvnc->drm;
+    if (!drm->plane && kmsvnc->source_plane > 0)
+    {
+        drm->plane = drmModeGetPlane(drm->drm_fd, kmsvnc->source_plane);
+        if (!drm->plane)
+            KMSVNC_FATAL("Failed to get plane %d: %s\n", kmsvnc->source_plane, strerror(errno));
+        if (drm->plane->fb_id == 0)
+            fprintf(stderr, "Place %d does not have an attached framebuffer\n", kmsvnc->source_plane);
+    }
+    if (!drm->plane || (kmsvnc->capture_cursor && !drm->cursor_plane)) {
+        drmModePlane *current_plane = NULL;
+        if (drm->plane_res) {
+            drmModeFreePlaneResources(kmsvnc->drm->plane_res);
+            drm->plane_res = NULL;
+        }
+        drm->plane_res = drmModeGetPlaneResources(drm->drm_fd);
+        if (!drm->plane_res)
+            KMSVNC_FATAL("Failed to get plane resources: %s\n", strerror(errno));
+        int i;
+        for (i = 0; i < drm->plane_res->count_planes; i++)
+        {
+            current_plane = drmModeGetPlane(drm->drm_fd, drm->plane_res->planes[i]);
+            if (!current_plane)
+            {
+                fprintf(stderr, "Failed to get plane %u: %s\n", drm->plane_res->planes[i], strerror(errno));
+                continue;
+            }
+            // get plane type
+            uint64_t plane_type = 114514;
+            drmModeObjectPropertiesPtr plane_props = drmModeObjectGetProperties(drm->drm_fd, current_plane->plane_id, DRM_MODE_OBJECT_PLANE);
+            if (!plane_props) {
+                fprintf(stderr, "Failed to get plane prop %u: %s\n", drm->plane_res->planes[i], strerror(errno));
+            }
+            else {
+                for (int i = 0; i < plane_props->count_props; i++) {
+                    drmModePropertyPtr plane_prop = drmModeGetProperty(drm->drm_fd, plane_props->props[i]);
+                    if (strcmp(plane_prop->name, "type") == 0) {
+                        plane_type = plane_props->prop_values[i];
+                    }
+                    drmModeFreeProperty(plane_prop);
+                }
+                drmModeFreeObjectProperties(plane_props);
+            }
+            assert(drm->plane_res->planes[i] == current_plane->plane_id);
+            if (first_time) {
+                printf("Plane %u CRTC %u FB %u Type %s\n", current_plane->plane_id, current_plane->crtc_id, current_plane->fb_id, drm_get_plane_type_name(plane_type));
+            }
+            // populate drm->plane and drm->cursor_plane
+            char nofree = 0;
+            if (current_plane->fb_id != 0) {
+                if (!drm->plane) {
+                    if (kmsvnc->source_crtc == 0 || current_plane->crtc_id == kmsvnc->source_crtc) {
+                        nofree = 1;
+                        drm->plane = current_plane;
+                    }
+                }
+                // assume cursor plane is always after primary plane
+                if (!drm->cursor_plane) {
+                    if (drm->plane && drm->plane->crtc_id == current_plane->crtc_id && plane_type == DRM_PLANE_TYPE_CURSOR) {
+                        nofree = 1;
+                        drm->cursor_plane = current_plane;
+                    }
+                }
+            }
+            if ((!kmsvnc->capture_cursor || drm->cursor_plane) && drm->plane) {
+                break;
+            }
+            if (!nofree) {
+                drmModeFreePlane(current_plane);
+            }
+            current_plane = NULL;
+        }
+        if (!first_time) return 0;
+        if (i == drm->plane_res->count_planes)
+        {
+            if (!drm->plane) {
+                if (kmsvnc->source_crtc != 0)
+                {
+                    KMSVNC_FATAL("No usable planes found on CRTC %d\n", kmsvnc->source_crtc);
+                }
+                else
+                {
+                    KMSVNC_FATAL("No usable planes found\n");
+                }
+            }
+            else if (!drm->cursor_plane) {
+                fprintf(stderr, "No usable cursor plane found, cursor capture currently unavailable\n");
+            }
+        }
+        printf("Using plane %u to locate framebuffers\n", drm->plane->plane_id);
+        if (drm->cursor_plane) {
+            printf("Using cursor plane %u\n", drm->cursor_plane->plane_id);
+        }
+    }
+    return 0;
+}
+
+int drm_dump_cursor_plane(char **data, int *width, int *height) {
+    struct kmsvnc_drm_data *drm = kmsvnc->drm;
+
+    if (!drm->cursor_plane) {
+        drm_refresh_planes(0); // ignore error
+        if (drm->cursor_plane) {
+            printf("Using cursor plane %u\n", drm->cursor_plane->plane_id);
+        }
+    }
+    else {
+        uint32_t plane_id = drm->cursor_plane->plane_id;
+        drmModeFreePlane(drm->cursor_plane);
+        drm->cursor_plane = NULL;
+        drm->cursor_plane = drmModeGetPlane(drm->drm_fd, plane_id);
+    }
+    if (!drm->cursor_plane) {
+        data = NULL;
+        return 1;
+    }
+    if (drm->cursor_mfb) drmModeFreeFB2(drm->cursor_mfb);
+    drm->cursor_mfb = drmModeGetFB2(drm->drm_fd, drm->cursor_plane->fb_id);
+    if (!drm->cursor_mfb) {
+        KMSVNC_DEBUG("Cursor framebuffer missing\n");
+        return 1;
+    }
+
+    if (drm->cursor_mfb->modifier != DRM_FORMAT_MOD_NONE && drm->cursor_mfb->modifier != DRM_FORMAT_MOD_LINEAR) {
+        //kmsvnc->capture_cursor = 0;
+        KMSVNC_DEBUG("Cursor plane modifier is not linear: %lu\n", drm->cursor_mfb->modifier);
+        return 1;
+    }
+
+    if (
+        drm->cursor_mfb->pixel_format != KMSVNC_FOURCC_TO_INT('A', 'R', '2', '4') &&
+        drm->cursor_mfb->pixel_format != KMSVNC_FOURCC_TO_INT('A', 'R', '3', '0')
+    )
+    {
+        //kmsvnc->capture_cursor = 0;
+        char *fmtname = drmGetFormatName(drm->cursor_mfb->pixel_format);
+        KMSVNC_DEBUG("Cursor plane pixel format unsupported (%u, %s)\n", drm->cursor_mfb->pixel_format, fmtname);
+        free(fmtname);
+        return 1;
+    }
+
+    struct drm_gem_flink flink;
+    flink.handle = drm->cursor_mfb->handles[0];
+    DRM_IOCTL_MUST(drm->drm_fd, DRM_IOCTL_GEM_FLINK, &flink);
+
+    struct drm_gem_open open_arg;
+    open_arg.name = flink.name;
+    DRM_IOCTL_MUST(drm->drm_fd, DRM_IOCTL_GEM_OPEN, &open_arg);
+
+    struct drm_mode_map_dumb mreq;
+    memset(&mreq, 0, sizeof(mreq));
+    mreq.handle = open_arg.handle;
+    DRM_IOCTL_MUST(drm->drm_fd, DRM_IOCTL_MODE_MAP_DUMB, &mreq);
+
+    size_t mmap_size = open_arg.size;
+    if (mmap_size != drm->cursor_mfb->width * drm->cursor_mfb->height * BYTES_PER_PIXEL) {
+        KMSVNC_DEBUG("Cursor plane mmap_size != calculated size (%ld, %d)\n", mmap_size, drm->cursor_mfb->width * drm->cursor_mfb->height * BYTES_PER_PIXEL);
+        return 1;
+    }
+
+    off_t mmap_offset = mreq.offset;
+    if (drm->cursor_mapped && drm->cursor_mapped != MAP_FAILED) munmap(drm->cursor_mapped, drm->cursor_mmap_size);
+    drm->cursor_mapped = mmap(NULL, mmap_size, PROT_READ, MAP_SHARED, drm->drm_fd, mmap_offset);
+    if (drm->cursor_mapped == MAP_FAILED)
+    {
+        KMSVNC_DEBUG("Failed to mmap cursor: %s\n", strerror(errno));
+        return 1;
+    }
+    else
+    {
+        if (kmsvnc->drm->kms_cursor_buf_len < mmap_size)
+        {
+            if (kmsvnc->drm->kms_cursor_buf)
+                free(kmsvnc->drm->kms_cursor_buf);
+            kmsvnc->drm->kms_cursor_buf = malloc(mmap_size);
+            if (!kmsvnc->drm->kms_cursor_buf) return 1;
+            kmsvnc->drm->kms_cursor_buf_len = mmap_size;
+        }
+        memcpy(drm->kms_cursor_buf, drm->cursor_mapped, mmap_size);
+        if (drm->cursor_mfb->pixel_format == KMSVNC_FOURCC_TO_INT('X', 'R', '3', '0') ||
+            drm->cursor_mfb->pixel_format == KMSVNC_FOURCC_TO_INT('A', 'R', '3', '0'))
+        {
+            for (int i = 0; i < drm->cursor_mfb->width * drm->cursor_mfb->height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
+                uint32_t pixdata = __builtin_bswap32(htonl(*((uint32_t*)(kmsvnc->drm->kms_cursor_buf + i))));
+                kmsvnc->drm->kms_cursor_buf[i] = (pixdata & 0x3ff00000) >> 20 >> 2;
+                kmsvnc->drm->kms_cursor_buf[i+1] = (pixdata & 0xffc00) >> 10 >> 2;
+                kmsvnc->drm->kms_cursor_buf[i+2] = (pixdata & 0x3ff) >> 2;
+                kmsvnc->drm->kms_cursor_buf[i+3] = (pixdata & 0xc0000000) >> 30 << 6;
+            }
+        }
+        if (drm->cursor_mfb->pixel_format == KMSVNC_FOURCC_TO_INT('X', 'R', '2', '4') || 
+            drm->cursor_mfb->pixel_format == KMSVNC_FOURCC_TO_INT('A', 'R', '2', '4'))
+        {
+            // bgra to rgba
+            for (int i = 0; i < drm->cursor_mfb->width * drm->cursor_mfb->height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
+                uint32_t pixdata = htonl(*((uint32_t*)(kmsvnc->drm->kms_cursor_buf + i)));
+                kmsvnc->drm->kms_cursor_buf[i+0] = (pixdata & 0x0000ff00) >> 8;
+                kmsvnc->drm->kms_cursor_buf[i+2] = (pixdata & 0xff000000) >> 24;
+            }
+        }
+        *width = drm->cursor_mfb->width;
+        *height = drm->cursor_mfb->height;
+        *data = drm->kms_cursor_buf;
+    }
+    return 0;
+}
+
 int drm_open() {
     struct kmsvnc_drm_data *drm = malloc(sizeof(struct kmsvnc_drm_data));
+    if (!drm) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
     memset(drm, 0, sizeof(struct kmsvnc_drm_data));
     kmsvnc->drm = drm;
 
     drm->drm_fd = open(kmsvnc->card, O_RDONLY);
     if (drm->drm_fd < 0)
     {
-        DRM_FATAL("card %s open failed: %s\n", kmsvnc->card, strerror(errno));
+        KMSVNC_FATAL("card %s open failed: %s\n", kmsvnc->card, strerror(errno));
     }
+    if (!kmsvnc->screen_blank && drmIsMaster(drm->drm_fd)) {
+        if (drmDropMaster(drm->drm_fd)) fprintf(stderr, "Failed to drop master");
+    }
+#ifndef DISABLE_KMSVNC_SCREEN_BLANK
+    if (kmsvnc->screen_blank && !drmIsMaster(drm->drm_fd)) {
+        drm->drm_master_fd = drm_get_master_fd();
+        drm->drm_master_fd = drm->drm_master_fd > 0 ? drm->drm_master_fd : 0;
+        if (kmsvnc->debug_enabled) {
+            fprintf(stderr, "not master client, master fd %d\n", drm->drm_master_fd);
+        }
+    }
+#endif
+
     drm->drm_ver = drmGetVersion(drm->drm_fd);
     printf("drm driver is %s\n", drm->drm_ver->name);
 
@@ -157,84 +525,94 @@ int drm_open() {
     {
         perror("Failed to set universal planes capability: primary planes will not be usable");
     }
-    if (kmsvnc->source_plane > 0)
-    {
-        drm->plane = drmModeGetPlane(drm->drm_fd, kmsvnc->source_plane);
-        if (!drm->plane)
-            DRM_FATAL("Failed to get plane %d: %s\n", kmsvnc->source_plane, strerror(errno));
-        if (drm->plane->fb_id == 0)
-            fprintf(stderr, "Place %d does not have an attached framebuffer\n", kmsvnc->source_plane);
-    }
-    else
-    {
-        drm->plane_res = drmModeGetPlaneResources(drm->drm_fd);
-        if (!drm->plane_res)
-            DRM_FATAL("Failed to get plane resources: %s\n", strerror(errno));
-        int i;
-        for (i = 0; i < drm->plane_res->count_planes; i++)
-        {
-            drm->plane = drmModeGetPlane(drm->drm_fd, drm->plane_res->planes[i]);
-            if (!drm->plane)
-            {
-                fprintf(stderr, "Failed to get plane %u: %s\n", drm->plane_res->planes[i], strerror(errno));
-                continue;
-            }
-            printf("Plane %u CRTC %u FB %u\n", drm->plane->plane_id, drm->plane->crtc_id, drm->plane->fb_id);
-            if ((kmsvnc->source_crtc != 0 && drm->plane->crtc_id != kmsvnc->source_crtc) || drm->plane->fb_id == 0)
-            {
-                // Either not connected to the target source CRTC
-                // or not active.
-                drmModeFreePlane(drm->plane);
-                drm->plane = NULL;
-                continue;
-            }
-            break;
-        }
-        if (i == drm->plane_res->count_planes)
-        {
-            if (kmsvnc->source_crtc != 0)
-            {
-                DRM_FATAL("No usable planes found on CRTC %d\n", kmsvnc->source_crtc);
-            }
-            else
-            {
-                DRM_FATAL("No usable planes found\n");
-            }
-        }
-        printf("Using plane %u to locate framebuffers\n", drm->plane->plane_id);
-    }
-    uint32_t plane_id = drm->plane->plane_id;
 
+    if (drm_refresh_planes(1)) return 1;
+
+#ifndef DISABLE_KMSVNC_SCREEN_BLANK
+    if (kmsvnc->screen_blank) {
+        drm->gamma = malloc(sizeof(struct kmsvnc_drm_gamma_data));
+        if (!drm->gamma) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
+        memset(drm->gamma, 0, sizeof(struct kmsvnc_drm_gamma_data));
+        drmModeCrtc *target_crtc = drmModeGetCrtc(drm->drm_fd, drm->plane->crtc_id);
+        if (target_crtc) {
+            drm->gamma->size = (uint32_t)target_crtc->gamma_size;
+            drm->gamma->red = malloc(drm->gamma->size*sizeof(uint16_t)*3);
+            if (!drm->gamma->size) {
+                fprintf(stderr, "drm->gamma->size = %u, not setting gamma.\n", drm->gamma->size);
+            }
+            else if (!drm->gamma->red) {
+                fprintf(stderr, "memory allocation error at %s:%d\n", __FILE__, __LINE__);
+                fprintf(stderr, "not setting gamma.\n");
+            }
+            else {
+                memset(drm->gamma->red, 0, drm->gamma->size*sizeof(uint16_t)*3);
+                drm->gamma->green = drm->gamma->red + drm->gamma->size;
+                drm->gamma->blue = drm->gamma->red + drm->gamma->size*2;
+                if (kmsvnc->screen_blank_restore) {
+                    int step = 0x10000 / drm->gamma->size;
+                    for (int i = 0; i < drm->gamma->size; i++) {
+                        drm->gamma->red[i] = drm->gamma->green[i] = drm->gamma->blue[i] = step * i;
+                    }
+                }
+                else {
+                    // legacy api, but weston also uses this, so whatever
+                    drmModeCrtcGetGamma(drm->drm_fd, drm->plane->crtc_id, drm->gamma->size, drm->gamma->red, drm->gamma->green, drm->gamma->blue);
+                }
+                if (kmsvnc->debug_enabled) {
+                    for (int i = 0; i < drm->gamma->size; i++) {
+                        fprintf(stderr, "gamma: %05d %05hu %05hu %05hu\n", i, drm->gamma->red[i], drm->gamma->green[i], drm->gamma->blue[i]);
+                    }
+                }
+                uint16_t *new_gamma_red = malloc(drm->gamma->size*sizeof(uint16_t)*3);
+                if (!new_gamma_red) {
+                    fprintf(stderr, "memory allocation error at %s:%d\n", __FILE__, __LINE__);
+                    fprintf(stderr, "not setting gamma.\n");
+                }
+                else {
+                    memset(new_gamma_red, 0, drm->gamma->size*sizeof(uint16_t)*3);
+                    uint16_t *new_gamma_green = new_gamma_red + drm->gamma->size;
+                    uint16_t *new_gamma_blue = new_gamma_red + drm->gamma->size*2;
+                    if (drmModeCrtcSetGamma(drm->drm_master_fd ?: drm->drm_fd, drm->plane->crtc_id, drm->gamma->size, new_gamma_red, new_gamma_green, new_gamma_blue)) perror("Failed to set gamma");
+                }
+                if (new_gamma_red) {
+                    free(new_gamma_red);
+                    new_gamma_red = NULL;
+                }
+            }
+        }
+        else {
+            fprintf(stderr, "Did not get a crtc structure, not setting gamma.\n");
+        }
+        if (target_crtc) {
+            drmModeFreeCrtc(target_crtc);
+            target_crtc = NULL;
+        }
+    }
+#endif
 
     drm->mfb = drmModeGetFB2(drm->drm_fd, drm->plane->fb_id);
     if (!drm->mfb) {
-        DRM_FATAL("Failed to get framebuffer %u: %s\n", drm->plane->fb_id, strerror(errno));
+        KMSVNC_FATAL("Failed to get framebuffer %u: %s\n", drm->plane->fb_id, strerror(errno));
     }
-    printf("Template framebuffer is %u: %ux%u fourcc:%u mod:%u flags:%u\n", drm->mfb->fb_id, drm->mfb->width, drm->mfb->height, drm->mfb->pixel_format, drm->mfb->modifier, drm->mfb->flags);
+    drm->pixfmt_name = drmGetFormatName(drm->mfb->pixel_format);
+    drm->mod_vendor = drmGetFormatModifierVendor(drm->mfb->modifier);
+    drm->mod_name = drmGetFormatModifierName(drm->mfb->modifier);
+    printf("Template framebuffer is %u: %ux%u fourcc:%u mod:%lu flags:%u\n", drm->mfb->fb_id, drm->mfb->width, drm->mfb->height, drm->mfb->pixel_format, drm->mfb->modifier, drm->mfb->flags);
     printf("handles %u %u %u %u\n", drm->mfb->handles[0], drm->mfb->handles[1], drm->mfb->handles[2], drm->mfb->handles[3]);
     printf("offsets %u %u %u %u\n", drm->mfb->offsets[0], drm->mfb->offsets[1], drm->mfb->offsets[2], drm->mfb->offsets[3]);
     printf("pitches %u %u %u %u\n", drm->mfb->pitches[0], drm->mfb->pitches[1], drm->mfb->pitches[2], drm->mfb->pitches[3]);
-    printf("format %s, modifier %s:%s\n", drmGetFormatName(drm->mfb->pixel_format), drmGetFormatModifierVendor(drm->mfb->modifier), drmGetFormatModifierName(drm->mfb->modifier));
+    printf("format %s, modifier %s:%s\n", drm->pixfmt_name, drm->mod_vendor, drm->mod_name);
-
-    if (
-        drm->mfb->pixel_format != KMSVNC_FOURCC_TO_INT('X', 'R', '2', '4') &&
-        drm->mfb->pixel_format != KMSVNC_FOURCC_TO_INT('A', 'R', '2', '4')
-    )
-    {
-        DRM_FATAL("Unsupported pixfmt\n");
-    }
 
     if (!drm->mfb->handles[0])
     {
-        DRM_FATAL("No handle set on framebuffer: maybe you need some additional capabilities?\n");
+        KMSVNC_FATAL("No handle set on framebuffer: maybe you need some additional capabilities?\n");
     }
 
-    int ioctl_err = 0;
-
     drm->mmap_fd = drm->drm_fd;
     drm->mmap_size = drm->mfb->width * drm->mfb->height * BYTES_PER_PIXEL;
     drm->funcs = malloc(sizeof(struct kmsvnc_drm_funcs));
-    drm->funcs->convert = convert_bgrx_to_rgb;
+    if (!drm->funcs) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
+    drm->funcs->convert = convert_bgra_to_rgba;
     drm->funcs->sync_start = drm_sync_noop;
     drm->funcs->sync_end = drm_sync_noop;
 
@@ -250,7 +628,7 @@ static int drm_kmsbuf_prime() {
     int err = drmPrimeHandleToFD(drm->drm_fd, drm->mfb->handles[0], O_RDWR, &drm->prime_fd);
     if (err < 0 || drm->prime_fd < 0)
     {
-        DRM_FATAL("Failed to get PRIME fd from framebuffer handle");
+        KMSVNC_FATAL("Failed to get PRIME fd from framebuffer handle\n");
     }
     drm->funcs->sync_start = &drm_sync_start;
     drm->funcs->sync_end = &drm_sync_end;
@@ -258,6 +636,22 @@ static int drm_kmsbuf_prime() {
     return 0;
 }
 
+static int drm_kmsbuf_prime_vaapi() {
+    struct kmsvnc_drm_data *drm = kmsvnc->drm;
+
+    int err = drmPrimeHandleToFD(drm->drm_fd, drm->mfb->handles[0], O_RDWR, &drm->prime_fd);
+    if (err < 0 || drm->prime_fd < 0)
+    {
+        KMSVNC_FATAL("Failed to get PRIME fd from framebuffer handle\n");
+    }
+
+    if (va_init()) return 1;
+
+    drm->mmap_fd = drm->prime_fd;
+    drm->skip_map = 1;
+    return 0;
+}
+
 static int drm_kmsbuf_dumb() {
     struct kmsvnc_drm_data *drm = kmsvnc->drm;
 
@@ -291,33 +685,23 @@ int drm_vendors() {
         driver_name = drm->drm_ver->name;
     }
 
-    if (strcmp(driver_name, "i915") == 0)
+    if (strcmp(driver_name, "i915") == 0 || strcmp(driver_name, "amdgpu") == 0)
     {
-        drm->funcs->convert = &convert_intel_x_tiled_kmsbuf;
+        if (fourcc_mod_is_vendor(drm->mfb->modifier, INTEL)) {
-        if (drm_kmsbuf_prime()) return 1;
+            if (strstr(drm->mod_name, "CCS")) {
-    }
+                printf("warn: intel with CCS modifier detected, please set INTEL_DEBUG=noccs\n");
-    else if (strcmp(driver_name, "amdgpu") == 0)
+            }
-    {
+        };
-        struct drm_gem_flink flink;
+        drm->funcs->convert = &convert_vaapi;
-        flink.handle = drm->mfb->handles[0];
+        if (drm_kmsbuf_prime_vaapi()) return 1;
-        DRM_IOCTL_MUST(drm->drm_fd, DRM_IOCTL_GEM_FLINK, &flink);
-
-        struct drm_gem_open open_arg;
-        open_arg.name = flink.name;
-        DRM_IOCTL_MUST(drm->drm_fd, DRM_IOCTL_GEM_OPEN, &open_arg);
-
-        union drm_amdgpu_gem_mmap mmap_arg;
-        memset(&mmap_arg, 0, sizeof(mmap_arg));
-        mmap_arg.in.handle = open_arg.handle;
-        DRM_IOCTL_MUST(drm->drm_fd, DRM_IOCTL_AMDGPU_GEM_MMAP, &mmap_arg);
-
-        drm->mmap_size = open_arg.size;
-        drm->mmap_offset = mmap_arg.out.addr_ptr;
     }
     else if (strcmp(driver_name, "nvidia-drm") == 0)
     {
-        // quirky and slow
+        if (check_pixfmt_non_vaapi()) return 1;
-        drm->funcs->convert = &convert_nvidia_x_tiled_kmsbuf;
+        printf("warn: nvidia card detected. Currently only x-tiled framebuffer is supported. Performance may suffer.\n");
+        if (drm->mfb->modifier != DRM_FORMAT_MOD_NONE && drm->mfb->modifier != DRM_FORMAT_MOD_LINEAR) {
+            drm->funcs->convert = &convert_nvidia_x_tiled_kmsbuf;
+        }
         if (drm_kmsbuf_dumb()) return 1;
     }
     else if (strcmp(driver_name, "vmwgfx") == 0 ||
@@ -325,19 +709,26 @@ int drm_vendors() {
              strcmp(driver_name, "virtio_gpu") == 0
     )
     {
+        if (check_pixfmt_non_vaapi()) return 1;
+        if (drm->mfb->modifier != DRM_FORMAT_MOD_NONE && drm->mfb->modifier != DRM_FORMAT_MOD_LINEAR) {
+            printf("warn: modifier is not LINEAR, please create an issue with your modifier.\n");
+        }
         // virgl does not work
         if (drm_kmsbuf_dumb()) return 1;
     }
     else if (strcmp(driver_name, "test-prime") == 0)
     {
+        if (check_pixfmt_non_vaapi()) return 1;
         if (drm_kmsbuf_prime()) return 1;
     }
     else if (strcmp(driver_name, "test-map-dumb") == 0)
     {
+        if (check_pixfmt_non_vaapi()) return 1;
         if (drm_kmsbuf_dumb()) return 1;
     }
     else if (strcmp(driver_name, "test-i915-gem") == 0)
     {
+        if (check_pixfmt_non_vaapi()) return 1;
         struct drm_gem_flink flink;
         flink.handle = drm->mfb->handles[0];
         DRM_IOCTL_MUST(drm->drm_fd, DRM_IOCTL_GEM_FLINK, &flink);
@@ -352,19 +743,29 @@ int drm_vendors() {
         drm->mmap_size = open_arg.size;
         drm->mmap_offset = mmap_arg.offset;
     }
+    else if (strcmp(driver_name, "test-i915-prime-xtiled") == 0)
+    {
+        if (check_pixfmt_non_vaapi()) return 1;
+        drm->funcs->convert = &convert_intel_x_tiled_kmsbuf;
+        if (drm_kmsbuf_prime()) return 1;
+    }
     else
     {
+        if (check_pixfmt_non_vaapi()) return 1;
         fprintf(stderr, "Untested drm driver, use at your own risk!\n");
+        if (drm->mfb->modifier != DRM_FORMAT_MOD_NONE && drm->mfb->modifier != DRM_FORMAT_MOD_LINEAR) {
+            printf("warn: modifier is not LINEAR, please create an issue with your driver and modifier.\n");
+        }
         if (drm_kmsbuf_dumb()) return 1;
     }
 
-    if (!drm->mapped)
+    if (!drm->skip_map && !drm->mapped)
     {
-        printf("mapping with size = %d, offset = %d, fd = %d\n", drm->mmap_size, drm->mmap_offset, drm->mmap_fd);
+        printf("mapping with size = %lu, offset = %ld, fd = %d\n", drm->mmap_size, drm->mmap_offset, drm->mmap_fd);
         drm->mapped = mmap(NULL, drm->mmap_size, PROT_READ, MAP_SHARED, drm->mmap_fd, drm->mmap_offset);
         if (drm->mapped == MAP_FAILED)
         {
-            DRM_FATAL("Failed to mmap: %s\n", strerror(errno));
+            KMSVNC_FATAL("Failed to mmap: %s\n", strerror(errno));
         }
     }
 

drm.h

@@ -2,12 +2,12 @@
 
 #include "kmsvnc.h"
 
-#define KMSVNC_FOURCC_TO_INT(a,b,c,d) (((a) << 0) + ((b) << 8) + ((c) << 16) + ((d) << 24))
+#define DRM_IOCTL_MUST(...) do{ int e; if ((e = drmIoctl(__VA_ARGS__))) KMSVNC_FATAL("DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
+#define DRM_IOCTL_MAY(...) do{ int e; if ((e = drmIoctl(__VA_ARGS__))) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
+#define DRM_R_IOCTL_MAY(...) do{ int e; if ((e = ioctl(__VA_ARGS__))) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
 
-#define DRM_FATAL(...) { fprintf(stderr, __VA_ARGS__); return 1; }
-#define DRM_IOCTL_MUST(...) { int e; if (e = drmIoctl(__VA_ARGS__)) DRM_FATAL("DRM ioctl error %d on line %d\n", e, __LINE__) }
-#define DRM_IOCTL_MAY(...) { int e; if (e = drmIoctl(__VA_ARGS__)) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); }
 
 void drm_cleanup();
 int drm_open();
 int drm_vendors();
+int drm_dump_cursor_plane(char **data, int *width, int *height);

drm_master.c

@@ -0,0 +1,110 @@
+#define _GNU_SOURCE
+
+#include <stdio.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+#include <dirent.h>
+#include <stdlib.h>
+
+#include "drm_master.h"
+
+extern struct kmsvnc_data *kmsvnc;
+
+
+static inline int clone_fd(pid_t pid, int target_fd) {
+    int pidfd = syscall(SYS_pidfd_open, pid, 0);
+    if (pidfd <= 0) {
+        perror("pidfd_open");
+        return -1;
+    }
+    int cloned = syscall(SYS_pidfd_getfd, pidfd, target_fd, 0);
+    if (cloned <= 0) {
+        perror("pidfd_getfd");
+    }
+    close(pidfd);
+    return cloned;
+}
+
+static inline int cmp_fds(pid_t pid, const char *drm_pth) {
+    char path[PATH_MAX+1];
+    snprintf(path, PATH_MAX+1, "/proc/%d/fd", pid);
+
+    struct dirent **fdlist;
+    int count = scandir(path, &fdlist, NULL, versionsort);
+    int ret = -1;
+    if (count >= 0) {
+        for (int n = 0; n < count; n++) {
+            if (ret == -1 && fdlist[n]->d_type == DT_LNK) {
+                char link_pth[PATH_MAX+1];
+                char real_pth[PATH_MAX+1];
+                #pragma GCC diagnostic push
+                #pragma GCC diagnostic ignored "-Wpragmas"
+                #pragma GCC diagnostic ignored "-Wunknown-warning-option"
+                #pragma GCC diagnostic ignored "-Wformat-truncation"
+                snprintf(link_pth, PATH_MAX+1, "%s/%s", path, fdlist[n]->d_name);
+                #pragma GCC diagnostic pop
+                memset(real_pth, 0, PATH_MAX+1);
+                #pragma GCC diagnostic push
+                #pragma GCC diagnostic ignored "-Wunused-result"
+                realpath(link_pth, real_pth);
+                #pragma GCC diagnostic pop
+                if (!strncmp(real_pth, drm_pth, PATH_MAX)) {
+                    int fd = atoi(fdlist[n]->d_name);
+                    if (fd > 0) {
+                        int cloned = clone_fd(pid, fd);
+                        if (cloned > 0 && drmIsMaster(cloned)) {
+                            ret = cloned;
+                            if (kmsvnc->debug_enabled) {
+                                fprintf(stderr, "found drm master pid=%d, fd=%d, cloned=%d\n", pid, fd, cloned);
+                            }
+                        }
+                        else {
+                            if (cloned > 0) close(cloned);
+                        }
+                    }
+                }
+            }
+            free(fdlist[n]);
+            fdlist[n] = NULL;
+        }
+        free(fdlist);
+        fdlist = NULL;
+    }
+    return ret;
+}
+
+int drm_get_master_fd() {
+    char drm_pth[PATH_MAX+1];
+    memset(drm_pth, 0, PATH_MAX+1);
+    #pragma GCC diagnostic push
+    #pragma GCC diagnostic ignored "-Wunused-result"
+    realpath(kmsvnc->card, drm_pth);
+    #pragma GCC diagnostic pop
+
+    struct dirent **proclist;
+    int count = scandir("/proc", &proclist, NULL, versionsort);
+    int ret = -1;
+    if (count >= 0) {
+        for (int n = 0; n < count; n++) {
+            if (ret == -1 && proclist[n]->d_type == DT_DIR) {
+                pid_t pid = (pid_t)atoi(proclist[n]->d_name);
+                if (pid > 0) {
+                    int cloned = cmp_fds(pid, drm_pth);
+                    if (cloned > 0) {
+                        ret = cloned;
+                    }
+                }
+            }
+            free(proclist[n]);
+            proclist[n] = NULL;
+        }
+        free(proclist);
+        proclist = NULL;
+    }
+    else {
+        perror("open /proc");
+    }
+    return ret;
+}

drm_master.h

@@ -0,0 +1,5 @@
+#pragma once
+
+#include "kmsvnc.h"
+
+int drm_get_master_fd();

input.c

@@ -26,16 +26,18 @@ void uinput_cleanup()
     }
 }
 
+static void wake_system_up();
 int uinput_init()
 {
     struct kmsvnc_input_data *inp = malloc(sizeof(struct kmsvnc_input_data));
+    if (!inp) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
     memset(inp, 0, sizeof(struct kmsvnc_input_data));
     kmsvnc->input = inp;
 
     inp->uinput_fd = open("/dev/uinput", O_WRONLY | O_NONBLOCK);
     if (inp->uinput_fd <= 0)
     {
-        INP_FATAL("Failed to open uinput\n");
+        KMSVNC_FATAL("Failed to open uinput\n");
     }
     INP_IOCTL_MUST(inp->uinput_fd, UI_SET_EVBIT, EV_KEY);
     INP_IOCTL_MUST(inp->uinput_fd, UI_SET_EVBIT, EV_SYN);
@@ -48,6 +50,10 @@ int uinput_init()
     INP_IOCTL_MUST(inp->uinput_fd, UI_SET_ABSBIT, ABS_X);
     INP_IOCTL_MUST(inp->uinput_fd, UI_SET_ABSBIT, ABS_Y);
 
+    INP_IOCTL_MUST(inp->uinput_fd, UI_SET_EVBIT, EV_REL);
+    INP_IOCTL_MUST(inp->uinput_fd, UI_SET_RELBIT, REL_X);
+    INP_IOCTL_MUST(inp->uinput_fd, UI_SET_RELBIT, REL_Y);
+
     INP_IOCTL_MUST(inp->uinput_fd, UI_SET_KEYBIT, BTN_LEFT);
     INP_IOCTL_MUST(inp->uinput_fd, UI_SET_KEYBIT, BTN_MIDDLE);
     INP_IOCTL_MUST(inp->uinput_fd, UI_SET_KEYBIT, BTN_RIGHT);
@@ -74,8 +80,16 @@ int uinput_init()
     INP_IOCTL_MUST(inp->uinput_fd, UI_DEV_CREATE);
 
     inp->keystate = malloc(UINPUT_MAX_KEY);
+    if (!inp->keystate) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
     memset(inp->keystate, 0, UINPUT_MAX_KEY);
 
+    if (kmsvnc->input_wakeup) {
+        printf("waiting for 1 second for userspace to detect the input devive...\n");
+        sleep(1);
+        wake_system_up();
+        printf("waiting for 1 second for mouse input to be processed...\n");
+        sleep(1);
+    }
     return 0;
 }
 
@@ -114,9 +128,9 @@ void rfb_key_hook(rfbBool down, rfbKeySym keysym, rfbClientPtr cl)
                 .value = 0,
             },
         };
-        for (int i = 0; i < ARRAY_SIZE(ies); i++)
+        for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(ies); i++)
         {
-            write(kmsvnc->input->uinput_fd, &ies[i], sizeof(ies[0]));
+            KMSVNC_WRITE_MAY(kmsvnc->input->uinput_fd, &ies[i], sizeof(ies[0]));
         }
 
         kmsvnc->input->keystate[search.keycode] = down;
@@ -126,10 +140,10 @@ void rfb_key_hook(rfbBool down, rfbKeySym keysym, rfbClientPtr cl)
 void rfb_ptr_hook(int mask, int screen_x, int screen_y, rfbClientPtr cl)
 {
     // printf("pointer to %d, %d\n", screen_x, screen_y);
-    float global_x = (float)screen_x;
+    float global_x = (float)(screen_x + kmsvnc->input_offx);
-    float global_y = (float)screen_y;
+    float global_y = (float)(screen_y + kmsvnc->input_offy);
-    int touch_x = round(global_x / kmsvnc->drm->mfb->width * UINPUT_ABS_MAX);
+    int touch_x = round(global_x / (kmsvnc->input_width ?: kmsvnc->drm->mfb->width) * UINPUT_ABS_MAX);
-    int touch_y = round(global_y / kmsvnc->drm->mfb->height * UINPUT_ABS_MAX);
+    int touch_y = round(global_y / (kmsvnc->input_height ?: kmsvnc->drm->mfb->height) * UINPUT_ABS_MAX);
     struct input_event ies1[] = {
         {
             .type = EV_ABS,
@@ -159,9 +173,9 @@ void rfb_ptr_hook(int mask, int screen_x, int screen_y, rfbClientPtr cl)
             .value = 0,
         },
     };
-    for (int i = 0; i < ARRAY_SIZE(ies1); i++)
+    for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(ies1); i++)
     {
-        write(kmsvnc->input->uinput_fd, &ies1[i], sizeof(ies1[0]));
+        KMSVNC_WRITE_MAY(kmsvnc->input->uinput_fd, &ies1[i], sizeof(ies1[0]));
     }
     if (mask & 0b11000)
     {
@@ -177,9 +191,39 @@ void rfb_ptr_hook(int mask, int screen_x, int screen_y, rfbClientPtr cl)
                 .value = 0,
             },
         };
-        for (int i = 0; i < ARRAY_SIZE(ies2); i++)
+        for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(ies2); i++)
         {
-            write(kmsvnc->input->uinput_fd, &ies2[i], sizeof(ies2[0]));
+            KMSVNC_WRITE_MAY(kmsvnc->input->uinput_fd, &ies2[i], sizeof(ies2[0]));
         }
     }
 }
+
+static void wake_system_up()
+{
+    struct input_event ies1[] = {
+        {
+            .type = EV_REL,
+            .code = REL_X,
+            .value = 1,
+        },
+        {
+            .type = EV_SYN,
+            .code = SYN_REPORT,
+            .value = 0,
+        },
+        {
+            .type = EV_REL,
+            .code = REL_X,
+            .value = -1,
+        },
+        {
+            .type = EV_SYN,
+            .code = SYN_REPORT,
+            .value = 0,
+        },
+    };
+    for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(ies1); i++)
+    {
+        KMSVNC_WRITE_MAY(kmsvnc->input->uinput_fd, &ies1[i], sizeof(ies1[0]));
+    }
+}

input.h

@@ -4,14 +4,11 @@
 
 #include "kmsvnc.h"
 
-#define ARRAY_SIZE(x) (sizeof(x) / sizeof(x[0]))
-
 #define UINPUT_ABS_MAX INT16_MAX
 #define UINPUT_MAX_KEY 256
 
-#define INP_FATAL(...) { fprintf(stderr, __VA_ARGS__); return 1; }
+#define INP_IOCTL_MUST(...) do{ int e; if ((e = ioctl(__VA_ARGS__))) KMSVNC_FATAL("uinput ioctl error %d on line %d\n", e, __LINE__); } while(0)
-#define INP_IOCTL_MUST(...) { int e; if (e = ioctl(__VA_ARGS__)) INP_FATAL("uinput ioctl error %d on line %d\n", e, __LINE__) }
+#define INP_IOCTL_MAY(...) do{ int e; if ((e = ioctl(__VA_ARGS__))) fprintf(stderr, "uinput ioctl error %d on line %d\n", e, __LINE__); } while(0)
-#define INP_IOCTL_MAY(...) { int e; if (e = ioctl(__VA_ARGS__)) fprintf(stderr, "uinput ioctl error %d on line %d\n", e, __LINE__); }
 
 void uinput_cleanup();
 int uinput_init();

keymap.c

@@ -25,13 +25,14 @@ void xkb_cleanup() {
 int xkb_init()
 {
     struct kmsvnc_keymap_data *xkb = malloc(sizeof(struct kmsvnc_keymap_data));
+    if (!xkb) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
     memset(xkb, 0, sizeof(struct kmsvnc_keymap_data));
     kmsvnc->keymap = xkb;
 
     xkb->ctx = xkb_context_new(XKB_CONTEXT_NO_FLAGS);
     if (xkb->ctx == NULL)
     {
-        XKB_FATAL("Failed to create XKB context\n");
+        KMSVNC_FATAL("Failed to create XKB context\n");
     }
     struct xkb_rule_names names = {
         .rules = NULL,
@@ -43,7 +44,7 @@ int xkb_init()
     xkb->map = xkb_keymap_new_from_names(xkb->ctx, &names, 0);
     if (xkb->map == NULL)
     {
-        XKB_FATAL("Failed to create XKB keymap\n");
+        KMSVNC_FATAL("Failed to create XKB keymap\n");
     }
     // printf("xkb: keymap string\n%s\n", xkb_keymap_get_as_string(xkb->map, XKB_KEYMAP_USE_ORIGINAL_FORMAT));
     return 0;

keymap.h

@@ -2,8 +2,6 @@
 
 #include "kmsvnc.h"
 
-#define XKB_FATAL(...) { fprintf(stderr, __VA_ARGS__); return 1; }
-
 void xkb_cleanup();
 int xkb_init();
 void key_iter(struct xkb_keymap *xkb, xkb_keycode_t key, void *data);

kmsvnc.c

@@ -1,9 +1,12 @@
+#define _GNU_SOURCE
+
 #include <fcntl.h>
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
 #include <signal.h>
 #include <time.h>
+#include <unistd.h>
 #include <argp.h>
 #include <arpa/inet.h>
 
@@ -11,6 +14,7 @@
 #include "keymap.h"
 #include "input.h"
 #include "drm.h"
+#include "va.h"
 
 struct kmsvnc_data *kmsvnc = NULL;
 
@@ -88,6 +92,89 @@ static void update_screen_buf(char* to, char *from, int width, int height) {
     }
 }
 
+static inline void update_vnc_cursor(char *data, int width, int height) {
+    uint8_t r, g, b, a;
+    #define CURSOR_MIN_A 160 // ~63%
+    int min_x = width;
+    int max_x = -1;
+    int min_y = height;
+    int max_y = -1;
+    int x, y;
+
+    for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
+        uint32_t pixdata = htonl(*((uint32_t*)(data + i)));
+        //r = (pixdata & 0xff000000u) >> 24;
+        //g = (pixdata & 0x00ff0000u) >> 16;
+        //b = (pixdata & 0x0000ff00u) >> 8;
+        a = pixdata & 0xff;
+        if (a > CURSOR_MIN_A) {
+            x = (i / BYTES_PER_PIXEL) % width;
+            y = (i / BYTES_PER_PIXEL) / width;
+            if (x < min_x) min_x = x;
+            if (y < min_y) min_y = y;
+            if (x > max_x) max_x = x;
+            if (y > max_y) max_y = y;
+        }
+    }
+    if (min_x > max_x || min_y > max_y) {
+        // no cursor detected
+        return;
+    }
+    int rwidth = max_x - min_x + 1;
+    int rheight = max_y - min_y + 1;
+    if (kmsvnc->cursor_bitmap_len < rwidth * rheight * BYTES_PER_PIXEL)
+    {
+        if (kmsvnc->cursor_bitmap)
+            free(kmsvnc->cursor_bitmap);
+        kmsvnc->cursor_bitmap = malloc(rwidth * rheight * BYTES_PER_PIXEL);
+        if (!kmsvnc->cursor_bitmap) return;
+        kmsvnc->cursor_bitmap_len = rwidth * rheight * BYTES_PER_PIXEL;
+    }
+    unsigned char *rich_source = malloc(rwidth * rheight * BYTES_PER_PIXEL);
+    if (!rich_source) return;
+    char *maskString = malloc(rwidth * rheight);
+    if (!maskString) {
+        free(rich_source);
+        return;
+    }
+    memset(maskString, ' ', rwidth * rheight);
+    for (int i = 0; i < rwidth; i++) {
+        for (int j = 0; j < rheight; j++) {
+            int t = (i + j * rwidth) * BYTES_PER_PIXEL;
+            int s = ((i+min_x) + (j+min_y) * width) * BYTES_PER_PIXEL;
+            *((uint32_t*)(rich_source + t)) = *((uint32_t*)(data + s));
+            if ((uint8_t)*(rich_source + t + 3) > CURSOR_MIN_A) {
+                maskString[i + j * rwidth] = 'x';
+            }
+        }
+    }
+
+    if ((kmsvnc->server->cursor->width != rwidth || kmsvnc->server->cursor->height != rheight) || memcmp(kmsvnc->cursor_bitmap, rich_source, rwidth * rheight * BYTES_PER_PIXEL)) {
+        KMSVNC_DEBUG("cursor update %dx%d\n", rwidth, rheight);
+        memcpy(kmsvnc->cursor_bitmap, rich_source, kmsvnc->cursor_bitmap_len);
+        char *cursorString = malloc(rwidth * rheight);
+        if (!cursorString) {
+            free(rich_source);
+            free(maskString);
+            return;
+        }
+
+        memset(cursorString, 'x', rwidth * rheight);
+
+        rfbCursorPtr cursor = rfbMakeXCursor(rwidth, rheight, cursorString, maskString);
+        free(cursorString);
+        cursor->richSource = rich_source;
+        cursor->cleanupRichSource = TRUE;
+        cursor->xhot = 0;
+        cursor->yhot = 0;
+        rfbSetCursor(kmsvnc->server, cursor);
+    }
+    else {
+        free(rich_source);
+        free(maskString);
+    }
+}
+
 static void cleanup() {
     if (kmsvnc->keymap) {
         xkb_cleanup();
@@ -98,6 +185,9 @@ static void cleanup() {
     if (kmsvnc->drm) {
         drm_cleanup();
     }
+    if (kmsvnc->va) {
+        va_cleanup();
+    }
     if (kmsvnc) {
         if (kmsvnc->vnc_opt) {
             free(kmsvnc->vnc_opt);
@@ -111,11 +201,17 @@ static void cleanup() {
             free(kmsvnc->buf);
             kmsvnc->buf = NULL;
         }
+        if (kmsvnc->cursor_bitmap) {
+            free(kmsvnc->cursor_bitmap);
+            kmsvnc->cursor_bitmap = NULL;
+        }
+        kmsvnc->cursor_bitmap_len = 0;
         free(kmsvnc);
         kmsvnc = NULL;
     }
 }
 
+void signal_handler_noop(int signum){}
 void signal_handler(int signum){
     if (kmsvnc->shutdown) {
         return;
@@ -127,7 +223,7 @@ void signal_handler(int signum){
 }
 
 static struct argp_option kmsvnc_main_options[] = {
-    {"device", 'd', "/dev/dri/card0", 0, "DRM device"},
+    {"device", 'd', "/dev/dri/cardX", 0, "DRM device"},
     {"source-plane", 0xfefc, "0", 0, "Use specific plane"},
     {"source-crtc", 0xfefd, "0", 0, "Use specific crtc (to list all crtcs and planes, set this to -1)"},
     {"force-driver", 0xfefe, "i915", 0, "force a certain driver (for debugging)"},
@@ -138,7 +234,20 @@ static struct argp_option kmsvnc_main_options[] = {
     {"fps", 0xff00, "30", 0, "Target frames per second"},
     {"disable-always-shared", 0xff01, 0, OPTION_ARG_OPTIONAL, "Do not always treat incoming connections as shared"},
     {"disable-compare-fb", 0xff02, 0, OPTION_ARG_OPTIONAL, "Do not compare pixels"},
+    {"capture-cursor", 'c', 0, OPTION_ARG_OPTIONAL, "Capture mouse cursor"},
     {"capture-raw-fb", 0xff03, "/tmp/rawfb.bin", 0, "Capture RAW framebuffer instead of starting the vnc server (for debugging)"},
+    {"va-derive", 0xff04, "off", 0, "Enable derive with vaapi"},
+    {"debug", 0xff05, 0, OPTION_ARG_OPTIONAL, "Print debug message"},
+    {"input-width", 0xff06, "0", 0, "Explicitly set input width, normally this is inferred from screen width on a single display system"},
+    {"input-height", 0xff07, "0", 0, "Explicitly set input height"},
+    {"input-offx", 0xff08, "0", 0, "Set input offset of x axis on a multi display system"},
+    {"input-offy", 0xff09, "0", 0, "Set input offset of y axis on a multi display system"},
+#ifndef DISABLE_KMSVNC_SCREEN_BLANK
+    {"screen-blank", 0xff0a, 0, OPTION_ARG_OPTIONAL, "Blank screen with gamma set on crtc"},
+    {"screen-blank-restore-linear", 0xff0b, 0, OPTION_ARG_OPTIONAL, "Restore linear values on exit in case of messed up gamma"},
+#endif
+    {"va-byteorder-swap", 0xff0c, 0, OPTION_ARG_OPTIONAL, "Force swap vaapi image rgb byteorder"},
+    {"wakeup", 'w', 0, OPTION_ARG_OPTIONAL, "Move mouse to wake the system up before start"},
     {"disable-input", 'i', 0, OPTION_ARG_OPTIONAL, "Disable uinput"},
     {"desktop-name", 'n', "kmsvnc", 0, "Specify vnc desktop name"},
     {0}
@@ -169,24 +278,28 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
             kmsvnc->vnc_opt->bind6 = arg;
             break;
         case 'p':
-            int port = atoi(arg);
+            {
-            if (port > 0 && port < 65536) {
+                int port = atoi(arg);
-                kmsvnc->vnc_opt->port = port;
+                if (port > 0 && port < 65536) {
-            }
+                    kmsvnc->vnc_opt->port = port;
-            else {
+                }
-                argp_error(state, "invalid port %s", arg);
+                else {
+                    argp_error(state, "invalid port %s", arg);
+                }
             }
             break;
         case '4':
             kmsvnc->vnc_opt->disable_ipv6 = 1;
             break;
         case 0xff00:
-            int fps = atoi(arg);
+            {
-            if (fps > 0 && fps < 1000) {
+                int fps = atoi(arg);
-                kmsvnc->vnc_opt->sleep_ns = NS_IN_S / fps;
+                if (fps > 0 && fps < 1000) {
-            }
+                    kmsvnc->vnc_opt->sleep_ns = NS_IN_S / fps;
-            else {
+                }
-                argp_error(state, "invalid fps %s", arg);
+                else {
+                    argp_error(state, "invalid fps %s", arg);
+                }
             }
             break;
         case 0xff01:
@@ -195,10 +308,68 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
         case 0xff02:
             kmsvnc->vnc_opt->disable_cmpfb = 1;
             break;
+        case 'c':
+            kmsvnc->capture_cursor = 1;
+            break;
         case 0xff03:
             kmsvnc->debug_capture_fb = arg;
             kmsvnc->disable_input = 1;
             break;
+        case 0xff04:
+            if (!strcmp("on", arg) || !strcmp("y", arg) || !strcmp("yes", arg) || !strcmp("1", arg)) {
+                kmsvnc->va_derive_enabled = 1;
+            }
+            else {
+                kmsvnc->va_derive_enabled = 0;
+            }
+            break;
+        case 0xff05:
+            kmsvnc->debug_enabled = 1;
+            break;
+        case 0xff06:
+            {
+                int width = atoi(arg);
+                if (width > 0) {
+                    kmsvnc->input_width = width;
+                }
+            }
+            break;
+        case 0xff07:
+            {
+                int height = atoi(arg);
+                if (height > 0) {
+                    kmsvnc->input_height = height;
+                }
+            }
+            break;
+        case 0xff08:
+            {
+                int offset_x = atoi(arg);
+                if (offset_x > 0) {
+                    kmsvnc->input_offx = offset_x;
+                }
+            }
+            break;
+        case 0xff09:
+            {
+                int offset_y = atoi(arg);
+                if (offset_y > 0) {
+                    kmsvnc->input_offy = offset_y;
+                }
+            }
+            break;
+        case 0xff0a:
+            kmsvnc->screen_blank = 1;
+            break;
+        case 0xff0b:
+            kmsvnc->screen_blank_restore = 1;
+            break;
+        case 0xff0c:
+            kmsvnc->va_byteorder_swap = 1;
+            break;
+        case 'w':
+            kmsvnc->input_wakeup = 1;
+            break;
         case 'i':
             kmsvnc->disable_input = 1;
             break;
@@ -215,15 +386,24 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
 
 int main(int argc, char **argv)
 {
-    struct vnc_opt *vncopt = malloc(sizeof(struct vnc_opt));
-    memset(vncopt, 0, sizeof(struct vnc_opt));
-
     kmsvnc = malloc(sizeof(struct kmsvnc_data));
+    if (!kmsvnc) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
     memset(kmsvnc, 0, sizeof(struct kmsvnc_data));
 
+    struct vnc_opt *vncopt = malloc(sizeof(struct vnc_opt));
+    if (!vncopt) {
+        free(kmsvnc);
+        KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
+    }
+    memset(vncopt, 0, sizeof(struct vnc_opt));
+
     kmsvnc->vnc_opt = vncopt;
 
-    kmsvnc->card = "/dev/dri/card0";
+    #define DEVICE_EXAMPLE_MAX_SIZE 15
+    #define DEVICE_EXAMPLE_FALLBACK "/dev/dri/card0"
+    static char device_example[DEVICE_EXAMPLE_MAX_SIZE] = DEVICE_EXAMPLE_FALLBACK;
+    kmsvnc->card = device_example;
+    kmsvnc->va_derive_enabled = -1;
     kmsvnc->vnc_opt->bind = &(struct in_addr){0};
     kmsvnc->vnc_opt->always_shared = 1;
     kmsvnc->vnc_opt->port = 5900;
@@ -236,6 +416,18 @@ int main(int argc, char **argv)
     struct argp argp = {kmsvnc_main_options, parse_opt, args_doc, doc};
     argp_parse(&argp, argc, argv, 0, 0, NULL);
 
+    if (kmsvnc->card == device_example) {
+        for (int i = 0; i < 10; i++) {
+            snprintf(kmsvnc->card, DEVICE_EXAMPLE_MAX_SIZE, "/dev/dri/card%d", i);
+            if (!access(kmsvnc->card, F_OK)) {
+                break;
+            }
+            else {
+                snprintf(kmsvnc->card, DEVICE_EXAMPLE_MAX_SIZE, DEVICE_EXAMPLE_FALLBACK);
+            }
+        }
+    }
+
     if (!kmsvnc->disable_input) {
         const char* XKB_DEFAULT_LAYOUT = getenv("XKB_DEFAULT_LAYOUT");
         if (!XKB_DEFAULT_LAYOUT || strcmp(XKB_DEFAULT_LAYOUT, "") == 0) {
@@ -260,22 +452,58 @@ int main(int argc, char **argv)
 
     if (kmsvnc->debug_capture_fb) {
         int wfd = open(kmsvnc->debug_capture_fb, O_WRONLY | O_CREAT, 00644);
+        int max_size = 0;
+        for (int i = 0; i < 4; i++) {
+            int size = kmsvnc->drm->mfb->offsets[i] + kmsvnc->drm->mfb->height * kmsvnc->drm->mfb->pitches[i];
+            if (size > max_size) max_size = size;
+        }
+        printf("attempt to write %d bytes\n", max_size);
         if (wfd > 0) {
-            write(wfd, kmsvnc->drm->mapped, kmsvnc->drm->mfb->width * kmsvnc->drm->mfb->height * BYTES_PER_PIXEL);
+            if (kmsvnc->va) {
+                if (!kmsvnc->drm->mapped) kmsvnc->drm->mapped = malloc(max_size);
+                if (!kmsvnc->drm->mapped) {
+                    cleanup();
+                    KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
+                }
+                va_hwframe_to_vaapi(kmsvnc->drm->mapped);
+            }
+            KMSVNC_WRITE_MAY(wfd, kmsvnc->drm->mapped, (ssize_t)max_size);
             fsync(wfd);
             printf("wrote raw frame buffer to %s\n", kmsvnc->debug_capture_fb);
         }
         else {
             fprintf(stderr, "open file %s failed, %s\n", kmsvnc->debug_capture_fb, strerror(errno));
         }
+        if (kmsvnc->screen_blank) {
+            sigset_t signal_set;
+            int sig;
+            sigemptyset(&signal_set);
+            signal(SIGHUP, &signal_handler_noop);
+            signal(SIGINT, &signal_handler_noop);
+            signal(SIGTERM, &signal_handler_noop);
+            sigaddset(&signal_set, SIGHUP);
+            sigaddset(&signal_set, SIGINT);
+            sigaddset(&signal_set, SIGTERM);
+            fprintf(stderr, "blanking screen...\n");
+            sigwait(&signal_set, &sig);
+            fprintf(stderr, "got sig %d\n", sig);
+        }
         cleanup();
         return 0;
     }
 
     size_t buflen = kmsvnc->drm->mfb->width * kmsvnc->drm->mfb->height * BYTES_PER_PIXEL;
     kmsvnc->buf = malloc(buflen);
+    if (!kmsvnc->buf) {
+        cleanup();
+        KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
+    }
     memset(kmsvnc->buf, 0, buflen);
     kmsvnc->buf1 = malloc(buflen);
+    if (!kmsvnc->buf1) {
+        cleanup();
+        KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
+    }
     memset(kmsvnc->buf1, 0, buflen);
 
     signal(SIGHUP, &signal_handler);
@@ -300,6 +528,7 @@ int main(int argc, char **argv)
     }
     rfbInitServer(kmsvnc->server);
     rfbRunEventLoop(kmsvnc->server, -1, TRUE);
+    int cursor_frame = 0;
     while (rfbIsActive(kmsvnc->server))
     {
         between_frames();
@@ -309,6 +538,18 @@ int main(int argc, char **argv)
             kmsvnc->drm->funcs->convert(kmsvnc->drm->mapped, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, kmsvnc->buf1);
             kmsvnc->drm->funcs->sync_end(kmsvnc->drm->prime_fd);
             update_screen_buf(kmsvnc->buf, kmsvnc->buf1, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height);
+            if (kmsvnc->capture_cursor) {
+                cursor_frame++;
+                cursor_frame %= CURSOR_FRAMESKIP;
+                if (!cursor_frame) {
+                    char *data = NULL;
+                    int width, height;
+                    int err = drm_dump_cursor_plane(&data, &width, &height);
+                    if (!err && data) {
+                        update_vnc_cursor(data, width, height);
+                    }
+                }
+            }
         }
     }
     cleanup();

kmsvnc.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <rfb/rfb.h>
+#include <stdint.h>
 #include <xkbcommon/xkbcommon.h>
 
 #include <xf86drm.h>
@@ -8,9 +9,11 @@
 #include <amdgpu_drm.h>
 #include <xf86drmMode.h>
 #include <linux/dma-buf.h>
+#include <va/va.h>
 
 
 #define BYTES_PER_PIXEL 4
+#define CURSOR_FRAMESKIP 15
 
 struct vnc_opt
 {
@@ -30,14 +33,28 @@ struct kmsvnc_data
     char *card;
     char *force_driver;
     struct vnc_opt *vnc_opt;
+    char input_wakeup;
     char disable_input;
+    int va_derive_enabled;
+    char debug_enabled;
     int source_plane;
     int source_crtc;
+    int input_width;
+    int input_height;
+    int input_offx;
+    int input_offy;
+    char screen_blank;
+    char screen_blank_restore;
+    char va_byteorder_swap;
     struct kmsvnc_drm_data *drm;
     struct kmsvnc_input_data *input;
     struct kmsvnc_keymap_data *keymap;
+    struct kmsvnc_va_data *va;
     rfbScreenInfoPtr server;
     char shutdown;
+    char capture_cursor;
+    char *cursor_bitmap;
+    int cursor_bitmap_len;
     char *buf;
     char *buf1;
 };
@@ -72,19 +89,66 @@ struct kmsvnc_drm_funcs
     void (*convert)(const char *, int, int, char *);
 };
 
+struct kmsvnc_drm_gamma_data
+{
+    uint32_t size;
+    uint16_t *red;
+    uint16_t *green;
+    uint16_t *blue;
+};
+
 struct kmsvnc_drm_data
 {
     int drm_fd;
+    int drm_master_fd;
     drmVersionPtr drm_ver;
     int prime_fd;
     drmModePlane *plane;
+    drmModePlane *cursor_plane;
     drmModePlaneRes *plane_res;
     drmModeFB2 *mfb;
-    u_int32_t plane_id;
+    drmModeFB2 *cursor_mfb;
+    uint32_t plane_id;
     int mmap_fd;
     size_t mmap_size;
     off_t mmap_offset;
     char *mapped;
+    char *cursor_mapped;
+    size_t cursor_mmap_size;
+    char skip_map;
     struct kmsvnc_drm_funcs *funcs;
+    char *pixfmt_name;
+    char *mod_vendor;
+    char *mod_name;
+    char *kms_convert_buf;
+    size_t kms_convert_buf_len;
+    char *kms_cpy_tmp_buf;
+    size_t kms_cpy_tmp_buf_len;
+    char *kms_cursor_buf;
+    size_t kms_cursor_buf_len;
+    struct kmsvnc_drm_gamma_data *gamma;
 };
 
+struct kmsvnc_va_data
+{
+    VADisplay dpy;
+    int render_node_fd;
+    VASurfaceID surface_id;
+    VAImage *image;
+    char *imgbuf;
+    char derive_enabled;
+    VAImageFormat* img_fmts;
+    int img_fmt_count;
+    VAImageFormat* selected_fmt;
+    const char *vendor_string;
+};
+
+#define KMSVNC_FATAL(...) do{ fprintf(stderr, __VA_ARGS__); return 1; } while(0)
+#define KMSVNC_ARRAY_ELEMENTS(x) (sizeof(x) / sizeof(x[0]))
+#define KMSVNC_FOURCC_TO_INT(a,b,c,d) (((a) << 0) + ((b) << 8) + ((c) << 16) + ((d) << 24))
+#define KMSVNC_WRITE_MAY(fd,buf,count) do { ssize_t e = write((fd), (buf), (count)); if (e != (count)) fprintf(stderr, "should write %ld bytes, actually wrote %ld, on line %d\n", (count), e, __LINE__); } while (0)
+
+#define KMSVNC_DEBUG(...) do{ if (kmsvnc->debug_enabled) fprintf(stdout, __VA_ARGS__); } while(0)
+
+#define likely(x)   __builtin_expect(!!(x), 1)
+#define unlikely(x) __builtin_expect(!!(x), 0)

va.c

@@ -0,0 +1,397 @@
+#define _GNU_SOURCE
+
+#include <stdint.h>
+#include <stdio.h>
+#include <va/va.h>
+#include <va/va_drm.h>
+#include <va/va_drmcommon.h>
+#include <fcntl.h>
+#include <string.h>
+
+#include "va.h"
+#include "kmsvnc.h"
+
+extern struct kmsvnc_data *kmsvnc;
+
+void va_cleanup() {
+    VAStatus s;
+    if (kmsvnc->va) {
+        if (kmsvnc->va->img_fmts) {
+            free(kmsvnc->va->img_fmts);
+            kmsvnc->va->img_fmts = NULL;
+        }
+        if (kmsvnc->va->imgbuf) {
+            VA_MAY(vaUnmapBuffer(kmsvnc->va->dpy, kmsvnc->va->image->buf));
+            kmsvnc->va->imgbuf = NULL;
+        }
+        if (kmsvnc->va->image) {
+            if ((s = vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id)) == VA_STATUS_SUCCESS) {
+                free(kmsvnc->va->image);
+            }
+            VA_MAY(s);
+            kmsvnc->va->image = NULL;
+        }
+        if (kmsvnc->va->surface_id > 0) {
+            VA_MAY(vaDestroySurfaces(kmsvnc->va->dpy, &kmsvnc->va->surface_id, 1));
+            kmsvnc->va->surface_id = 0;
+        }
+        if (kmsvnc->va->dpy) {
+            VA_MAY(vaTerminate(kmsvnc->va->dpy));
+            kmsvnc->va->dpy = NULL;
+        }
+        if (kmsvnc->va->vendor_string) {
+            kmsvnc->va->vendor_string = NULL;
+        }
+        free(kmsvnc->va);
+        kmsvnc->va = NULL;
+    }
+}
+
+static void va_msg_callback(void *user_context, const char *message) {
+    if (kmsvnc->debug_enabled) {
+        printf("va msg: %s", message);
+    }
+}
+
+static void va_error_callback(void *user_context, const char *message) {
+    printf("va error: %s", message);
+}
+
+static char* fourcc_to_str(int fourcc) {
+    static char ret[5];
+    ret[4] = 0;
+    for (int i = 0; i < 4; i++) {
+        ret[i] = fourcc >> 8*i & 0xff;
+    }
+    return ret;
+}
+
+static const struct {
+    uint32_t drm_fourcc;
+    uint32_t va_fourcc;
+    uint32_t va_rt_format;
+    char alpha;
+} va_format_map[] = {
+    {KMSVNC_FOURCC_TO_INT('X', 'R', '2', '4'), KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'X'), VA_RT_FORMAT_RGB32, 0},
+    {KMSVNC_FOURCC_TO_INT('A', 'R', '2', '4'), KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'A'), VA_RT_FORMAT_RGB32, 1},
+    {KMSVNC_FOURCC_TO_INT('X', 'R', '3', '0'), KMSVNC_FOURCC_TO_INT('X', 'R', '3', '0'), VA_RT_FORMAT_RGB32_10, 0},
+    {KMSVNC_FOURCC_TO_INT('A', 'R', '3', '0'), KMSVNC_FOURCC_TO_INT('A', 'R', '3', '0'), VA_RT_FORMAT_RGB32_10, 1},
+};
+
+struct va_fmt_data {
+    uint32_t va_fourcc;
+    VAImageFormat *fmt;
+    char is_alpha;
+    uint32_t va_rt_format;
+    uint32_t depth;
+};
+
+static VAImageFormat* vaImgFmt_apply_quirks(struct va_fmt_data* data) {
+    static VAImageFormat ret = {0};
+    memcpy(&ret, data->fmt, sizeof(VAImageFormat));
+    if ((kmsvnc->va_byteorder_swap ^ !strncmp(kmsvnc->va->vendor_string, "Mesa", 4)) && data->depth != 30) {
+        printf("applying rgb mask byte order swap\n");
+        ret.blue_mask = __builtin_bswap32(data->fmt->blue_mask);
+        ret.green_mask = __builtin_bswap32(data->fmt->green_mask);
+        ret.red_mask = __builtin_bswap32(data->fmt->red_mask);
+    }
+    return &ret;
+}
+
+static void print_va_image_fmt(VAImageFormat *fmt) {
+    printf("image fmt: fourcc %d, %s, byte_order %s, bpp %d, depth %d, blue_mask %#x, green_mask %#x, red_mask %#x, alpha_mask %#x, reserved %#x %#x %#x %#x\n", fmt->fourcc,
+        fourcc_to_str(fmt->fourcc),
+        fmt->byte_order == 1 ? "VA_LSB_FIRST" : "VA_MSB_FIRST",
+        fmt->bits_per_pixel,
+        fmt->depth,
+        fmt->blue_mask,
+        fmt->green_mask,
+        fmt->red_mask,
+        fmt->alpha_mask,
+        fmt->va_reserved[0],
+        fmt->va_reserved[1],
+        fmt->va_reserved[2],
+        fmt->va_reserved[3]
+    );
+}
+
+int va_init() {
+    if (!kmsvnc->drm || !kmsvnc->drm->drm_fd || !kmsvnc->drm->prime_fd) {
+        KMSVNC_FATAL("drm is not initialized\n");
+    }
+
+    setenv("DISPLAY", "", 1);
+    setenv("WAYLAND_DISPLAY", "", 1);
+
+    struct kmsvnc_va_data *va = malloc(sizeof(struct kmsvnc_va_data));
+    if (!va) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
+    memset(va, 0, sizeof(struct kmsvnc_va_data));
+    kmsvnc->va = va;
+
+    char* render_node;
+    int effective_fd = 0;
+    if ((render_node = drmGetRenderDeviceNameFromFd(kmsvnc->drm->drm_fd))) {
+        va->render_node_fd = open(render_node, O_RDWR);
+        free(render_node);
+    }
+    else {
+        printf("Using non-render node because the device does not have an associated render node.\n");
+    }
+    if (va->render_node_fd > 0) {
+        effective_fd = va->render_node_fd;
+    }
+    else {
+        printf("Using non-render node because render node fails to open.\n");
+        effective_fd = kmsvnc->drm->drm_fd;
+    }
+
+    va->dpy = vaGetDisplayDRM(effective_fd);
+    if (!va->dpy) {
+        KMSVNC_FATAL("vaGetDisplayDRM failed\n");
+    }
+
+    vaSetErrorCallback(va->dpy, &va_error_callback, NULL);
+    vaSetInfoCallback(va->dpy, &va_msg_callback, NULL);
+
+    int major, minor;
+    VAStatus status;
+    VA_MUST(vaInitialize(va->dpy, &major, &minor));
+
+    va->vendor_string = vaQueryVendorString(va->dpy);
+    printf("vaapi vendor %s\n", va->vendor_string);
+
+    VADRMPRIMESurfaceDescriptor prime_desc;
+    VASurfaceAttrib prime_attrs[2] = {
+        {
+            .type  = VASurfaceAttribMemoryType,
+            .flags = VA_SURFACE_ATTRIB_SETTABLE,
+            .value.type    = VAGenericValueTypeInteger,
+            .value.value.i = VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2,
+        },
+        {
+            .type  = VASurfaceAttribExternalBufferDescriptor,
+            .flags = VA_SURFACE_ATTRIB_SETTABLE,
+            .value.type    = VAGenericValueTypePointer,
+            .value.value.p = &prime_desc,
+        }
+    };
+
+    uint32_t rt_format = 0;
+    char is_alpha = 0;
+    for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(va_format_map); i++) {
+        if (kmsvnc->drm->mfb->pixel_format == va_format_map[i].drm_fourcc) {
+            prime_desc.fourcc = va_format_map[i].va_fourcc;
+            rt_format = va_format_map[i].va_rt_format;
+            is_alpha = va_format_map[i].alpha;
+            break;
+        }
+    }
+    if (!rt_format) {
+        KMSVNC_FATAL("Unsupported pixfmt %s for vaapi, please create an issue with your pixfmt.", kmsvnc->drm->pixfmt_name);
+    }
+    if (kmsvnc->debug_enabled) {
+        printf("selected rt_format %u, alpha %d\n", rt_format, is_alpha);
+    }
+    prime_desc.width = kmsvnc->drm->mfb->width;
+    prime_desc.height = kmsvnc->drm->mfb->height;
+
+    int i;
+    int max_size = 0;
+    for (i = 0; i < 4; i++) {
+        int size = kmsvnc->drm->mfb->offsets[i] + kmsvnc->drm->mfb->height * kmsvnc->drm->mfb->pitches[i];
+        if (size > max_size) max_size = size;
+    }
+    for (i = 0; i < 4; i++) {
+        prime_desc.objects[i].fd = kmsvnc->drm->prime_fd;
+        prime_desc.objects[i].size = max_size;
+        prime_desc.objects[i].drm_format_modifier = kmsvnc->drm->mfb->modifier;
+    }
+
+    prime_desc.num_layers = 1;
+    prime_desc.layers[0].drm_format = kmsvnc->drm->mfb->pixel_format;
+    for (i = 0; i < 4; i++) {
+        prime_desc.layers[0].object_index[i] = 0;
+        prime_desc.layers[0].offset[i] = kmsvnc->drm->mfb->offsets[i];
+        prime_desc.layers[0].pitch[i] = kmsvnc->drm->mfb->pitches[i];
+    }
+    for (i = 0; i < 4; i++) {
+        if (!kmsvnc->drm->mfb->handles[i]) {
+            break;
+        }
+    }
+    prime_desc.layers[0].num_planes = i;
+    prime_desc.num_objects = 1;
+
+    VAStatus s;
+    if ((s = vaCreateSurfaces(va->dpy, rt_format,
+                            kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, &va->surface_id, 1,
+                            prime_attrs, KMSVNC_ARRAY_ELEMENTS(prime_attrs))) != VA_STATUS_SUCCESS)
+    {
+        printf("vaCreateSurfaces prime2 error %#x %s, trying prime\n", s, vaErrorStr(s));
+
+        VASurfaceAttribExternalBuffers buffer_desc;
+        VASurfaceAttrib buffer_attrs[2] = {
+            {
+                .type  = VASurfaceAttribMemoryType,
+                .flags = VA_SURFACE_ATTRIB_SETTABLE,
+                .value.type    = VAGenericValueTypeInteger,
+                .value.value.i = VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME,
+            },
+            {
+                .type  = VASurfaceAttribExternalBufferDescriptor,
+                .flags = VA_SURFACE_ATTRIB_SETTABLE,
+                .value.type    = VAGenericValueTypePointer,
+                .value.value.p = &buffer_desc,
+            }
+        };
+
+        unsigned long fd = kmsvnc->drm->prime_fd;
+
+        buffer_desc.pixel_format = prime_desc.fourcc;
+        buffer_desc.width        = kmsvnc->drm->mfb->width;
+        buffer_desc.height       = kmsvnc->drm->mfb->height;
+        buffer_desc.data_size    = max_size;
+        buffer_desc.buffers      = &fd;
+        buffer_desc.num_buffers  = 1;
+        buffer_desc.flags        = 0;
+
+        for (i = 0; i < 4; i++) {
+            buffer_desc.pitches[i] = kmsvnc->drm->mfb->pitches[i];
+            buffer_desc.offsets[i] = kmsvnc->drm->mfb->offsets[i];
+        }
+        buffer_desc.num_planes = prime_desc.layers[0].num_planes;
+
+
+        VA_MUST(vaCreateSurfaces(va->dpy, rt_format,
+                kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, &va->surface_id, 1,
+                buffer_attrs, KMSVNC_ARRAY_ELEMENTS(buffer_attrs)));
+    }
+
+
+    va->img_fmt_count = vaMaxNumImageFormats(va->dpy);
+    va->img_fmts = malloc(sizeof(VAImageFormat) * va->img_fmt_count);
+    if (!va->img_fmts) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
+    {
+        int got;
+        vaQueryImageFormats(va->dpy, va->img_fmts, &got);
+        if (got != va->img_fmt_count) {
+            printf("got less VAImageFormats, %d instead of %d\n", got, va->img_fmt_count);
+            va->img_fmt_count = got;
+        }
+    }
+
+    if (kmsvnc->debug_enabled) {
+        for (int i = 0; i < va->img_fmt_count; i++) {
+            print_va_image_fmt(va->img_fmts + i);
+        }
+    }
+
+    struct va_fmt_data format_to_try[] = {
+        {KMSVNC_FOURCC_TO_INT('R','G','B','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24},
+        {KMSVNC_FOURCC_TO_INT('R','G','B','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32},
+
+        {KMSVNC_FOURCC_TO_INT('X','B','G','R'), NULL, 0, VA_RT_FORMAT_RGB32, 24},
+        {KMSVNC_FOURCC_TO_INT('A','B','G','R'), NULL, 1, VA_RT_FORMAT_RGB32, 32},
+
+        {KMSVNC_FOURCC_TO_INT('X','R','G','B'), NULL, 0, VA_RT_FORMAT_RGB32, 24},
+        {KMSVNC_FOURCC_TO_INT('A','R','G','B'), NULL, 1, VA_RT_FORMAT_RGB32, 32},
+
+        {KMSVNC_FOURCC_TO_INT('B','G','R','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24},
+        {KMSVNC_FOURCC_TO_INT('B','G','R','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32},
+
+
+        {KMSVNC_FOURCC_TO_INT('X','R','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30},
+        {KMSVNC_FOURCC_TO_INT('A','R','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30},
+        {KMSVNC_FOURCC_TO_INT('X','B','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30},
+        {KMSVNC_FOURCC_TO_INT('A','B','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30},
+    };
+
+    for (int i = 0; i < va->img_fmt_count; i++) {
+        for (int j = 0; j < KMSVNC_ARRAY_ELEMENTS(format_to_try); j++) {
+            if (va->img_fmts[i].fourcc == format_to_try[j].va_fourcc) {
+                format_to_try[j].fmt = va->img_fmts + i;
+            }
+        }
+    }
+
+    va->image = malloc(sizeof(VAImage));
+    if (!va->image) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
+
+    va->derive_enabled = 0;
+    va->derive_enabled = kmsvnc->va_derive_enabled < 0 ? va->derive_enabled : kmsvnc->va_derive_enabled != 0;
+    if (va->derive_enabled) {
+        if ((s = vaDeriveImage(va->dpy, va->surface_id, va->image)) == VA_STATUS_SUCCESS) {
+            for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(format_to_try); i++) {
+                if (format_to_try[i].fmt == NULL) continue;
+                if (va->image->format.fourcc == format_to_try[i].fmt->fourcc) {
+                    va->selected_fmt = vaImgFmt_apply_quirks(format_to_try + i);
+                    break;
+                }
+            }
+            if (!va->selected_fmt) {
+                va->derive_enabled = 0;
+                printf("vaDeriveImage returned unknown fourcc %d %s\n", va->image->format.fourcc, fourcc_to_str(va->image->format.fourcc));
+                VA_MAY(vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id));
+            }
+        }
+        VA_MAY(s);
+    }
+    if (va->derive_enabled) {
+        if ((s = vaMapBuffer(va->dpy, va->image->buf, (void**)&va->imgbuf)) != VA_STATUS_SUCCESS) {
+            VA_MAY(s);
+            VA_MAY(vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id));
+            va->derive_enabled = 0;
+        }
+    }
+    if (!va->derive_enabled) {
+        for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(format_to_try); i++) {
+            if (format_to_try[i].fmt == NULL) continue;
+            if (!kmsvnc->debug_enabled && rt_format != format_to_try[i].va_rt_format) continue;
+            if (is_alpha != format_to_try[i].is_alpha) continue;
+
+            VAImageFormat *fmt = format_to_try[i].fmt;
+            if ((s = vaCreateImage(va->dpy, fmt, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, va->image)) != VA_STATUS_SUCCESS) {
+                VA_MAY(s);
+                continue;
+            }
+            if ((s = vaMapBuffer(va->dpy, va->image->buf, (void**)&va->imgbuf)) != VA_STATUS_SUCCESS) {
+                VA_MAY(s);
+                VA_MAY(vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id));
+                continue;
+            }
+            if ((s = vaGetImage(kmsvnc->va->dpy, kmsvnc->va->surface_id, 0, 0,
+                    kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height,
+                    kmsvnc->va->image->image_id)) != VA_STATUS_SUCCESS)
+            {
+                VA_MAY(s);
+                VA_MAY(vaUnmapBuffer(kmsvnc->va->dpy, kmsvnc->va->image->buf));
+                VA_MAY(vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id));
+                continue;
+            }
+            else {
+                va->selected_fmt = vaImgFmt_apply_quirks(format_to_try + i);
+                break;
+            }
+        }
+        if (!va->selected_fmt) {
+            va->imgbuf = NULL;
+            KMSVNC_FATAL("failed to get vaapi image\n");
+        }
+    }
+    printf("got vaapi %simage:\n", va->derive_enabled ? "derive " : "");
+    print_va_image_fmt(&va->image->format);
+    if (kmsvnc->debug_enabled) {
+        fprintf(stderr, "selected image format:\n");
+        print_va_image_fmt(va->selected_fmt);
+    }
+    return 0;
+}
+
+int va_hwframe_to_vaapi(char *out) {
+    if (!kmsvnc->va->derive_enabled) {
+        VA_MUST(vaGetImage(kmsvnc->va->dpy, kmsvnc->va->surface_id, 0, 0,
+                kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, kmsvnc->va->image->image_id));
+    }
+    memcpy(out, kmsvnc->va->imgbuf, kmsvnc->drm->mfb->width * kmsvnc->drm->mfb->height * BYTES_PER_PIXEL);
+    return 0;
+}

va.h

@@ -0,0 +1,8 @@
+#pragma once
+
+#define VA_MUST(x) do{VAStatus _s; if ((_s = (x)) != VA_STATUS_SUCCESS) KMSVNC_FATAL("va operation error %#x %s on line %d\n", _s, vaErrorStr(_s), __LINE__); } while (0)
+#define VA_MAY(x) do{VAStatus _s; if ((_s = (x)) != VA_STATUS_SUCCESS) fprintf(stderr, "va operation error %#x %s on line %d\n", _s, vaErrorStr(_s), __LINE__); } while (0)
+
+void va_cleanup();
+int va_init();
+int va_hwframe_to_vaapi(char *out);