panfrost/ci: Initial commit

These files implement running almost all of deqp-gles2 on Chomebooks of the rk3399-gru-kevin type in Collabora's LAVA lab. The approach follows what is currently being used for virglrenderer, but scheduling the actual test jobs via LAVA. We start by building a container in Docker that contains a suitable rootfs and kernel for the DUT, deqp and all dependencies for building Mesa itself. The Mesa is built and the rootfs, deqp and Mesa are combined in a cpio ramdisk. A LAVA job is generated, submitted to LAVA and the results are processed by simply comparing them to the expectations that are stored in git. Any code that changes the expectations (hopefully tests are fixed) needs to also update the expectations file. The next step is adding support for other devices, possibly in other LAVA labs. In order to use this, the repository has to be configured to run the gitlab-ci.yaml file from the panfrost/ci dir, and a LAVA token needs to be setup. Signed-off-by: Tomeu Vizoso <tomeu.vizoso@collabora.com> Reviewed-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
2024-11-28 04:34:25 +08:00 · 2019-04-29 16:33:22 +00:00 · 2019-04-29 16:33:22 +00:00 · cc6bbf6397
commit cc6bbf6397
parent b15f5cfd20
8 changed files with 3248 additions and 0 deletions
--- a/src/gallium/drivers/panfrost/ci/Dockerfile
+++ b/src/gallium/drivers/panfrost/ci/Dockerfile
@ -0,0 +1,121 @@
+FROM debian:testing
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN echo 'path-exclude=/usr/share/doc/*' > /etc/dpkg/dpkg.cfg.d/99-exclude-cruft
+RUN echo 'path-exclude=/usr/share/man/*' >> /etc/dpkg/dpkg.cfg.d/99-exclude-cruft
+RUN echo '#!/bin/sh' > /usr/sbin/policy-rc.d
+RUN echo 'exit 101' >> /usr/sbin/policy-rc.d
+RUN chmod +x /usr/sbin/policy-rc.d
+
+RUN dpkg --add-architecture arm64
+RUN echo deb-src http://deb.debian.org/debian testing main >> /etc/apt/sources.list
+RUN apt-get update && \
+    apt-get -y install ca-certificates && \
+    apt-get -y install --no-install-recommends \
+      crossbuild-essential-arm64 \
+      meson \
+      g++ \
+      git \
+      ccache \
+      pkg-config \
+      python3-mako \
+      python-numpy \
+      python-six \
+      python-mako \
+      python3-pip \
+      python3-setuptools \
+      python3-six \
+      python3-wheel \
+      python3-jinja2 \
+      bison \
+      flex \
+      libwayland-dev \
+      gettext \
+      cmake \
+      bc \
+      libssl-dev \
+      lavacli \
+      csvkit \
+      curl \
+      unzip \
+      wget \
+      debootstrap \
+      procps \
+      qemu-user-static \
+      cpio \
+      \
+      libdrm-dev:arm64 \
+      libx11-dev:arm64 \
+      libxxf86vm-dev:arm64 \
+      libexpat1-dev:arm64 \
+      libsensors-dev:arm64 \
+      libxfixes-dev:arm64 \
+      libxdamage-dev:arm64 \
+      libxext-dev:arm64 \
+      x11proto-dev:arm64 \
+      libx11-xcb-dev:arm64 \
+      libxcb-dri2-0-dev:arm64 \
+      libxcb-glx0-dev:arm64 \
+      libxcb-xfixes0-dev:arm64 \
+      libxcb-dri3-dev:arm64 \
+      libxcb-present-dev:arm64 \
+      libxcb-randr0-dev:arm64 \
+      libxcb-sync-dev:arm64 \
+      libxrandr-dev:arm64 \
+      libxshmfence-dev:arm64 \
+      libelf-dev:arm64 \
+      libwayland-dev:arm64 \
+      libwayland-egl-backend-dev:arm64 \
+      libclang-7-dev:arm64 \
+      zlib1g-dev:arm64 \
+      libglvnd-core-dev:arm64 \
+      wayland-protocols:arm64 \
+      libpng-dev:arm64 && \
+    rm -rf /var/lib/apt/lists
+
+RUN mkdir -p /artifacts/rootfs/deqp                                             && \
+  wget https://github.com/KhronosGroup/VK-GL-CTS/archive/opengl-es-cts-3.2.5.0.zip && \
+  unzip opengl-es-cts-3.2.5.0.zip -d /                                          && \
+  rm opengl-es-cts-3.2.5.0.zip                                                  && \
+  cd /VK-GL-CTS-opengl-es-cts-3.2.5.0                                           && \
+  python3 external/fetch_sources.py                                             && \
+  cd /artifacts/rootfs/deqp                                                     && \
+  cmake -DDEQP_TARGET=wayland                                                      \
+    -DCMAKE_BUILD_TYPE=Release                                                     \
+    -DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc                                       \
+    -DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++                                     \
+    /VK-GL-CTS-opengl-es-cts-3.2.5.0                                            && \
+  make -j$(nproc)                                                               && \
+  rm -rf /artifacts/rootfs/deqp/external                                        && \
+  rm -rf /artifacts/rootfs/deqp/modules/gles3                                   && \
+  rm -rf /artifacts/rootfs/deqp/modules/gles31                                  && \
+  rm -rf /artifacts/rootfs/deqp/modules/internal                                && \
+  rm -rf /artifacts/rootfs/deqp/executor                                        && \
+  rm -rf /artifacts/rootfs/deqp/execserver                                      && \
+  rm -rf /artifacts/rootfs/deqp/modules/egl                                     && \
+  rm -rf /artifacts/rootfs/deqp/framework                                       && \
+  find . -name CMakeFiles | xargs rm -rf                                        && \
+  find . -name lib\*.a | xargs rm -rf                                           && \
+  du -sh *                                                                      && \
+  rm -rf /VK-GL-CTS-opengl-es-cts-3.2.5.0
+
+# TODO: Switch to 5.2-rc* when the time comes
+COPY arm64.config /panfrost-ci/
+RUN mkdir -p /kernel                                                                   && \
+  wget https://github.com/freedesktop/drm-misc/archive/drm-misc-next-2019-04-18.tar.gz && \
+  tar xvfz drm-misc-next-2019-04-18.tar.gz -C /kernel --strip-components=1             && \
+  rm drm-misc-next-2019-04-18.tar.gz                                                   && \
+  cd /kernel                                                                           && \
+  ARCH=arm64 CROSS_COMPILE="aarch64-linux-gnu-" ./scripts/kconfig/merge_config.sh arch/arm64/configs/defconfig /panfrost-ci/arm64.config && \
+  ARCH=arm64 CROSS_COMPILE="aarch64-linux-gnu-" make -j12 Image dtbs                   && \
+  cp arch/arm64/boot/Image /artifacts/.                                                && \
+  cp arch/arm64/boot/dts/rockchip/rk3399-gru-kevin.dtb /artifacts/.                    && \
+  rm -rf /kernel
+
+COPY create-rootfs.sh /artifacts/rootfs/
+RUN debootstrap --variant=minbase --arch=arm64 testing /artifacts/rootfs/ http://deb.debian.org/debian && \
+    chroot /artifacts/rootfs sh /create-rootfs.sh                                                      && \
+    rm /artifacts/rootfs/create-rootfs.sh
+
+ENTRYPOINT [""]
--- a/src/gallium/drivers/panfrost/ci/arm64.config
+++ b/src/gallium/drivers/panfrost/ci/arm64.config
@ -0,0 +1,82 @@
+CONFIG_LOCALVERSION="ccu"
+
+CONFIG_DEBUG_KERNEL=y
+
+CONFIG_DEVFREQ_GOV_PERFORMANCE=y
+CONFIG_DEVFREQ_GOV_POWERSAVE=y
+CONFIG_DEVFREQ_GOV_USERSPACE=y
+CONFIG_DEVFREQ_GOV_PASSIVE=y
+
+CONFIG_DRM=y
+CONFIG_DRM_ROCKCHIP=y
+CONFIG_DRM_PANFROST=y
+CONFIG_DRM_PANEL_SIMPLE=y
+CONFIG_PWM_CROS_EC=y
+CONFIG_BACKLIGHT_PWM=y
+
+CONFIG_ROCKCHIP_CDN_DP=n
+
+CONFIG_SPI_ROCKCHIP=y
+CONFIG_PWM_ROCKCHIP=y
+CONFIG_PHY_ROCKCHIP_DP=y
+CONFIG_DWMAC_ROCKCHIP=y
+CONFIG_STMMAC_ETH=y
+CONFIG_TYPEC_FUSB302=y
+CONFIG_TYPEC=y
+CONFIG_TYPEC_TCPM=y
+
+CONFIG_ARCH_SUNXI=n
+CONFIG_ARCH_ALPINE=n
+CONFIG_ARCH_BCM2835=n
+CONFIG_ARCH_BCM_IPROC=n
+CONFIG_ARCH_BERLIN=n
+CONFIG_ARCH_BRCMSTB=n
+CONFIG_ARCH_EXYNOS=n
+CONFIG_ARCH_K3=n
+CONFIG_ARCH_LAYERSCAPE=n
+CONFIG_ARCH_LG1K=n
+CONFIG_ARCH_HISI=n
+CONFIG_ARCH_MEDIATEK=n
+CONFIG_ARCH_MESON=n
+CONFIG_ARCH_MVEBU=n
+CONFIG_ARCH_QCOM=n
+CONFIG_ARCH_SEATTLE=n
+CONFIG_ARCH_SYNQUACER=n
+CONFIG_ARCH_RENESAS=n
+CONFIG_ARCH_R8A774A1=n
+CONFIG_ARCH_R8A774C0=n
+CONFIG_ARCH_R8A7795=n
+CONFIG_ARCH_R8A7796=n
+CONFIG_ARCH_R8A77965=n
+CONFIG_ARCH_R8A77970=n
+CONFIG_ARCH_R8A77980=n
+CONFIG_ARCH_R8A77990=n
+CONFIG_ARCH_R8A77995=n
+CONFIG_ARCH_STRATIX10=n
+CONFIG_ARCH_TEGRA=n
+CONFIG_ARCH_SPRD=n
+CONFIG_ARCH_THUNDER=n
+CONFIG_ARCH_THUNDER2=n
+CONFIG_ARCH_UNIPHIER=n
+CONFIG_ARCH_VEXPRESS=n
+CONFIG_ARCH_XGENE=n
+CONFIG_ARCH_ZX=n
+CONFIG_ARCH_ZYNQMP=n
+
+CONFIG_ACPI=n
+
+CONFIG_REGULATOR_FAN53555=y
+CONFIG_REGULATOR=y
+
+CONFIG_REGULATOR_VCTRL=y
+
+CONFIG_KASAN=n
+CONFIG_KASAN_INLINE=n
+CONFIG_STACKTRACE=n
+
+CONFIG_TMPFS=y
+
+CONFIG_PROVE_LOCKING=n
+CONFIG_DEBUG_LOCKDEP=n
+CONFIG_SOFTLOCKUP_DETECTOR=n
+CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=n
--- a/src/gallium/drivers/panfrost/ci/create-rootfs.sh
+++ b/src/gallium/drivers/panfrost/ci/create-rootfs.sh
@ -0,0 +1,185 @@
+#!/bin/sh
+
+set -ex
+
+apt-get -y install --no-install-recommends initramfs-tools libpng16-16 weston strace libsensors5
+passwd root -d
+chsh -s /bin/sh
+ln -s /bin/sh /init
+
+#######################################################################
+# Strip the image to a small minimal system without removing the debian
+# toolchain.
+
+# Copy timezone file and remove tzdata package
+rm -rf /etc/localtime
+cp /usr/share/zoneinfo/Etc/UTC /etc/localtime
+
+
+UNNEEDED_PACKAGES=" libfdisk1"\
+" tzdata"\
+
+export DEBIAN_FRONTEND=noninteractive
+
+# Removing unused packages
+for PACKAGE in ${UNNEEDED_PACKAGES}
+do
+	echo ${PACKAGE}
+	if ! apt-get remove --purge --yes "${PACKAGE}"
+	then
+		echo "WARNING: ${PACKAGE} isn't installed"
+	fi
+done
+
+apt-get autoremove --yes || true
+
+# Dropping logs
+rm -rf /var/log/*
+
+# Dropping documentation, localization, i18n files, etc
+rm -rf /usr/share/doc/*
+rm -rf /usr/share/locale/*
+rm -rf /usr/share/man
+rm -rf /usr/share/i18n/*
+rm -rf /usr/share/info/*
+rm -rf /usr/share/lintian/*
+rm -rf /usr/share/common-licenses/*
+rm -rf /usr/share/mime/*
+
+# Dropping reportbug scripts
+rm -rf /usr/share/bug
+
+# Drop udev hwdb not required on a stripped system
+rm -rf /lib/udev/hwdb.bin /lib/udev/hwdb.d/*
+
+# Drop all gconv conversions && binaries
+rm -rf usr/bin/iconv
+rm -rf usr/sbin/iconvconfig
+rm -rf usr/lib/*/gconv/
+
+# Remove libusb database
+rm -rf usr/sbin/update-usbids
+rm -rf var/lib/usbutils/usb.ids
+rm -rf usr/share/misc/usb.ids
+
+#######################################################################
+# Crush into a minimal production image to be deployed via some type of image
+# updating system.
+# IMPORTANT: The Debian system is not longer functional at this point,
+# for example, apt and dpkg will stop working
+
+UNNEEDED_PACKAGES="apt libapt-pkg5.0 "\
+"ncurses-bin ncurses-base libncursesw5 libncurses5 "\
+"perl-base "\
+"debconf libdebconfclient0 "\
+"e2fsprogs e2fslibs libfdisk1 "\
+"insserv "\
+"udev "\
+"init-system-helpers "\
+"bash "\
+"cpio "\
+"passwd "\
+"libsemanage1 libsemanage-common "\
+"libsepol1 "\
+"gzip "\
+"gnupg "\
+"gpgv "\
+"hostname "\
+"adduser "\
+"debian-archive-keyring "\
+"libgl1 libgl1-mesa-dri libglapi-mesa libglvnd0 libglx-mesa0 libegl-mesa0 libgles2 "\
+"libllvm7 "\
+"libx11-data libthai-data "\
+"systemd dbus "\
+
+# Removing unneeded packages
+for PACKAGE in ${UNNEEDED_PACKAGES}
+do
+	echo "Forcing removal of ${PACKAGE}"
+	if ! dpkg --purge --force-remove-essential --force-depends "${PACKAGE}"
+	then
+		echo "WARNING: ${PACKAGE} isn't installed"
+	fi
+done
+
+# Show what's left package-wise before dropping dpkg itself
+COLUMNS=300 dpkg-query -W --showformat='${Installed-Size;10}\t${Package}\n' | sort -k1,1n
+
+# Drop dpkg
+dpkg --purge --force-remove-essential --force-depends  dpkg
+
+# No apt or dpkg, no need for its configuration archives
+rm -rf etc/apt
+rm -rf etc/dpkg
+
+# Drop directories not part of ostree
+# Note that /var needs to exist as ostree bind mounts the deployment /var over
+# it
+rm -rf var/* opt srv share
+
+# ca-certificates are in /etc drop the source
+rm -rf usr/share/ca-certificates
+
+# No bash, no need for completions
+rm -rf usr/share/bash-completion
+
+# No zsh, no need for comletions
+rm -rf usr/share/zsh/vendor-completions
+
+# drop gcc-6 python helpers
+rm -rf usr/share/gcc-6
+
+# Drop sysvinit leftovers
+rm -rf etc/init.d
+rm -rf etc/rc[0-6S].d
+
+# Drop upstart helpers
+rm -rf etc/init
+
+# Various xtables helpers
+rm -rf usr/lib/xtables
+
+# Drop all locales
+# TODO: only remaining locale is actually "C". Should we really remove it?
+rm -rf usr/lib/locale/*
+
+# partition helpers
+rm usr/sbin/*fdisk
+
+# local compiler
+rm usr/bin/localedef
+
+# Systemd dns resolver
+find usr etc -name '*systemd-resolve*' -prune -exec rm -r {} \;
+
+# Systemd network configuration
+find usr etc -name '*networkd*' -prune -exec rm -r {} \;
+
+# systemd ntp client
+find usr etc -name '*timesyncd*' -prune -exec rm -r {} \;
+
+# systemd hw database manager
+find usr etc -name '*systemd-hwdb*' -prune -exec rm -r {} \;
+
+# No need for fuse
+find usr etc -name '*fuse*' -prune -exec rm -r {} \;
+
+# lsb init function leftovers
+rm -rf usr/lib/lsb
+
+# Only needed when adding libraries
+rm usr/sbin/ldconfig*
+
+# Games, unused
+rmdir usr/games
+
+# Remove pam module to authenticate against a DB
+# plus libdb-5.3.so that is only used by this pam module
+rm usr/lib/*/security/pam_userdb.so
+rm usr/lib/*/libdb-5.3.so
+
+# remove NSS support for nis, nisplus and hesiod
+rm usr/lib/*/libnss_hesiod*
+rm usr/lib/*/libnss_nis*
+
+rm usr/bin/tar
--- a/src/gallium/drivers/panfrost/ci/deqp-runner.sh
+++ b/src/gallium/drivers/panfrost/ci/deqp-runner.sh
@ -0,0 +1,54 @@
+#!/bin/sh
+
+set -x
+
+# To prevent memory leaks from slowing throughput, restart everything between batches
+BATCH_SIZE=5000
+
+export XDG_RUNTIME_DIR=/tmp
+export LIBGL_DRIVERS_PATH=/mesa/lib/aarch64-linux-gnu/dri/
+export LD_LIBRARY_PATH=/mesa/lib/aarch64-linux-gnu
+export XDG_CONFIG_HOME=$(pwd)
+
+echo "[core]\nidle-time=0\nrequire-input=false\n[shell]\nlocking=false" > weston.ini
+
+cd /deqp/modules/gles2
+
+# Generate test case list file
+weston --tty=7 &
+sleep 1  # Give some time for Weston to start up
+./deqp-gles2 --deqp-runmode=stdout-caselist | grep dEQP-GLES2 | cut -d ' ' -f 2 > /tmp/case-list.txt
+
+# Disable for now tests that are very slow, either by just using lots of CPU or by crashing
+sed -i '/dEQP-GLES2.performance/d' /tmp/case-list.txt
+sed -i '/dEQP-GLES2.functional.texture.filtering.2d.linear_mipmap_linear_/d' /tmp/case-list.txt
+sed -i '/dEQP-GLES2.functional.texture.filtering.cube.linear_mipmap_linear_/d' /tmp/case-list.txt
+sed -i '/dEQP-GLES2.functional.texture.filtering.cube.linear_mipmap_nearest_/d' /tmp/case-list.txt
+
+# Cannot use tee because dash doesn't have pipefail
+touch /tmp/result.txt
+tail -f /tmp/result.txt &
+
+while [ -s /tmp/case-list.txt ]; do
+    head -$BATCH_SIZE /tmp/case-list.txt > /tmp/next-batch.txt
+	./deqp-gles2 --deqp-log-filename=/dev/null --deqp-caselist-file=/tmp/next-batch.txt --deqp-watchdog=enable --deqp-crashhandler=enable >> /tmp/result.txt
+	deqp_status=$?
+
+	kill $(pidof weston)
+	sleep 1  # Give some time for Weston to release the VT
+	weston --tty=7 &
+	sleep 1  # Give some time for Weston to start up
+
+	if [ $deqp_status -ne 0 ]; then
+		# Continue from the subtest after the failing one
+		crashed_test=$(grep "Test case" /tmp/result.txt | tail -1 | sed "s/Test case '\(.*\)'\.\./\1/")
+		sed -i "0,/^$crashed_test$/d" /tmp/case-list.txt
+
+		# So LAVA knows what happened
+		echo "Test case '$crashed_test'.."
+		echo "  Crash"
+	else
+		# Consume a whole batch
+	    sed -i '1,'$BATCH_SIZE'd' /tmp/case-list.txt
+	fi
+done
--- a/src/gallium/drivers/panfrost/ci/expected-failures.txt
+++ b/src/gallium/drivers/panfrost/ci/expected-failures.txt
--- a/src/gallium/drivers/panfrost/ci/generate_lava.py
+++ b/src/gallium/drivers/panfrost/ci/generate_lava.py
@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+
+from jinja2 import Environment, FileSystemLoader
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--template")
+parser.add_argument("--base-artifacts-url")
+args = parser.parse_args()
+
+env = Environment(loader = FileSystemLoader('.'), trim_blocks=True, lstrip_blocks=True)
+template = env.get_template(args.template)
+
+values = {}
+values['base_artifacts_url'] = args.base_artifacts_url
+
+print(template.render(values))
--- a/src/gallium/drivers/panfrost/ci/gitlab-ci.yml
+++ b/src/gallium/drivers/panfrost/ci/gitlab-ci.yml
@ -0,0 +1,117 @@
+variables:
+  DOCKER_DRIVER: overlay2
+
+stages:
+  - build-container
+  - build
+  - test
+
+# Build Docker image with deqp, the rootfs and the build deps for Mesa
+build-container:
+  stage: build-container
+  when: always
+  services:
+    - docker:dind
+  image: docker:latest
+  before_script:
+    - mkdir -p results
+  script:
+    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
+    - time docker pull $CI_REGISTRY_IMAGE:latest || true
+    - time docker build
+             -t $CI_REGISTRY_IMAGE
+             --cache-from $CI_REGISTRY_IMAGE:latest
+             src/gallium/drivers/panfrost/ci
+             2>&1 | tee results/docker_build_log.txt
+    - time docker push $CI_REGISTRY_IMAGE:latest 2>&1 | tee results/docker_push_log.txt
+
+    - docker history $CI_REGISTRY_IMAGE:latest
+
+    - "grep -o 'digest: sha256:[0-9a-f]\\+' results/docker_push_log.txt |
+       cut -f 2 -d ' ' |
+       tee results/docker_image_digest.txt"
+  artifacts:
+    when: always
+    paths:
+      - results/
+
+build:
+  stage: build
+  image: $CI_REGISTRY_IMAGE:latest
+  cache:
+    paths:
+      - ccache
+  before_script:
+    - mkdir -p results mesa-build
+    - mkdir -p ccache
+  script:
+    - export CCACHE_BASEDIR=$CI_PROJECT_DIR
+    - export CCACHE_DIR=$CI_PROJECT_DIR/ccache
+    - export PATH="/usr/lib/ccache:$PATH"
+    - ccache -s
+
+    - /usr/share/meson/debcrossgen --arch arm64 -o /tmp/cross_file.txt
+    - meson . mesa-build
+        --cross-file /tmp/cross_file.txt
+        --libdir /artifacts/rootfs/mesa/lib/aarch64-linux-gnu
+        --buildtype release
+        -Dgallium-drivers=kmsro,panfrost
+        -Ddri-drivers=
+        -Dprefix=/artifacts/rootfs/mesa
+    - ninja -C mesa-build install
+    - du -sh /artifacts/rootfs/mesa/*
+    - rm -rf /artifacts/rootfs/mesa/include
+    - cp src/gallium/drivers/panfrost/ci/deqp-runner.sh /artifacts/rootfs/deqp/.
+    - du -sh /artifacts/rootfs/deqp/*
+    - find /artifacts/rootfs/ -type f -printf "%s\t%p\n" | sort -n
+    - cd /artifacts/rootfs/ ; find -H  |  cpio -H newc -v -o | gzip -c - > $CI_PROJECT_DIR/results/panfrost-rootfs.cpio.gz
+    - cp /artifacts/Image /artifacts/rk3399-gru-kevin.dtb $CI_PROJECT_DIR/results/.
+
+    - cd $CI_PROJECT_DIR
+    - src/gallium/drivers/panfrost/ci/generate_lava.py
+        --template src/gallium/drivers/panfrost/ci/lava-deqp.yml.jinja2
+        --base-artifacts-url $CI_PROJECT_URL/-/jobs/$CI_JOB_ID/artifacts/raw/results
+        > results/lava-deqp.yml
+  artifacts:
+    when: always
+    paths:
+      - results/
+
+test:
+  stage: test
+  image: $CI_REGISTRY_IMAGE:latest
+  script:
+    - mkdir -p ~/.config/
+    - |
+      echo "default:
+        uri: https://lava.collabora.co.uk/RPC2
+        timeout: 120
+        username: jenkins-fdo
+        token: $LAVA_TOKEN
+      " > ~/.config/lavacli.yaml
+    - lava_job_id=`lavacli jobs submit $CI_PROJECT_DIR/results/lava-deqp.yml`
+    - echo $lava_job_id
+    - lavacli jobs logs $lava_job_id | grep -a -v "{'case':" | tee results/lava-deqp-$lava_job_id.log
+    - lavacli jobs show $lava_job_id
+    - curl "https://lava.collabora.co.uk/results/$lava_job_id/csv?user=jenkins-fdo&token=$LAVA_TOKEN" > raw_results.csv
+    - cat raw_results.csv | csvcut -c 12,3 | grep dEQP-GLES2 | sort > results/results-$lava_job_id.csv
+
+    # FIXME: Remove flip-flops from comparison files
+    - sed -i '/dEQP-GLES2.functional.shaders.invariance.highp.loop_4/d' src/gallium/drivers/panfrost/ci/expected-failures.txt results/results-$lava_job_id.csv
+    - sed -i '/dEQP-GLES2.functional.texture.units.2_units.mixed.1/d' src/gallium/drivers/panfrost/ci/expected-failures.txt results/results-$lava_job_id.csv
+    - sed -i '/dEQP-GLES2.functional.texture.units.4_units.mixed.5/d' src/gallium/drivers/panfrost/ci/expected-failures.txt results/results-$lava_job_id.csv
+    - sed -i '/dEQP-GLES2.functional.texture.units.all_units.only_cube.7/d' src/gallium/drivers/panfrost/ci/expected-failures.txt results/results-$lava_job_id.csv
+
+    - PASSED=$(grep pass$ results/results-$lava_job_id.csv | wc -l)
+    - FAILED=$(grep fail$ results/results-$lava_job_id.csv | wc -l)
+    - TOTAL=$(wc -l < results/results-$lava_job_id.csv)
+    - 'echo "Passed: $PASSED ($(expr $PASSED \* 100 / $TOTAL)%)"'
+    - 'echo "Failed: $FAILED ($(expr $FAILED \* 100 / $TOTAL)%)"'
+    - 'echo "Total: $TOTAL"'
+
+    - sed '/,pass/d' results/results-$lava_job_id.csv | sed 's/,fail//' > results/failures-$lava_job_id.txt
+    - diff -u src/gallium/drivers/panfrost/ci/expected-failures.txt results/failures-$lava_job_id.txt
+  artifacts:
+    when: always
+    paths:
+      - results/
--- a/src/gallium/drivers/panfrost/ci/lava-deqp.yml.jinja2
+++ b/src/gallium/drivers/panfrost/ci/lava-deqp.yml.jinja2
@ -0,0 +1,70 @@
+job_name: panfrost-deqp
+device_type: rk3399-gru-kevin
+timeouts:
+  job:
+    minutes: 40
+  action:
+   minutes: 10
+  actions:
+    power-off:
+      seconds: 30
+priority: medium
+visibility: public
+actions:
+- deploy:
+    timeout:
+      minutes: 2
+    to: tftp
+    kernel:
+      url: {{ base_artifacts_url }}/Image
+    ramdisk:
+      url: {{ base_artifacts_url }}/panfrost-rootfs.cpio.gz
+      compression: gz
+    dtb:
+      url: {{ base_artifacts_url }}/rk3399-gru-kevin.dtb
+    os: oe
+- boot:
+    timeout:
+      minutes: 5
+    method: depthcharge
+    commands: ramdisk
+    prompts:
+      - '#' 
+- test:
+    timeout:
+      minutes: 40
+    definitions:
+    - repository:
+        metadata:
+          format: Lava-Test Test Definition 1.0
+          name: igt
+          description: "IGT test plan"
+          os:
+          - oe
+          scope:
+          - functional
+        run:
+          steps:
+          - mount -t proc none /proc
+          - mount -t sysfs none /sys 
+          - mount -t devtmpfs none /dev
+          - mkdir -p /dev/pts
+          - mount -t devpts devpts /dev/pts
+          - echo 1 > /proc/sys/kernel/printk
+          - echo performance > /sys/devices/platform/ff9a0000.gpu/devfreq/devfreq0/governor
+          - sh /deqp/deqp-runner.sh
+        parse:
+          pattern: 'Test case ''(?P<test_case_id>\S*)''..\s+(?P<result>(Pass|NotSupported|QualityWarning|CompatibilityWarning|Fail|ResourceError|Crash|Timeout|InternalError))'
+          fixupdict:
+            Pass: pass
+            NotSupported: pass
+            QualityWarning: pass
+            CompatibilityWarning: pass
+            Fail: fail
+            ResourceError: fail
+            Crash: fail
+            Timeout: fail
+            InternalError: fail
+      from: inline
+      name: deqp
+      path: inline/lava-deqp.yaml