Merge pull request #119 from EthicalML/114_further_tests_on_ci

Added further tests to CI and provide Dockerimage with builds to swiftshader
2021-01-24 17:54:14 +00:00 · 2021-01-24 17:54:14 +00:00 · 7af493e02e
commit 7af493e02e
parent 661a50e62d a29d241a79
20 changed files with 355 additions and 66 deletions
--- a/.dockerignore
+++ b/.dockerignore
@ -0,0 +1,5 @@
+build/*
+examples/*
+docker-builders/
+swiftshader/
+
--- a/.github/workflows/cpp_tests.yml
+++ b/.github/workflows/cpp_tests.yml
@ -0,0 +1,34 @@
+name: C++ Tests
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  cpp-tests:
+
+    runs-on: ubuntu-18.04
+    container: axsauze/kompute-builder:0.2
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+    - name: configure-cpp
+      run: |
+        cmake -Bbuild/ \
+                  -DKOMPUTE_OPT_INSTALL=0                    \
+                  -DKOMPUTE_OPT_REPO_SUBMODULE_BUILD=1       \
+                  -DKOMPUTE_OPT_BUILD_TESTS=1                \
+                  -DKOMPUTE_OPT_ENABLE_SPDLOG=1              \
+                  -DSPDLOG_INSTALL=1
+    - name: build-cpp
+      run: |
+        make mk_build_tests
+    - name: test-cpp
+      run: |
+        export VK_ICD_FILENAMES=/swiftshader/vk_swiftshader_icd.json
+        make mk_run_tests_cpu_only
+
--- a/.github/workflows/cpp_ubuntu.yml
+++ b/.github/workflows/cpp_ubuntu.yml
@ -1,42 +0,0 @@
-name: C++ Ubuntu
-
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
-
-jobs:
-  build-ubuntu:
-
-    runs-on: ubuntu-latest
-    
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        submodules: recursive
-    - name: Install Dependencies
-      run: |
-        sudo apt-get install libvulkan-dev
-        #swiftshader vulkan cpu implementation
-        sudo wget https://www.dropbox.com/s/d0weho6l8dovm71/libvk_swiftshader.so?dl=1 -O /usr/lib/libvk_swiftshader.so
-        sudo mkdir -p /usr/share/vulkan/icd.d
-        sudo wget https://www.dropbox.com/s/5oly49ev3vvcfdu/vk_swiftshader_icd.json?dl=1 -O /usr/share/vulkan/icd.d/vk_swiftshader_icd.json
-    - name: configure
-      run: |
-        cmake . -Bbuild/ \
-                  -DKOMPUTE_OPT_BUILD_TESTS=1                \
-                  -DKOMPUTE_OPT_INSTALL=1                    \
-                  -DKOMPUTE_OPT_ENABLE_SPDLOG=1              \
-                  -DSPDLOG_INSTALL=1                         \
-                  -DKOMPUTE_OPT_REPO_SUBMODULE_BUILD=1       \
-                  -DKOMPUTE_OPT_BUILD_TESTS=1
-    - name: build
-      run: | 
-        make -C build/ -j
-    - name: tests
-      run: |
-        #only running tests that dont require GLSL compilation
-        ./build/test/test_kompute --gtest_filter=*Tensor*
-        ./build/test/test_kompute --gtest_filter=TestManager*
-        
--- a/.github/workflows/python_tests.yml
+++ b/.github/workflows/python_tests.yml
@ -0,0 +1,32 @@
+name: Python Tests
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  python-tests:
+
+    runs-on: ubuntu-18.04
+    container: axsauze/kompute-builder:0.2
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+    - name: configure-python
+      run: |
+        pip3 install -r python/test/requirements-dev.txt
+    - name: build-python
+      run: |
+        pip3 install .
+    - name: test-python
+      run: |
+        export VK_ICD_FILENAMES=/swiftshader/vk_swiftshader_icd.json
+        pytest -v python/test/test_array_multiplication.py
+        pytest -v python/test/test_kompute.py -k "test_opmult"
+        pytest -v python/test/test_logistic_regression.py
+
+
--- a/.gitignore
+++ b/.gitignore
@ -183,4 +183,7 @@ tmp/
 build/
 release/

+# Kompute
+swiftshader/
+

--- a/51
+++ b/51
@ -56,30 +56,30 @@ mk_cmake:
 	cmake \
 		-Bbuild \
 		$(MK_CMAKE_EXTRA_FLAGS) \
-		-DCMAKE_TOOLCHAIN_FILE=$(VCPKG_UNIX_PATH) \
+		-DKOMPUTE_EXTRA_CXX_FLAGS=$(MK_KOMPUTE_EXTRA_CXX_FLAGS) \
 		-DCMAKE_BUILD_TYPE=$(MK_BUILD_TYPE) \
 		-DCMAKE_INSTALL_PREFIX=$(MK_INSTALL_PATH) \
-		-DKOMPUTE_EXTRA_CXX_FLAGS=$(MK_KOMPUTE_EXTRA_CXX_FLAGS) \
 		-DKOMPUTE_OPT_INSTALL=1 \
-		-DKOMPUTE_OPT_REPO_SUBMODULE_BUILD=0 \
+		-DKOMPUTE_OPT_REPO_SUBMODULE_BUILD=1 \
 		-DKOMPUTE_OPT_BUILD_TESTS=1 \
 		-DKOMPUTE_OPT_BUILD_DOCS=1 \
 		-DKOMPUTE_OPT_BUILD_SHADERS=1 \
 		-DKOMPUTE_OPT_BUILD_SINGLE_HEADER=1 \
 		-DKOMPUTE_OPT_ENABLE_SPDLOG=1 \
+		-DSPDLOG_INSTALL=1 \
 		-G "Unix Makefiles"

 mk_build_all:
-	make -C build/
+	cmake --build build/. --parallel

 mk_build_docs:
-	make -C build/ docs
+	cmake --build build/. --target docs --parallel

 mk_build_kompute:
-	make -C build/ kompute
+	cmake --build build/. --target kompute --parallel

 mk_build_tests:
-	make -C build/ test_kompute
+	cmake --build build/ --target test_kompute --parallel

 mk_run_docs: mk_build_docs
 	(cd build/docs/sphinx && python2.7 -m SimpleHTTPServer)
@ -87,6 +87,25 @@ mk_run_docs: mk_build_docs
 mk_run_tests: mk_build_tests
 	./build/test/test_kompute $(FILTER_TESTS)

+mk_build_swiftshader_library:
+	git clone https://github.com/google/swiftshader || echo "Assuming already cloned"
+	# GCC 8 or above is required otherwise error on "filesystem" lib will appear
+	CC="/usr/bin/gcc-8" CXX="/usr/bin/g++-8" cmake swiftshader/. -Bswiftshader/build/
+	cmake --build swiftshader/build/. --parallel
+
+mk_run_tests_cpu: export VK_ICD_FILENAMES=$(PWD)/swiftshader/build/vk_swiftshader_icd.json
+mk_run_tests_cpu: mk_build_swiftshader_library mk_build_tests mk_run_tests_cpu_only
+
+mk_run_tests_cpu_only:
+	./build/test/test_kompute --gtest_filter="TestLogisticRegressionAlgorithm.*"
+	./build/test/test_kompute --gtest_filter="TestManager.*"
+	./build/test/test_kompute --gtest_filter="TestOpAlgoBase.ShaderCompiledDataFromConstructor"
+	./build/test/test_kompute --gtest_filter="TestOpTensorCopy.*"
+	./build/test/test_kompute --gtest_filter="TestOpTensorCreate.*"
+	./build/test/test_kompute --gtest_filter="TestOpTensorSync.*"
+	./build/test/test_kompute --gtest_filter="TestSequence.*"
+	./build/test/test_kompute --gtest_filter="TestTensor.*"
+

 ####### Visual studio build shortcut commands #######

@ -110,6 +129,7 @@ vs_cmake:
 		-DKOMPUTE_OPT_BUILD_SHADERS=1 \
 		-DKOMPUTE_OPT_BUILD_SINGLE_HEADER=1 \
 		-DKOMPUTE_OPT_ENABLE_SPDLOG=1 \
+		-DSPDLOG_INSTALL=1 \
 		-G "Visual Studio 16 2019"

 vs_build_all:
@ -136,9 +156,14 @@ vs_run_tests: vs_build_tests
 ####### Create release ######

 update_builder_image:
+	docker build . -f docker-builders/KomputeBuilder.Dockerfile \
+		-t axsauze/kompute-builder:0.2
+	docker push axsauze/kompute-builder:0.2
+
+update_vulkan_sdk:
 	docker build -f builders/Dockerfile.linux . \
-		-t axsauze/kompute-builder:0.1
-	docker push axsauze/kompute-builder:0.1
+		-t axsauze/vulkan-sdk:0.1
+	docker push axsauze/vulkan-sdk:0.1

 create_linux_release:
 	docker run -it \
@ -146,6 +171,14 @@ create_linux_release:
 		axsauze/kompute-builder:0.1 \
 		/workspace/scripts/build_release_linux.sh

+
+####### Run CI Commands #######
+
+# This command uses act to replicate github action
+# https://github.com/nektos/act
+run_ci:
+	act
+
 ####### General project commands #######

 install_python_reqs:
--- a/README.md
+++ b/README.md
@ -46,7 +46,7 @@

 Kompute is provided as a single header file [`Kompute.hpp`](#setup). See [build-system section](#build-overview) for configurations available.

-### Trying the Colab Notebook
+### Interactive Notebook & Hands on Videos

 You are able to try out the interactive Colab Notebooks which allow you to use a free GPU. The available examples are the Python and C++ examples below:

@ -80,6 +80,40 @@ You are able to try out the interactive Colab Notebooks which allow you to use a
 </table>


+You can also check out the two following getting started videos presented at the FOSDEM 2021 conference. 
+
+Both talks have annotated sections - the intro for both is almost the same so you can skip to the more specific content.
+
+<table>
+<tr>
+
+<td width="50%">
+<h5>For <a href="https://www.youtube.com/watch?v=Xz4fiQNmGSA">C++ & Vulkan SDK</a> Enthusiasts</h5>
+</td>
+
+<td>
+<h5>For <a href="https://www.youtube.com/watch?v=AJRyZ09IUdg">Python & Machine Learning</a> Enthusiasts</h5>
+</td>
+
+</tr>
+<tr>
+
+<td width="50%">
+<a href="https://www.youtube.com/watch?v=Xz4fiQNmGSA">
+<img src="https://raw.githubusercontent.com/EthicalML/vulkan-kompute/master/docs/images/kompute-cpp-video.png">
+</a>
+</td>
+
+<td>
+<a href="https://www.youtube.com/watch?v=AJRyZ09IUdg">
+<img src="https://raw.githubusercontent.com/EthicalML/vulkan-kompute/master/docs/images/kompute-python-video.png">
+</a>
+</td>
+
+</tr>
+</table>
+
+
 ### Your First Kompute

 Below you can find both the C++ and Python version of a simple GPU multiplication snippet with Kompute.
@ -336,21 +370,27 @@ To update the documentation you will need to:

 ##### Running tests

-To run tests you can use the helper top level Makefile
+Running the unit tests has been significantly simplified for contributors.

-For visual studio you can run
+The tests run on CPU, and can be triggered using the ACT command line interface (https://github.com/nektos/act) - once you install the command line (And start the Docker daemon) you just have to type:

 ```
-make vs_cmake
-make vs_run_tests VS_BUILD_TYPE="Release"
+$ act
+
+[Python Tests/python-tests] 🚀  Start image=axsauze/kompute-builder:0.2
+[C++ Tests/cpp-tests      ] 🚀  Start image=axsauze/kompute-builder:0.2
+[C++ Tests/cpp-tests      ]   🐳  docker run image=axsauze/kompute-builder:0.2 entrypoint=["/usr/bin/tail" "-f" "/dev/null"] cmd=[]
+[Python Tests/python-tests]   🐳  docker run image=axsauze/kompute-builder:0.2 entrypoint=["/usr/bin/tail" "-f" "/dev/null"] cmd=[]
+...
 ```

-For unix you can run
+The repository contains unit tests for the C++ and Python code, and can be found under the `test/` and `python/test` folder.

-```
-make mk_cmake MK_BUILD_TYPE="Release"
-make mk_run_tests
-```
+The tests are currently run through the CI using Github Actions. It uses the images found in `docker-builders/`.
+
+In order to minimise hardware requirements the tests can run without a GPU, directly in the CPU using [Swiftshader](https://github.com/google/swiftshader).
+
+For more information on how the CI and tests are setup, you can go to the [CI, Docker and Tests Section](https://kompute.cc/overview/ci-tests.html) in the documentation.

 ## Motivations

--- a/docker-builders/KomputeBuilder.Dockerfile
+++ b/docker-builders/KomputeBuilder.Dockerfile
@ -0,0 +1,35 @@
+FROM ubuntu:18.04
+
+# Base packages from default ppa
+RUN apt-get update -y
+RUN apt-get install -y wget
+RUN apt-get install -y gnupg
+RUN apt-get install -y ca-certificates
+RUN apt-get install -y software-properties-common
+
+# Repository to latest cmake
+RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add -
+RUN apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main'
+
+# Repository for latest git (needed for gh actions)
+RUN add-apt-repository -y ppa:git-core/ppa
+
+# Refresh repositories
+RUN apt update -y
+
+RUN apt install -y git
+RUN apt-get install -y cmake g++
+RUN apt-get install -y libvulkan-dev
+# Swiftshader dependencies
+RUN apt-get install -y libx11-dev zlib1g-dev
+RUN apt-get install -y libxext-dev
+
+COPY --from=axsauze/swiftshader:0.1 /swiftshader/ /swiftshader/
+
+# Setup Python
+RUN apt-get install -y python3-pip
+
+RUN mkdir builder
+WORKDIR /builder
+
+
--- a/docker-builders/Makefile
+++ b/docker-builders/Makefile
@ -0,0 +1,19 @@
+
+build_kompute_builder:
+	docker build .. -f KomputeBuilder.Dockerfile -t axsauze/kompute-builder:0.2
+
+push_kompute_builder: build_kompute_builder
+	docker push axsauze/kompute-builder:0.2
+
+build_swiftshader:
+	docker build .. -f Swiftshader.Dockerfile -t axsauze/swiftshader:0.1
+
+push_swiftshader: build_swiftshader
+	docker push axsauze/swiftshader:0.1
+
+build_vulkan_sdk:
+	docker build .. -f VulkanSDK.Dockerfile -t axsauze/vulkan-sdk:0.1
+
+push_vulkan_sdk: build_vulkan_sdk
+	docker push axsauze/vulkan-sdk:0.1
+
--- a/docker-builders/Swiftshader.Dockerfile
+++ b/docker-builders/Swiftshader.Dockerfile
@ -0,0 +1,34 @@
+FROM ubuntu:18.04
+
+# Base packages from default ppa
+RUN apt-get update -y
+RUN apt-get install -y wget
+RUN apt-get install -y gnupg
+RUN apt-get install -y ca-certificates
+RUN apt-get install -y software-properties-common
+
+# Repository to latest cmake
+RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add -
+RUN apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main'
+
+# Repository for latest git (needed for gh actions)
+RUN add-apt-repository -y ppa:git-core/ppa
+
+# Refresh repositories
+RUN apt update -y
+
+RUN apt install -y git
+RUN apt-get install -y cmake g++
+RUN apt-get install -y libvulkan-dev
+
+# Dependencies for swiftshader
+RUN apt-get install -y g++-8 gcc-8
+RUN apt-get install -y libx11-dev zlib1g-dev
+RUN apt-get install -y libxext-dev
+
+# Run swiftshader via env VK_ICD_FILENAMES=/swiftshader/vk_swiftshader_icd.json
+RUN git clone https://github.com/google/swiftshader swiftshader-build
+RUN CC="/usr/bin/gcc-8" CXX="/usr/bin/g++-8" cmake swiftshader-build/. -Bswiftshader-build/build/
+RUN cmake --build swiftshader-build/build/. --parallel
+RUN cp -r swiftshader-build/build/Linux/ swiftshader/
+
--- a/docker-builders/VulkanSDK.Dockerfile
+++ b/docker-builders/VulkanSDK.Dockerfile
@ -29,7 +29,7 @@ RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/nul
 RUN apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main'
 # Adding security repo to add cmake dependency on libssl 1.0
 RUN echo "deb http://security.ubuntu.com/ubuntu bionic-security main" | tee -a /etc/apt/sources.list.d/bionic.list
-RUN apt-get update
+RUN apt-get update --fix-missing -y
 RUN apt-get upgrade -y

 # Install build dependencies
--- a/docs/images/kompute-cpp-video.png
+++ b/docs/images/kompute-cpp-video.png
--- a/docs/images/kompute-python-video.png
+++ b/docs/images/kompute-python-video.png
--- a/docs/index.rst
+++ b/docs/index.rst
@ -19,6 +19,7 @@ Index
    Converting GLSL/HLSL Shaders to C++ Headers <overview/shaders-to-headers>
    Mobile App Integration (Android) <overview/mobile-android>
    Game Engine Integration (Godot Engine) <overview/game-engine-godot>
+    CI, Docker Images & Tests <overview/ci-tests.rst>
    Python Class Documentation & Reference <overview/python-reference>
    C++ Class Documentation & Reference <overview/reference>
    Code Index <genindex>
--- a/docs/overview/build-system.rst
+++ b/docs/overview/build-system.rst
@ -1,5 +1,5 @@

-Build System Deep Dive
+C++ Build System Deep Dive
 ======================

 The recommended approach to build the project is as out-of-source build in the ``build`` folder. This project comes with a ``Makefile`` that provides a set of commands that help with developer workflows. You can see some of the commands if you want to add some of the more advanced commands.
--- a/docs/overview/ci-tests.rst
+++ b/docs/overview/ci-tests.rst
@ -0,0 +1,66 @@
+
+CI, Docker Images & Tests
+======================
+
+This section contains an overview of the steps run on CI, as well as the tools used to simplify the testing (such as running Vulkan on CPU).
+
+We use Github Actions to run the tests, which simplifies the workflows significantly for contributors. 
+
+The tests run on CPU, and can be triggered using the ACT command line interface (https://github.com/nektos/act) - once you install the command line (And start the Docker daemon) you just have to type:
+
+.. code-block::
+
+    $ act
+
+    [Python Tests/python-tests] 🚀  Start image=axsauze/kompute-builder:0.2
+    [C++ Tests/cpp-tests      ] 🚀  Start image=axsauze/kompute-builder:0.2
+    [C++ Tests/cpp-tests      ]   🐳  docker run image=axsauze/kompute-builder:0.2 entrypoint=["/usr/bin/tail" "-f" "/dev/null"] cmd=[]
+    [Python Tests/python-tests]   🐳  docker run image=axsauze/kompute-builder:0.2 entrypoint=["/usr/bin/tail" "-f" "/dev/null"] cmd=[]
+    ...
+
+
+CI Commands Triggered
+~~~~~~~~~~~~~
+
+The simplest way to see how this works is by looking at the github actions commands that are run.
+
+These can be found through the following files:
+
+* `CPP Tests <https://github.com/EthicalML/vulkan-kompute/blob/master/.github/workflows/cpp_tests.yml>`_
+* `Python Tests <https://github.com/EthicalML/vulkan-kompute/blob/master/.github/workflows/python_tests.yml>`_
+
+When submitting a PR or merging a PR into master, both of these will run - you can see the logs through the github interface.
+
+
+
+Running Vulkan on the CPU
+~~~~~~~~~~~~~
+
+We use `Swiftshader <https://github.com/google/swiftshader>`_ to enable us to run the Vulkan Kompute framework directly on the CPU for the CI tests.
+
+Even though Swiftshader is optimized to function as a high-performance CPU backend for Vulkan, there are several limitations, the most notable in context of Kompute are:
+
+* Loading files (spirv or text) leads to segfault
+* Loading raw text string shaders leads to segfault
+
+This is one of the main reason why only a subset of the tests are run in the CI.
+
+Dockerfiles
+~~~~~~~~~~~~~
+
+The dockerfiles created provide functionality to simplify the interaction with the system. 
+
+.. list-table::
+   :header-rows: 1
+
+   * - Image
+     - Description
+   * - axsauze/kompute-builder:0.2
+     - Main CI builder image with all required dependencies to build and run C++ & Python tests.
+   * - axsauze/swiftshader:0.1
+     - Image building Swiftshader libraries only to reduce time via multi-staged builds
+   * - axsauze/vulkan-sdk:0.1
+     - Image contained a linux build of the full Vulkan SDK to reduce time via multi-staged builds
+
+
+
--- a/python/test/requirements-dev.txt
+++ b/python/test/requirements-dev.txt
@ -1,2 +1,3 @@
 pyshader==0.7.0
-numpy
+numpy==1.19.5
+pytest==6.2.1
--- a/setup.py
+++ b/setup.py
@ -76,6 +76,9 @@ setup(
    long_description=long_description,
    long_description_content_type='text/markdown',
    ext_modules=[CMakeExtension('kp')],
+    install_requires=[
+        "numpy<2.0.0"
+    ],
    cmdclass=dict(build_ext=CMakeBuild),
    zip_safe=False,
    include_package_data=True,
--- a/test/TestLogisticRegression.cpp
+++ b/test/TestLogisticRegression.cpp
@ -3,6 +3,8 @@

 #include "kompute/Kompute.hpp"

+#include "kompute_test/shaders/shadertest_logistic_regression.hpp"
+
 TEST(TestLogisticRegressionAlgorithm, TestMainLogisticRegression)
 {

@ -44,8 +46,18 @@ TEST(TestLogisticRegressionAlgorithm, TestMainLogisticRegression)

        sq->record<kp::OpTensorSyncDevice>({ wIn, bIn });

+#ifdef KOMPUTE_SHADER_FROM_STRING
        sq->record<kp::OpAlgoBase>(
-          params, "test/shaders/glsl/test_logistic_regression.comp");
+          params, "test/shaders/glsl/test_logistic_regression.comp.spv");
+#else
+        sq->record<kp::OpAlgoBase>(
+          params,
+          std::vector<char>(
+            kp::shader_data::shaders_glsl_logisticregression_comp_spv,
+            kp::shader_data::shaders_glsl_logisticregression_comp_spv +
+              kp::shader_data::
+                shaders_glsl_logisticregression_comp_spv_len));
+#endif

        sq->record<kp::OpTensorSyncLocal>({ wOutI, wOutJ, bOut, lOut });

@ -123,8 +135,18 @@ TEST(TestLogisticRegressionAlgorithm, TestMainLogisticRegressionManualCopy)
        // Record op algo base
        sq->begin();

+#ifdef KOMPUTE_SHADER_FROM_STRING
        sq->record<kp::OpAlgoBase>(
-          params, "test/shaders/glsl/test_logistic_regression.comp");
+          params, "test/shaders/glsl/test_logistic_regression.comp.spv");
+#else
+        sq->record<kp::OpAlgoBase>(
+          params,
+          std::vector<char>(
+            kp::shader_data::shaders_glsl_logisticregression_comp_spv,
+            kp::shader_data::shaders_glsl_logisticregression_comp_spv +
+              kp::shader_data::
+                shaders_glsl_logisticregression_comp_spv_len));
+#endif

        sq->record<kp::OpTensorSyncLocal>({ wOutI, wOutJ, bOut, lOut });

--- a/test/shaders/glsl/test_op_custom_shader.comp
+++ b/test/shaders/glsl/test_op_custom_shader.comp
@ -1,7 +1,10 @@
 #version 450
+
 layout (local_size_x = 1) in;
+
 layout(set = 0, binding = 0) buffer a { float pa[]; };
 layout(set = 0, binding = 1) buffer b { float pb[]; };
+
 void main() {
    uint index = gl_GlobalInvocationID.x;
    pb[index] = pa[index];