+
+
### Your First Kompute
Below you can find both the C++ and Python version of a simple GPU multiplication snippet with Kompute.
@@ -336,21 +370,27 @@ To update the documentation you will need to:
##### Running tests
-To run tests you can use the helper top level Makefile
+Running the unit tests has been significantly simplified for contributors.
-For visual studio you can run
+The tests run on CPU, and can be triggered using the ACT command line interface (https://github.com/nektos/act) - once you install the command line (And start the Docker daemon) you just have to type:
```
-make vs_cmake
-make vs_run_tests VS_BUILD_TYPE="Release"
+$ act
+
+[Python Tests/python-tests] 🚀 Start image=axsauze/kompute-builder:0.2
+[C++ Tests/cpp-tests ] 🚀 Start image=axsauze/kompute-builder:0.2
+[C++ Tests/cpp-tests ] 🐳 docker run image=axsauze/kompute-builder:0.2 entrypoint=["/usr/bin/tail" "-f" "/dev/null"] cmd=[]
+[Python Tests/python-tests] 🐳 docker run image=axsauze/kompute-builder:0.2 entrypoint=["/usr/bin/tail" "-f" "/dev/null"] cmd=[]
+...
```
-For unix you can run
+The repository contains unit tests for the C++ and Python code, and can be found under the `test/` and `python/test` folder.
-```
-make mk_cmake MK_BUILD_TYPE="Release"
-make mk_run_tests
-```
+The tests are currently run through the CI using Github Actions. It uses the images found in `docker-builders/`.
+
+In order to minimise hardware requirements the tests can run without a GPU, directly in the CPU using [Swiftshader](https://github.com/google/swiftshader).
+
+For more information on how the CI and tests are setup, you can go to the [CI, Docker and Tests Section](https://kompute.cc/overview/ci-tests.html) in the documentation.
## Motivations
diff --git a/docker-builders/KomputeBuilder.Dockerfile b/docker-builders/KomputeBuilder.Dockerfile
new file mode 100644
index 000000000..4cb31c9fd
--- /dev/null
+++ b/docker-builders/KomputeBuilder.Dockerfile
@@ -0,0 +1,35 @@
+FROM ubuntu:18.04
+
+# Base packages from default ppa
+RUN apt-get update -y
+RUN apt-get install -y wget
+RUN apt-get install -y gnupg
+RUN apt-get install -y ca-certificates
+RUN apt-get install -y software-properties-common
+
+# Repository to latest cmake
+RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add -
+RUN apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main'
+
+# Repository for latest git (needed for gh actions)
+RUN add-apt-repository -y ppa:git-core/ppa
+
+# Refresh repositories
+RUN apt update -y
+
+RUN apt install -y git
+RUN apt-get install -y cmake g++
+RUN apt-get install -y libvulkan-dev
+# Swiftshader dependencies
+RUN apt-get install -y libx11-dev zlib1g-dev
+RUN apt-get install -y libxext-dev
+
+COPY --from=axsauze/swiftshader:0.1 /swiftshader/ /swiftshader/
+
+# Setup Python
+RUN apt-get install -y python3-pip
+
+RUN mkdir builder
+WORKDIR /builder
+
+
diff --git a/docker-builders/Makefile b/docker-builders/Makefile
new file mode 100644
index 000000000..ef79b4dfc
--- /dev/null
+++ b/docker-builders/Makefile
@@ -0,0 +1,19 @@
+
+build_kompute_builder:
+ docker build .. -f KomputeBuilder.Dockerfile -t axsauze/kompute-builder:0.2
+
+push_kompute_builder: build_kompute_builder
+ docker push axsauze/kompute-builder:0.2
+
+build_swiftshader:
+ docker build .. -f Swiftshader.Dockerfile -t axsauze/swiftshader:0.1
+
+push_swiftshader: build_swiftshader
+ docker push axsauze/swiftshader:0.1
+
+build_vulkan_sdk:
+ docker build .. -f VulkanSDK.Dockerfile -t axsauze/vulkan-sdk:0.1
+
+push_vulkan_sdk: build_vulkan_sdk
+ docker push axsauze/vulkan-sdk:0.1
+
diff --git a/docker-builders/Swiftshader.Dockerfile b/docker-builders/Swiftshader.Dockerfile
new file mode 100644
index 000000000..1a6e9ba37
--- /dev/null
+++ b/docker-builders/Swiftshader.Dockerfile
@@ -0,0 +1,34 @@
+FROM ubuntu:18.04
+
+# Base packages from default ppa
+RUN apt-get update -y
+RUN apt-get install -y wget
+RUN apt-get install -y gnupg
+RUN apt-get install -y ca-certificates
+RUN apt-get install -y software-properties-common
+
+# Repository to latest cmake
+RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add -
+RUN apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main'
+
+# Repository for latest git (needed for gh actions)
+RUN add-apt-repository -y ppa:git-core/ppa
+
+# Refresh repositories
+RUN apt update -y
+
+RUN apt install -y git
+RUN apt-get install -y cmake g++
+RUN apt-get install -y libvulkan-dev
+
+# Dependencies for swiftshader
+RUN apt-get install -y g++-8 gcc-8
+RUN apt-get install -y libx11-dev zlib1g-dev
+RUN apt-get install -y libxext-dev
+
+# Run swiftshader via env VK_ICD_FILENAMES=/swiftshader/vk_swiftshader_icd.json
+RUN git clone https://github.com/google/swiftshader swiftshader-build
+RUN CC="/usr/bin/gcc-8" CXX="/usr/bin/g++-8" cmake swiftshader-build/. -Bswiftshader-build/build/
+RUN cmake --build swiftshader-build/build/. --parallel
+RUN cp -r swiftshader-build/build/Linux/ swiftshader/
+
diff --git a/builders/Dockerfile.linux b/docker-builders/VulkanSDK.Dockerfile
similarity index 98%
rename from builders/Dockerfile.linux
rename to docker-builders/VulkanSDK.Dockerfile
index cdece9e59..ce42da579 100644
--- a/builders/Dockerfile.linux
+++ b/docker-builders/VulkanSDK.Dockerfile
@@ -29,7 +29,7 @@ RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/nul
RUN apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main'
# Adding security repo to add cmake dependency on libssl 1.0
RUN echo "deb http://security.ubuntu.com/ubuntu bionic-security main" | tee -a /etc/apt/sources.list.d/bionic.list
-RUN apt-get update
+RUN apt-get update --fix-missing -y
RUN apt-get upgrade -y
# Install build dependencies
diff --git a/docs/images/kompute-cpp-video.png b/docs/images/kompute-cpp-video.png
new file mode 100755
index 000000000..983244f0e
Binary files /dev/null and b/docs/images/kompute-cpp-video.png differ
diff --git a/docs/images/kompute-python-video.png b/docs/images/kompute-python-video.png
new file mode 100755
index 000000000..bd521fbc1
Binary files /dev/null and b/docs/images/kompute-python-video.png differ
diff --git a/docs/index.rst b/docs/index.rst
index 340b3458b..ff31c3f1c 100755
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -19,6 +19,7 @@ Index
Converting GLSL/HLSL Shaders to C++ Headers
Mobile App Integration (Android)
Game Engine Integration (Godot Engine)
+ CI, Docker Images & Tests
Python Class Documentation & Reference
C++ Class Documentation & Reference
Code Index
diff --git a/docs/overview/build-system.rst b/docs/overview/build-system.rst
index dd4f8e8b9..6f5a602cf 100644
--- a/docs/overview/build-system.rst
+++ b/docs/overview/build-system.rst
@@ -1,5 +1,5 @@
-Build System Deep Dive
+C++ Build System Deep Dive
======================
The recommended approach to build the project is as out-of-source build in the ``build`` folder. This project comes with a ``Makefile`` that provides a set of commands that help with developer workflows. You can see some of the commands if you want to add some of the more advanced commands.
diff --git a/docs/overview/ci-tests.rst b/docs/overview/ci-tests.rst
new file mode 100644
index 000000000..81443ec86
--- /dev/null
+++ b/docs/overview/ci-tests.rst
@@ -0,0 +1,66 @@
+
+CI, Docker Images & Tests
+======================
+
+This section contains an overview of the steps run on CI, as well as the tools used to simplify the testing (such as running Vulkan on CPU).
+
+We use Github Actions to run the tests, which simplifies the workflows significantly for contributors.
+
+The tests run on CPU, and can be triggered using the ACT command line interface (https://github.com/nektos/act) - once you install the command line (And start the Docker daemon) you just have to type:
+
+.. code-block::
+
+ $ act
+
+ [Python Tests/python-tests] 🚀 Start image=axsauze/kompute-builder:0.2
+ [C++ Tests/cpp-tests ] 🚀 Start image=axsauze/kompute-builder:0.2
+ [C++ Tests/cpp-tests ] 🐳 docker run image=axsauze/kompute-builder:0.2 entrypoint=["/usr/bin/tail" "-f" "/dev/null"] cmd=[]
+ [Python Tests/python-tests] 🐳 docker run image=axsauze/kompute-builder:0.2 entrypoint=["/usr/bin/tail" "-f" "/dev/null"] cmd=[]
+ ...
+
+
+CI Commands Triggered
+~~~~~~~~~~~~~
+
+The simplest way to see how this works is by looking at the github actions commands that are run.
+
+These can be found through the following files:
+
+* `CPP Tests `_
+* `Python Tests `_
+
+When submitting a PR or merging a PR into master, both of these will run - you can see the logs through the github interface.
+
+
+
+Running Vulkan on the CPU
+~~~~~~~~~~~~~
+
+We use `Swiftshader `_ to enable us to run the Vulkan Kompute framework directly on the CPU for the CI tests.
+
+Even though Swiftshader is optimized to function as a high-performance CPU backend for Vulkan, there are several limitations, the most notable in context of Kompute are:
+
+* Loading files (spirv or text) leads to segfault
+* Loading raw text string shaders leads to segfault
+
+This is one of the main reason why only a subset of the tests are run in the CI.
+
+Dockerfiles
+~~~~~~~~~~~~~
+
+The dockerfiles created provide functionality to simplify the interaction with the system.
+
+.. list-table::
+ :header-rows: 1
+
+ * - Image
+ - Description
+ * - axsauze/kompute-builder:0.2
+ - Main CI builder image with all required dependencies to build and run C++ & Python tests.
+ * - axsauze/swiftshader:0.1
+ - Image building Swiftshader libraries only to reduce time via multi-staged builds
+ * - axsauze/vulkan-sdk:0.1
+ - Image contained a linux build of the full Vulkan SDK to reduce time via multi-staged builds
+
+
+
diff --git a/python/test/requirements-dev.txt b/python/test/requirements-dev.txt
index c28827d1a..8d5576a72 100644
--- a/python/test/requirements-dev.txt
+++ b/python/test/requirements-dev.txt
@@ -1,2 +1,3 @@
pyshader==0.7.0
-numpy
+numpy==1.19.5
+pytest==6.2.1
diff --git a/setup.py b/setup.py
index 1896b8fb6..2a475491e 100644
--- a/setup.py
+++ b/setup.py
@@ -76,6 +76,9 @@ setup(
long_description=long_description,
long_description_content_type='text/markdown',
ext_modules=[CMakeExtension('kp')],
+ install_requires=[
+ "numpy<2.0.0"
+ ],
cmdclass=dict(build_ext=CMakeBuild),
zip_safe=False,
include_package_data=True,
diff --git a/test/TestLogisticRegression.cpp b/test/TestLogisticRegression.cpp
index c360542b6..94f34b5da 100644
--- a/test/TestLogisticRegression.cpp
+++ b/test/TestLogisticRegression.cpp
@@ -3,6 +3,8 @@
#include "kompute/Kompute.hpp"
+#include "kompute_test/shaders/shadertest_logistic_regression.hpp"
+
TEST(TestLogisticRegressionAlgorithm, TestMainLogisticRegression)
{
@@ -44,8 +46,18 @@ TEST(TestLogisticRegressionAlgorithm, TestMainLogisticRegression)
sq->record({ wIn, bIn });
+#ifdef KOMPUTE_SHADER_FROM_STRING
sq->record(
- params, "test/shaders/glsl/test_logistic_regression.comp");
+ params, "test/shaders/glsl/test_logistic_regression.comp.spv");
+#else
+ sq->record(
+ params,
+ std::vector(
+ kp::shader_data::shaders_glsl_logisticregression_comp_spv,
+ kp::shader_data::shaders_glsl_logisticregression_comp_spv +
+ kp::shader_data::
+ shaders_glsl_logisticregression_comp_spv_len));
+#endif
sq->record({ wOutI, wOutJ, bOut, lOut });
@@ -123,8 +135,18 @@ TEST(TestLogisticRegressionAlgorithm, TestMainLogisticRegressionManualCopy)
// Record op algo base
sq->begin();
+#ifdef KOMPUTE_SHADER_FROM_STRING
sq->record(
- params, "test/shaders/glsl/test_logistic_regression.comp");
+ params, "test/shaders/glsl/test_logistic_regression.comp.spv");
+#else
+ sq->record(
+ params,
+ std::vector(
+ kp::shader_data::shaders_glsl_logisticregression_comp_spv,
+ kp::shader_data::shaders_glsl_logisticregression_comp_spv +
+ kp::shader_data::
+ shaders_glsl_logisticregression_comp_spv_len));
+#endif
sq->record({ wOutI, wOutJ, bOut, lOut });
diff --git a/test/shaders/glsl/test_op_custom_shader.comp b/test/shaders/glsl/test_op_custom_shader.comp
index 74241d141..1e0e0d199 100644
--- a/test/shaders/glsl/test_op_custom_shader.comp
+++ b/test/shaders/glsl/test_op_custom_shader.comp
@@ -1,7 +1,10 @@
#version 450
+
layout (local_size_x = 1) in;
+
layout(set = 0, binding = 0) buffer a { float pa[]; };
layout(set = 0, binding = 1) buffer b { float pb[]; };
+
void main() {
uint index = gl_GlobalInvocationID.x;
pb[index] = pa[index];
+
+
+
+