Merge pull request #235 from EthicalML/remove_glslang_as_core_dep

Removing GLSLang as core dependency

.ccls

@@ -24,4 +24,5 @@
 -I./single_include/
 -I./vk_ndk_wrapper_include/
 -I./test/compiled_shaders_include/
+-I./test/utils/
 

.github/workflows/cpp_tests.yml

@@ -10,7 +10,7 @@ jobs:
   cpp-tests:
 
     runs-on: ubuntu-18.04
-    container: axsauze/kompute-builder:0.2
+    container: axsauze/kompute-builder:0.3
 
     steps:
     - uses: actions/checkout@v2

.github/workflows/python_tests.yml

@@ -10,7 +10,7 @@ jobs:
   python-tests:
 
     runs-on: ubuntu-18.04
-    container: axsauze/kompute-builder:0.2
+    container: axsauze/kompute-builder:0.3
 
     steps:
     - uses: actions/checkout@v2

.gitignore

@@ -186,5 +186,5 @@ release/
 # Kompute
 swiftshader/
 vk_swiftshader_icd.json
-
+tmp_kp_shader.comp.spv
 

.gitmodules

@@ -14,10 +14,6 @@
 	path = python/pybind11
 	url = https://github.com/pybind/pybind11
     branch = v2.6.1
-[submodule "external/glslang"]
-	path = external/glslang
-	url = https://github.com/KhronosGroup/glslang/
-    branch = 11.1.0
 [submodule "external/fmt"]
 	path = external/fmt
 	url = https://github.com/fmtlib/fmt

CHANGELOG.md

@@ -34,7 +34,7 @@
 **Closed issues:**
 
 - Update memory barriers to align with tensor staging/primary memory revamp [\#181](https://github.com/EthicalML/vulkan-kompute/issues/181)
-- Move shader defaultResource inside kp::Shader class [\#175](https://github.com/EthicalML/vulkan-kompute/issues/175)
+- Move shader defaultResource inside kp_test_utils::Shader class [\#175](https://github.com/EthicalML/vulkan-kompute/issues/175)
 - Reach at least 90% code coverage on tests [\#170](https://github.com/EthicalML/vulkan-kompute/issues/170)
 - Add functionality to re-record sequence as now it's possible to update the underlying algorithm [\#169](https://github.com/EthicalML/vulkan-kompute/issues/169)
 - Use numpy arrays as default return value [\#166](https://github.com/EthicalML/vulkan-kompute/issues/166)

CMakeLists.txt

@@ -19,7 +19,6 @@ option(KOMPUTE_OPT_ENABLE_SPDLOG "Extra compile flags for Kompute, see docs for
 option(KOMPUTE_OPT_REPO_SUBMODULE_BUILD "Use the submodule repos instead of external package manager" 0)
 option(KOMPUTE_OPT_ANDROID_BUILD "Enable android compilation flags required" 0)
 option(KOMPUTE_OPT_DISABLE_VK_DEBUG_LAYERS "Explicitly disable debug layers even on debug" 0)
-option(KOMPUTE_OPT_DISABLE_SHADER_UTILS "Remove shader util code and dependencies including glslang" 0)
 option(KOMPUTE_OPT_DEPENDENCIES_SHARED_LIBS "Whether to use shared libraries for dependencies for install" 0)
 option(KOMPUTE_OPT_BUILD_AS_SHARED_LIB "Whether to build kompute as shared library" 0)
 # Build flags
@@ -55,19 +54,15 @@ if(KOMPUTE_OPT_DISABLE_VK_DEBUG_LAYERS)
     set(KOMPUTE_EXTRA_CXX_FLAGS "${KOMPUTE_EXTRA_CXX_FLAGS} -DKOMPUTE_DISABLE_VK_DEBUG_LAYERS=1")
 endif()
 
-if(NOT KOMPUTE_OPT_DISABLE_SHADER_UTILS)
-    if(KOMPUTE_OPT_INSTALL)
-        # Enable install parameters for glslang (overrides parameters passed)
-        # When install is enabled the glslang libraries become shared
-        set(ENABLE_GLSLANG_INSTALL ON CACHE BOOL "Enables install of glslang" FORCE)
+if(KOMPUTE_OPT_INSTALL)
+    # Enable install parameters for glslang (overrides parameters passed)
+    # When install is enabled the glslang libraries become shared
+    set(ENABLE_GLSLANG_INSTALL ON CACHE BOOL "Enables install of glslang" FORCE)
 
-        # By default we enable shared library based installation
-        if(KOMPUTE_OPT_DEPENDENCIES_SHARED_LIBS)
-            set(BUILD_SHARED_LIBS ON CACHE BOOL "Enables build of shared libraries" FORCE)
-        endif()
+    # By default we enable shared library based installation
+    if(KOMPUTE_OPT_DEPENDENCIES_SHARED_LIBS)
+        set(BUILD_SHARED_LIBS ON CACHE BOOL "Enables build of shared libraries" FORCE)
     endif()
-else()
-    set(KOMPUTE_EXTRA_CXX_FLAGS "${KOMPUTE_EXTRA_CXX_FLAGS} -DKOMPUTE_DISABLE_SHADER_UTILS=1")
 endif()
 
 set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DDEBUG=1 ${KOMPUTE_EXTRA_CXX_FLAGS} -DUSE_DEBUG_EXTENTIONS")

Makefile

@@ -194,7 +194,7 @@ build_single_header:
 		"single_include/kompute/Kompute.hpp"
 
 win_build_xxd:
-	cd external/bin/ && gcc -o xxd.exe xxd.c -DCYGWIN
+	cd external/bin/ && gcc.exe -o xxd.exe xxd.c -DCYGWIN
 
 format:
 	$(CLANG_FORMAT_BIN) -i -style="{BasedOnStyle: mozilla, IndentWidth: 4}" src/*.cpp src/include/kompute/*.hpp test/*cpp

README.md

@@ -74,7 +74,8 @@ void kompute(const std::string& shader) {
     kp::Constants pushConstsB({ 3.0 });
 
     auto algorithm = mgr.algorithm(params,
-                                   kp::Shader::compileSource(shader),
+                                   // See documentation shader section for compileSource
+                                   compileSource(shader),
                                    workgroup,
                                    specConsts,
                                    pushConstsA);
@@ -165,6 +166,7 @@ def kompute(shader):
     push_consts_a = [2]
     push_consts_b = [3]
 
+    # See documentation shader section for compile_source
     spirv = kp.Shader.compile_source(shader)
 
     algo = mgr.algorithm(params, spirv, workgroup, spec_consts, push_consts_a)
@@ -372,7 +374,7 @@ You can also access the <a href="https://github.com/EthicalML/vulkan-kompute/tre
 
 ### Simple examples
 
-* [Pass shader as raw string](https://kompute.cc/overview/advanced-examples.html#simple-shader-example)
+* [Simple multiplication example](https://kompute.cc/overview/advanced-examples.html#simple-shader-example)
 * [Record batch commands with a Kompute Sequence](https://kompute.cc/overview/advanced-examples.html#record-batch-commands)
 * [Run Asynchronous Operations](https://kompute.cc/overview/advanced-examples.html#asynchronous-operations)
 * [Run Parallel Operations Across Multiple GPU Queues](https://kompute.cc/overview/advanced-examples.html#parallel-operations)

docker-builders/KomputeBuilder.Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 
 # Base packages from default ppa
 RUN apt-get update -y
@@ -26,9 +26,14 @@ RUN apt-get install -y libxext-dev
 
 COPY --from=axsauze/swiftshader:0.1 /swiftshader/ /swiftshader/
 
+# GLSLANG tools for tests
+RUN apt-get install -y glslang-tools
+
 # Setup Python
 RUN apt-get install -y python3-pip
 
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10
+
 RUN mkdir builder
 WORKDIR /builder
 

docker-builders/Makefile

@@ -1,9 +1,9 @@
 
 build_kompute_builder:
-	docker build .. -f KomputeBuilder.Dockerfile -t axsauze/kompute-builder:0.2
+	docker build .. -f KomputeBuilder.Dockerfile -t axsauze/kompute-builder:0.3
 
 push_kompute_builder: build_kompute_builder
-	docker push axsauze/kompute-builder:0.2
+	docker push axsauze/kompute-builder:0.3
 
 build_swiftshader:
 	docker build .. -f Swiftshader.Dockerfile -t axsauze/swiftshader:0.1

docs/overview/advanced-examples.rst

@@ -55,7 +55,8 @@ The example below shows how you can enable the "VK_EXT_shader_atomic_float" exte
                  atomicAdd(pa[2], pcs.z);
              })");
 
-       std::vector<uint32_t> spirv = kp::Shader::compileSource(shader);
+       // See shader documentation section for compileSource
+       std::vector<uint32_t> spirv = compileSource(shader);
 
        std::shared_ptr<kp::Sequence> sq = nullptr;
 
@@ -102,7 +103,8 @@ We also provide tools that allow you to `convert shaders into C++ headers <https
                 throw std::runtime_error("Kompute OpMult expected 3 tensors but got " + tensors.size());
             }
 
-            std::vector<uint32_t> spirv = kp::Shader::compileSource(R"(
+           // See shader documentation section for compileSource
+            std::vector<uint32_t> spirv = compileSource(R"(
                 #version 450
 
                 layout(set = 0, binding = 0) buffer tensorLhs {
@@ -215,7 +217,8 @@ In this case we create a shader that should take a couple of milliseconds to run
         }
     )");
 
-    auto algo = mgr.algorithm({tensor}, kp::Shader::compileSource(shader));
+    // See shader documentation section for compileSource
+    auto algo = mgr.algorithm({tensor}, compileSource(shader));
 
 Now we are able to run the await function on the default sequence. 
 
@@ -361,7 +364,8 @@ Similar to the asyncrhonous usecase above, we can still run synchronous commands
            }
        )");
 
-       std::vector<uint32_t> spirv = kp::Shader::compileSource(shader);
+       // See shader documentation section for compileSource
+       std::vector<uint32_t> spirv = compileSource(shader);
 
        std::shared_ptr<kp::Algorithm> algo = mgr.algorithm({tensorA, tenssorB}, spirv);
 

docs/overview/build-system.rst

@@ -69,8 +69,6 @@ Compile Flags
      - Enable debug build including debug flags (enabled by cmake debug build)
    * - -DKOMPUTE_DISABLE_VK_DEBUG_LAYERS
      - Disable the debug Vulkan SDK Layers, mainly used for android builds
-   * - -DKOMPUTE_DISABLE_SHADER_UTILS
-     - Disable the shader utils and skip adding glslang as dependency
 
 Other CMake Flags
 ~~~~~~~~~~~~~~~~~

docs/overview/ci-tests.rst

@@ -38,10 +38,7 @@ Running on the CPU
 
 We use `Swiftshader <https://github.com/google/swiftshader>`_ to enable us to run the Kompute framework directly on the CPU for the CI tests.
 
-Even though Swiftshader is optimized to function as a high-performance CPU backend for the Vulkan SDK, there are several limitations, the most notable in context of Kompute are:
-
-* Loading files (spirv or text) leads to segfault
-* Loading raw text string shaders leads to segfault
+Even though Swiftshader is optimized to function as a high-performance CPU backend for the Vulkan SDK, there are several limitations, the most notable are limitations in extensions.
 
 This is one of the main reason why only a subset of the tests are run in the CI.
 

docs/overview/reference.rst

@@ -119,13 +119,4 @@ The :class:`kp::OpMemoryBarrier` is a tensor only operation which adds memory ba
 .. doxygenclass:: kp::OpTensorSyncDevice
    :members:
 
-Shader
---------
-
-The :class:`kp::Shader` class contains a set of utilities to compile and process shaders.
-
-.. doxygenclass:: kp::Shader
-   :members:
-
-
 

docs/overview/shaders-to-headers.rst

@@ -3,21 +3,29 @@
 Processing Shaders with Kompute
 =====================
 
-Kompute allows for two main ways of interacting with shaders - namely:
+Demo / testing function to compile shaders
+----------------------------------
 
-* Integration with [glslang](https://github.com/KhronosGroup/glslang) for online/runtime shader compilation
-* A CLI that coverts shaders into C++ header files
+GLSLANG was initially integrated as part of the framework but it now has been removed due to the license of the glslang pre-processor being under a custom NVIDIA license which explicitly excludes grant of any licenses to NVIDIA's patents in the preprocessor. This is covered in more detail here: https://github.com/EthicalML/vulkan-kompute/pull/235
 
-Processing Shaders Online via Kompute Shader Utils
----------------
+For users that are looking to quickly test the processors it is possible to use the function that is provided in the examples which provides a (non-thread-safe / non-robust) implementation that compiles a shader string into spirv bytes. It is not recommended to use in production but it does enable for faster iteration cycles during development.
 
-Kompute provides a set of helper functions that expose the C++ functionality of the glslang Khronos framework to process shader sources online during runtime.
+.. code-block:: cpp
+    :linenos:
 
-It's worth emphasising that the suggested approach is to process shaders offline, so the section below is suggested to convert shaders to either their respective SPV format, or convert them into C++ sources that would be embedded as part of the resulting binary.
-
-The Shader utility function can be skipped on build time through compiler flags - for more information on this you should read the `build section <build-system.rst>`_.
-
-More details on the shader utils can be found in the :class:`kp::Shader` section of the `C++ reference page <reference.rst>`_.
+    static std::vector<uint32_t>
+    compileSource(
+      const std::string& source)
+    {
+        if (system(std::string("glslangValidator --stdin -S comp -V -o tmp_kp_shader.comp.spv << END
+    " + source + "
+    END").c_str()))
+            throw std::runtime_error("Error running glslangValidator command");
+        std::ifstream fileStream("tmp_kp_shader.comp.spv", std::ios::binary);
+        std::vector<char> buffer;
+        buffer.insert(buffer.begin(), std::istreambuf_iterator<char>(fileStream), {});
+        return {(uint32_t*)buffer.data(), (uint32_t*)(buffer.data() + buffer.size())};
+    }
 
 Converting Shaders into C / C++ Header Files
 ----------------------------------

examples/array_multiplication/src/Main.cpp

@@ -5,6 +5,18 @@
 
 #include "kompute/Kompute.hpp"
 
+static std::vector<uint32_t>
+compileSource(
+  const std::string& source)
+{
+    if (system(std::string("glslangValidator --stdin -S comp -V -o tmp_kp_shader.comp.spv << END\n" + source + "\nEND").c_str()))
+        throw std::runtime_error("Error running glslangValidator command");
+    std::ifstream fileStream("tmp_kp_shader.comp.spv", std::ios::binary);
+    std::vector<char> buffer;
+    buffer.insert(buffer.begin(), std::istreambuf_iterator<char>(fileStream), {});
+    return {(uint32_t*)buffer.data(), (uint32_t*)(buffer.data() + buffer.size())};
+}
+
 int main()
 {
 #if KOMPUTE_ENABLE_SPDLOG
@@ -39,7 +51,7 @@ int main()
 
     std::vector<std::shared_ptr<kp::Tensor>> params = { tensorInA, tensorInB, tensorOut };
 
-    std::shared_ptr<kp::Algorithm> algo = mgr.algorithm(params, kp::Shader::compileSource(shader));
+    std::shared_ptr<kp::Algorithm> algo = mgr.algorithm(params, kp_test_utils::compileSource(shader));
 
     mgr.sequence()
         ->record<kp::OpTensorSyncDevice>(params)

examples/godot_examples/custom_module/kompute_summator/KomputeSummatorNode.cpp

@@ -4,6 +4,18 @@
 
 #include "KomputeSummatorNode.h"
 
+static std::vector<uint32_t>
+compileSource(
+  const std::string& source)
+{
+    if (system(std::string("glslangValidator --stdin -S comp -V -o tmp_kp_shader.comp.spv << END\n" + source + "\nEND").c_str()))
+        throw std::runtime_error("Error running glslangValidator command");
+    std::ifstream fileStream("tmp_kp_shader.comp.spv", std::ios::binary);
+    std::vector<char> buffer;
+    buffer.insert(buffer.begin(), std::istreambuf_iterator<char>(fileStream), {});
+    return {(uint32_t*)buffer.data(), (uint32_t*)(buffer.data() + buffer.size())};
+}
+
 KomputeSummatorNode::KomputeSummatorNode() {
     this->_init();
 }
@@ -52,9 +64,9 @@ void KomputeSummatorNode::_init() {
         )");
 
         std::shared_ptr<kp::Algorithm> algo =
-          mgr.algorithm(
+          this->mManager.algorithm(
                 { this->mPrimaryTensor, this->mSecondaryTensor },
-                kp::Shader::compileSource(shader));
+                compileSource(shader));
 
 
         // First we ensure secondary tensor loads to GPU
@@ -63,7 +75,7 @@ void KomputeSummatorNode::_init() {
                 { this->mSecondaryTensor });
 
         // Then we run the operation with both tensors
-        sq->record<kp::OpAlgoDispatch>(algo)
+        sq->record<kp::OpAlgoDispatch>(algo);
 
         // We map the result back to local 
         sq->record<kp::OpTensorSyncLocal>(

examples/godot_examples/gdnative_shared/src/KomputeSummator.cpp

@@ -5,6 +5,18 @@
 
 #include "KomputeSummator.hpp"
 
+static std::vector<uint32_t>
+compileSource(
+  const std::string& source)
+{
+    if (system(std::string("glslangValidator --stdin -S comp -V -o tmp_kp_shader.comp.spv << END\n" + source + "\nEND").c_str()))
+        throw std::runtime_error("Error running glslangValidator command");
+    std::ifstream fileStream("tmp_kp_shader.comp.spv", std::ios::binary);
+    std::vector<char> buffer;
+    buffer.insert(buffer.begin(), std::istreambuf_iterator<char>(fileStream), {});
+    return {(uint32_t*)buffer.data(), (uint32_t*)(buffer.data() + buffer.size())};
+}
+
 namespace godot {
 
 KomputeSummator::KomputeSummator() {
@@ -58,7 +70,7 @@ void KomputeSummator::_init() {
         // Then we run the operation with both tensors
         this->mSequence->record<kp::OpAlgoCreate>(
             { this->mPrimaryTensor, this->mSecondaryTensor }, 
-            kp::Shader::compileSource(shader));
+            compileSource(shader));
 
         // We map the result back to local 
         this->mSequence->record<kp::OpTensorSyncLocal>(

examples/neural_network_vgg7/sh_conv.py

@@ -1,8 +1,15 @@
+import os
+
 import kp
 
+def compile_source(source):
+    os.system("glslangValidator --stdin -S comp -V -o tmp_kp_shader.comp.spv << END\n" + source + "\nEND")
+    return open("tmp_kp_shader.comp.spv", "rb").read()
+
+
 # This is the convolution & leakyrelu shader.
 global conv_shader
-conv_shader = kp.Shader.compile_source("""
+conv_shader = compile_source("""
 #version 450
 
 layout (local_size_x = 8, local_size_y = 2) in;

external/glslang

@@ -1 +0,0 @@
-Subproject commit c594de23cdd790d64ad5f9c8b059baae0ee2941d

python/src/main.cpp

@@ -33,34 +33,6 @@ PYBIND11_MODULE(kp, m) {
         .value("storage", kp::Tensor::TensorTypes::eStorage, DOC(kp, Tensor, TensorTypes, eStorage))
         .export_values();
 
-#if !defined(KOMPUTE_DISABLE_SHADER_UTILS) || !KOMPUTE_DISABLE_SHADER_UTILS
-    py::class_<kp::Shader>(m, "Shader", "Shader class")
-        .def_static("compile_source", [](
-                                    const std::string& source,
-                                    const std::string& entryPoint,
-                                    const std::vector<std::pair<std::string,std::string>>& definitions) {
-                std::vector<uint32_t> spirv = kp::Shader::compileSource(source, entryPoint, definitions);
-                return py::bytes((const char*)spirv.data(), spirv.size() * sizeof(uint32_t));
-            },
-            DOC(kp, Shader, compileSource),
-            py::arg("source"),
-            py::arg("entryPoint") = "main",
-            py::arg("definitions") = std::vector<std::pair<std::string,std::string>>() )
-        .def_static("compile_sources", [](
-                                    const std::vector<std::string>& source,
-                                    const std::vector<std::string>& files,
-                                    const std::string& entryPoint,
-                                    const std::vector<std::pair<std::string,std::string>>& definitions) {
-                std::vector<uint32_t> spirv = kp::Shader::compileSources(source, files, entryPoint, definitions);
-                return py::bytes((const char*)spirv.data(), spirv.size() * sizeof(uint32_t));
-            },
-            DOC(kp, Shader, compileSources),
-            py::arg("sources"),
-            py::arg("files") = std::vector<std::string>(),
-            py::arg("entryPoint") = "main",
-            py::arg("definitions") = std::vector<std::pair<std::string,std::string>>() );
-#endif // KOMPUTE_DISABLE_SHADER_UTILS
-
     py::class_<kp::OpBase, std::shared_ptr<kp::OpBase>>(m, "OpBase", DOC(kp, OpBase));
 
     py::class_<kp::OpTensorSyncDevice, std::shared_ptr<kp::OpTensorSyncDevice>>(

python/test/test_kompute.py

@@ -5,10 +5,13 @@ import numpy as np
 import logging
 import pyshader as ps
 
+from .utils import compile_source
+
 DIRNAME = os.path.dirname(os.path.abspath(__file__))
 
 kp_log = logging.getLogger("kp")
 
+
 def test_end_to_end():
 
     mgr = kp.Manager()
@@ -52,7 +55,7 @@ def test_end_to_end():
     push_consts_a = [2]
     push_consts_b = [3]
 
-    algo = mgr.algorithm(params, kp.Shader.compile_source(shader), workgroup, spec_consts, push_consts_a)
+    algo = mgr.algorithm(params, compile_source(shader), workgroup, spec_consts, push_consts_a)
 
     (mgr.sequence()
         .record(kp.OpTensorSyncDevice(params))
@@ -88,7 +91,7 @@ void main()
 }
     """
 
-    spirv = kp.Shader.compile_source(shader)
+    spirv = compile_source(shader)
 
     mgr = kp.Manager()
 
@@ -108,6 +111,7 @@ void main()
 
     assert tensor_out.data().tolist() == [2.0, 4.0, 6.0]
 
+
 def test_sequence():
     """
     Test basic OpAlgoBase operation
@@ -127,7 +131,7 @@ def test_sequence():
         }
     """
 
-    spirv = kp.Shader.compile_source(shader)
+    spirv = compile_source(shader)
 
     mgr = kp.Manager(0)
 
@@ -164,9 +168,10 @@ def test_sequence():
     assert tensor_in_b.is_init() == False
     assert tensor_out.is_init() == False
 
+
 def test_pushconsts():
 
-    spirv = kp.Shader.compile_source("""
+    spirv = compile_source("""
           #version 450
           layout(push_constant) uniform PushConstants {
             float x;
@@ -197,6 +202,7 @@ def test_pushconsts():
 
     assert np.all(tensor.data() == np.array([0.4, 0.4, 0.4], dtype=np.float32))
 
+
 def test_workgroup():
     mgr = kp.Manager(0)
 
@@ -227,6 +233,7 @@ def test_workgroup():
     assert np.all(tensor_a.data() == np.stack([np.arange(16)]*8, axis=1).ravel())
     assert np.all(tensor_b.data() == np.stack([np.arange(8)]*16, axis=0).ravel())
 
+
 def test_mgr_utils():
     mgr = kp.Manager()
 

python/test/test_tensor_types.py

@@ -1,9 +1,10 @@
-import pyshader as ps
 import os
 import pytest
 import kp
 import numpy as np
 
+from .utils import compile_source
+
 VK_ICD_FILENAMES = os.environ.get("VK_ICD_FILENAMES", "")
 
 def test_type_float():
@@ -22,7 +23,7 @@ def test_type_float():
         }
     """
 
-    spirv = kp.Shader.compile_source(shader)
+    spirv = compile_source(shader)
 
     arr_in_a = np.array([123., 153., 231.], dtype=np.float32)
     arr_in_b = np.array([9482, 1208, 1238], dtype=np.float32)
@@ -61,7 +62,7 @@ def test_type_float_double_incorrect():
         }
     """
 
-    spirv = kp.Shader.compile_source(shader)
+    spirv = compile_source(shader)
 
     arr_in_a = np.array([123., 153., 231.], dtype=np.float32)
     arr_in_b = np.array([9482, 1208, 1238], dtype=np.uint32)
@@ -103,7 +104,7 @@ def test_type_double():
         }
     """
 
-    spirv = kp.Shader.compile_source(shader)
+    spirv = compile_source(shader)
 
     arr_in_a = np.array([123., 153., 231.], dtype=np.float64)
     arr_in_b = np.array([9482, 1208, 1238], dtype=np.float64)
@@ -143,7 +144,7 @@ def test_type_int():
         }
     """
 
-    spirv = kp.Shader.compile_source(shader)
+    spirv = compile_source(shader)
 
     arr_in_a = np.array([123, 153, 231], dtype=np.int32)
     arr_in_b = np.array([9482, 1208, 1238], dtype=np.int32)
@@ -183,7 +184,7 @@ def test_type_unsigned_int():
         }
     """
 
-    spirv = kp.Shader.compile_source(shader)
+    spirv = compile_source(shader)
 
     arr_in_a = np.array([123, 153, 231], dtype=np.uint32)
     arr_in_b = np.array([9482, 1208, 1238], dtype=np.uint32)

python/test/utils.py

@@ -0,0 +1,7 @@
+import os
+
+
+def compile_source(source):
+    os.system("glslangValidator --stdin -S comp -V -o tmp_kp_shader.comp.spv << END\n" + source + "\nEND")
+    return open("tmp_kp_shader.comp.spv", "rb").read()
+

single_include/AggregateHeaders.cpp

@@ -2,7 +2,6 @@
 #include "kompute/shaders/shaderopmult.hpp"
 #include "kompute/shaders/shaderlogisticregression.hpp"
 #include "kompute/Core.hpp"
-#include "kompute/Shader.hpp"
 #include "kompute/Tensor.hpp"
 #include "kompute/Algorithm.hpp"
 #include "kompute/operations/OpBase.hpp"

single_include/kompute/Kompute.hpp

@@ -725,78 +725,6 @@ extern py::object kp_debug, kp_info, kp_warning, kp_error;
 #endif // KOMPUTE_SPDLOG_ENABLED
 #endif // KOMPUTE_LOG_OVERRIDE
 
-#if !defined(KOMPUTE_DISABLE_SHADER_UTILS) || !KOMPUTE_DISABLE_SHADER_UTILS
-#include <iostream>
-#include <vector>
-
-#ifdef USE_EXTERNAL_GLSLANG
-#include <SPIRV/GlslangToSpv.h>
-#else 
-#include <glslang/SPIRV/GlslangToSpv.h>
-#endif
-
-#include <glslang/Include/ResourceLimits.h>
-#include <glslang/Public/ShaderLang.h>
-
-namespace kp {
-
-/**
-    Shader utily class with functions to compile and process glsl files.
-*/
-class Shader
-{
-  public:
-    // The default resource limit for the GLSL compiler, can be overwritten
-    // Has been adopted by:
-    // https://github.com/KhronosGroup/glslang/blob/master/StandAlone/ResourceLimits.cpp
-    const static TBuiltInResource defaultResource;
-
-    /**
-     * Compile multiple sources with optional filenames. Currently this function
-     * uses the glslang C++ interface which is not thread safe so this funciton
-     * should not be called from multiple threads concurrently. If you have a
-     * online shader processing multithreading use-case that can't use offline
-     * compilation please open an issue.
-     *
-     * @param sources A list of raw glsl shaders in string format
-     * @param files A list of file names respective to each of the sources
-     * @param entryPoint The function name to use as entry point
-     * @param definitions List of pairs containing key value definitions
-     * @param resourcesLimit A list that contains the resource limits for the
-     * GLSL compiler
-     * @return The compiled SPIR-V binary in unsigned int32 format
-     */
-    static std::vector<uint32_t> compileSources(
-      const std::vector<std::string>& sources,
-      const std::vector<std::string>& files = {},
-      const std::string& entryPoint = "main",
-      std::vector<std::pair<std::string, std::string>> definitions = {},
-      const TBuiltInResource& resources = Shader::defaultResource);
-
-    /**
-     * Compile a single glslang source from string value. Currently this
-     * function uses the glslang C++ interface which is not thread safe so this
-     * funciton should not be called from multiple threads concurrently. If you
-     * have a online shader processing multithreading use-case that can't use
-     * offline compilation please open an issue.
-     *
-     * @param source An individual raw glsl shader in string format
-     * @param entryPoint The function name to use as entry point
-     * @param definitions List of pairs containing key value definitions
-     * @param resourcesLimit A list that contains the resource limits for the
-     * GLSL compiler
-     * @return The compiled SPIR-V binary in unsigned int32 format
-     */
-    static std::vector<uint32_t> compileSource(
-      const std::string& source,
-      const std::string& entryPoint = "main",
-      std::vector<std::pair<std::string, std::string>> definitions = {},
-      const TBuiltInResource& resources = Shader::defaultResource);
-};
-
-}
-#endif // DKOMPUTE_DISABLE_SHADER_UTILS
-
 namespace kp {
 
 /**

src/CMakeLists.txt

@@ -141,41 +141,6 @@ if(KOMPUTE_OPT_BUILD_SINGLE_HEADER)
         build_single_header)
 endif()
 
-#####################################################
-#################### GLSLANG #######################
-#####################################################
-
-if(NOT KOMPUTE_OPT_DISABLE_SHADER_UTILS)
-    if(KOMPUTE_OPT_REPO_SUBMODULE_BUILD)
-        add_subdirectory(${PROJECT_SOURCE_DIR}/external/glslang
-            ${CMAKE_CURRENT_BINARY_DIR}/kompute_glslang)
-
-        target_include_directories(
-            kompute PRIVATE
-            ${PROJECT_SOURCE_DIR}/external/glslang)
-
-        target_link_libraries(kompute
-            # Not including hlsl support
-            # HLSL
-            # glslang includes OGLCompiler, OSDependent, MachineIndependent
-            glslang
-            SPIRV)
-    else()
-        find_package(glslang CONFIG REQUIRED)
-
-        target_include_directories(
-            kompute PRIVATE
-            ${GLSLANG_GENERATED_INCLUDEDIR})
-
-        target_link_libraries(kompute
-            # Not including hlsl support
-            # glslang::HLSL
-            # Adding explicit dependencies to match above
-            glslang::glslang
-            glslang::SPIRV)
-    endif()
-endif()
-
 
 add_library(kompute::kompute ALIAS kompute)
 

src/Shader.cpp

@@ -1,218 +0,0 @@
-
-#if !defined(KOMPUTE_DISABLE_SHADER_UTILS) || !KOMPUTE_DISABLE_SHADER_UTILS
-#include "kompute/Shader.hpp"
-
-namespace kp {
-
-std::vector<uint32_t>
-Shader::compileSources(
-  const std::vector<std::string>& sources,
-  const std::vector<std::string>& files,
-  const std::string& entryPoint,
-  std::vector<std::pair<std::string, std::string>> definitions,
-  const TBuiltInResource& resources)
-{
-
-    // Initialize glslang library.
-    glslang::InitializeProcess();
-
-    // Currently we don't support other shader types nor plan to
-    const EShLanguage language = EShLangCompute;
-    glslang::TShader shader(language);
-
-    std::vector<const char*> filesCStr(files.size()),
-      sourcesCStr(sources.size());
-    for (size_t i = 0; i < sources.size(); i++)
-        sourcesCStr[i] = sources[i].c_str();
-
-    if (files.size() > 1) {
-        assert(files.size() == sources.size());
-        for (size_t i = 0; i < files.size(); i++)
-            filesCStr[i] = files[i].c_str();
-        shader.setStringsWithLengthsAndNames(
-          sourcesCStr.data(), nullptr, filesCStr.data(), filesCStr.size());
-    } else {
-        filesCStr = { "" };
-        shader.setStringsWithLengthsAndNames(
-          sourcesCStr.data(), nullptr, filesCStr.data(), sourcesCStr.size());
-    }
-
-    shader.setEntryPoint(entryPoint.c_str());
-    shader.setSourceEntryPoint(entryPoint.c_str());
-
-    std::string info_log = "";
-    const EShMessages messages = static_cast<EShMessages>(
-      EShMsgDefault | EShMsgVulkanRules | EShMsgSpvRules);
-    if (!shader.parse(&resources, 100, false, messages)) {
-        info_log = std::string(shader.getInfoLog()) + "\n" +
-                   std::string(shader.getInfoDebugLog());
-        KP_LOG_ERROR("Kompute Shader Error: {}", info_log);
-        throw std::runtime_error(info_log);
-    }
-
-    // Add shader to new program object.
-    glslang::TProgram program;
-    program.addShader(&shader);
-    // Link program.
-    if (!program.link(messages)) {
-        info_log = std::string(program.getInfoLog()) + "\n" +
-                   std::string(program.getInfoDebugLog());
-        KP_LOG_ERROR("Kompute Shader Error: {}", info_log);
-        throw std::runtime_error(info_log);
-    }
-
-    // Save any info log that was generated.
-    if (shader.getInfoLog()) {
-        info_log += std::string(shader.getInfoLog()) + "\n" +
-                    std::string(shader.getInfoDebugLog()) + "\n";
-        KP_LOG_INFO("Kompute Shader Information: {}", info_log);
-    }
-
-    glslang::TIntermediate* intermediate = program.getIntermediate(language);
-    // Translate to SPIRV.
-    if (!intermediate) {
-        info_log += "Failed to get shared intermediate code.\n";
-        KP_LOG_ERROR("Kompute Shader Error: {}", info_log);
-        throw std::runtime_error(info_log);
-    }
-
-    spv::SpvBuildLogger logger;
-    std::vector<std::uint32_t> spirv;
-    glslang::GlslangToSpv(*intermediate, spirv, &logger);
-
-    if (shader.getInfoLog()) {
-        info_log += logger.getAllMessages() + "\n";
-        KP_LOG_DEBUG("Kompute Shader all result messages: {}", info_log);
-    }
-
-    // Shutdown glslang library.
-    glslang::FinalizeProcess();
-
-    return spirv;
-}
-
-std::vector<uint32_t>
-Shader::compileSource(
-  const std::string& source,
-  const std::string& entryPoint,
-  std::vector<std::pair<std::string, std::string>> definitions,
-  const TBuiltInResource& resource)
-{
-    return compileSources({ source },
-                          std::vector<std::string>({}),
-                          entryPoint,
-                          definitions,
-                          resource);
-}
-
-const TBuiltInResource Shader::defaultResource = {
-    /* .MaxLights = */ 0,
-    /* .MaxClipPlanes = */ 0,
-    /* .MaxTextureUnits = */ 0,
-    /* .MaxTextureCoords = */ 0,
-    /* .MaxVertexAttribs = */ 64,
-    /* .MaxVertexUniformComponents = */ 4096,
-    /* .MaxVaryingFloats = */ 64,
-    /* .MaxVertexTextureImageUnits = */ 0,
-    /* .MaxCombinedTextureImageUnits = */ 0,
-    /* .MaxTextureImageUnits = */ 0,
-    /* .MaxFragmentUniformComponents = */ 0,
-    /* .MaxDrawBuffers = */ 0,
-    /* .MaxVertexUniformVectors = */ 128,
-    /* .MaxVaryingVectors = */ 8,
-    /* .MaxFragmentUniformVectors = */ 0,
-    /* .MaxVertexOutputVectors = */ 16,
-    /* .MaxFragmentInputVectors = */ 0,
-    /* .MinProgramTexelOffset = */ -8,
-    /* .MaxProgramTexelOffset = */ 7,
-    /* .MaxClipDistances = */ 8,
-    /* .MaxComputeWorkGroupCountX = */ 65535,
-    /* .MaxComputeWorkGroupCountY = */ 65535,
-    /* .MaxComputeWorkGroupCountZ = */ 65535,
-    /* .MaxComputeWorkGroupSizeX = */ 1024,
-    /* .MaxComputeWorkGroupSizeY = */ 1024,
-    /* .MaxComputeWorkGroupSizeZ = */ 64,
-    /* .MaxComputeUniformComponents = */ 1024,
-    /* .MaxComputeTextureImageUnits = */ 16,
-    /* .MaxComputeImageUniforms = */ 8,
-    /* .MaxComputeAtomicCounters = */ 8,
-    /* .MaxComputeAtomicCounterBuffers = */ 1,
-    /* .MaxVaryingComponents = */ 60,
-    /* .MaxVertexOutputComponents = */ 64,
-    /* .MaxGeometryInputComponents = */ 64,
-    /* .MaxGeometryOutputComponents = */ 128,
-    /* .MaxFragmentInputComponents = */ 0,
-    /* .MaxImageUnits = */ 0,
-    /* .MaxCombinedImageUnitsAndFragmentOutputs = */ 0,
-    /* .MaxCombinedShaderOutputResources = */ 8,
-    /* .MaxImageSamples = */ 0,
-    /* .MaxVertexImageUniforms = */ 0,
-    /* .MaxTessControlImageUniforms = */ 0,
-    /* .MaxTessEvaluationImageUniforms = */ 0,
-    /* .MaxGeometryImageUniforms = */ 0,
-    /* .MaxFragmentImageUniforms = */ 0,
-    /* .MaxCombinedImageUniforms = */ 0,
-    /* .MaxGeometryTextureImageUnits = */ 0,
-    /* .MaxGeometryOutputVertices = */ 256,
-    /* .MaxGeometryTotalOutputComponents = */ 1024,
-    /* .MaxGeometryUniformComponents = */ 1024,
-    /* .MaxGeometryVaryingComponents = */ 64,
-    /* .MaxTessControlInputComponents = */ 128,
-    /* .MaxTessControlOutputComponents = */ 128,
-    /* .MaxTessControlTextureImageUnits = */ 0,
-    /* .MaxTessControlUniformComponents = */ 1024,
-    /* .MaxTessControlTotalOutputComponents = */ 4096,
-    /* .MaxTessEvaluationInputComponents = */ 128,
-    /* .MaxTessEvaluationOutputComponents = */ 128,
-    /* .MaxTessEvaluationTextureImageUnits = */ 16,
-    /* .MaxTessEvaluationUniformComponents = */ 1024,
-    /* .MaxTessPatchComponents = */ 120,
-    /* .MaxPatchVertices = */ 32,
-    /* .MaxTessGenLevel = */ 64,
-    /* .MaxViewports = */ 16,
-    /* .MaxVertexAtomicCounters = */ 0,
-    /* .MaxTessControlAtomicCounters = */ 0,
-    /* .MaxTessEvaluationAtomicCounters = */ 0,
-    /* .MaxGeometryAtomicCounters = */ 0,
-    /* .MaxFragmentAtomicCounters = */ 0,
-    /* .MaxCombinedAtomicCounters = */ 8,
-    /* .MaxAtomicCounterBindings = */ 1,
-    /* .MaxVertexAtomicCounterBuffers = */ 0,
-    /* .MaxTessControlAtomicCounterBuffers = */ 0,
-    /* .MaxTessEvaluationAtomicCounterBuffers = */ 0,
-    /* .MaxGeometryAtomicCounterBuffers = */ 0,
-    /* .MaxFragmentAtomicCounterBuffers = */ 0,
-    /* .MaxCombinedAtomicCounterBuffers = */ 1,
-    /* .MaxAtomicCounterBufferSize = */ 16384,
-    /* .MaxTransformFeedbackBuffers = */ 4,
-    /* .MaxTransformFeedbackInterleavedComponents = */ 64,
-    /* .MaxCullDistances = */ 8,
-    /* .MaxCombinedClipAndCullDistances = */ 8,
-    /* .MaxSamples = */ 4,
-    /* .maxMeshOutputVerticesNV = */ 256,
-    /* .maxMeshOutputPrimitivesNV = */ 512,
-    /* .maxMeshWorkGroupSizeX_NV = */ 32,
-    /* .maxMeshWorkGroupSizeY_NV = */ 1,
-    /* .maxMeshWorkGroupSizeZ_NV = */ 1,
-    /* .maxTaskWorkGroupSizeX_NV = */ 32,
-    /* .maxTaskWorkGroupSizeY_NV = */ 1,
-    /* .maxTaskWorkGroupSizeZ_NV = */ 1,
-    /* .maxMeshViewCountNV = */ 4,
-    /* .maxDualSourceDrawBuffersEXT = */ 1,
-
-    /* .limits = */
-    {
-      /* .nonInductiveForLoops = */ 1,
-      /* .whileLoops = */ 1,
-      /* .doWhileLoops = */ 1,
-      /* .generalUniformIndexing = */ 1,
-      /* .generalAttributeMatrixVectorIndexing = */ 1,
-      /* .generalVaryingIndexing = */ 1,
-      /* .generalSamplerIndexing = */ 1,
-      /* .generalVariableIndexing = */ 1,
-      /* .generalConstantMatrixVectorIndexing = */ 1,
-    }
-};
-
-}
-#endif // DKOMPUTE_DISABLE_SHADER_UTILS

src/include/kompute/Shader.hpp

@@ -1,75 +0,0 @@
-#pragma once
-
-#if !defined(KOMPUTE_DISABLE_SHADER_UTILS) || !KOMPUTE_DISABLE_SHADER_UTILS
-#include <iostream>
-#include <vector>
-
-#ifdef USE_EXTERNAL_GLSLANG
-#include <SPIRV/GlslangToSpv.h>
-#else 
-#include <glslang/SPIRV/GlslangToSpv.h>
-#endif
-
-#include <glslang/Include/ResourceLimits.h>
-#include <glslang/Public/ShaderLang.h>
-
-#include "kompute/Core.hpp"
-
-namespace kp {
-
-/**
-    Shader utily class with functions to compile and process glsl files.
-*/
-class Shader
-{
-  public:
-    // The default resource limit for the GLSL compiler, can be overwritten
-    // Has been adopted by:
-    // https://github.com/KhronosGroup/glslang/blob/master/StandAlone/ResourceLimits.cpp
-    const static TBuiltInResource defaultResource;
-
-    /**
-     * Compile multiple sources with optional filenames. Currently this function
-     * uses the glslang C++ interface which is not thread safe so this funciton
-     * should not be called from multiple threads concurrently. If you have a
-     * online shader processing multithreading use-case that can't use offline
-     * compilation please open an issue.
-     *
-     * @param sources A list of raw glsl shaders in string format
-     * @param files A list of file names respective to each of the sources
-     * @param entryPoint The function name to use as entry point
-     * @param definitions List of pairs containing key value definitions
-     * @param resourcesLimit A list that contains the resource limits for the
-     * GLSL compiler
-     * @return The compiled SPIR-V binary in unsigned int32 format
-     */
-    static std::vector<uint32_t> compileSources(
-      const std::vector<std::string>& sources,
-      const std::vector<std::string>& files = {},
-      const std::string& entryPoint = "main",
-      std::vector<std::pair<std::string, std::string>> definitions = {},
-      const TBuiltInResource& resources = Shader::defaultResource);
-
-    /**
-     * Compile a single glslang source from string value. Currently this
-     * function uses the glslang C++ interface which is not thread safe so this
-     * funciton should not be called from multiple threads concurrently. If you
-     * have a online shader processing multithreading use-case that can't use
-     * offline compilation please open an issue.
-     *
-     * @param source An individual raw glsl shader in string format
-     * @param entryPoint The function name to use as entry point
-     * @param definitions List of pairs containing key value definitions
-     * @param resourcesLimit A list that contains the resource limits for the
-     * GLSL compiler
-     * @return The compiled SPIR-V binary in unsigned int32 format
-     */
-    static std::vector<uint32_t> compileSource(
-      const std::string& source,
-      const std::string& entryPoint = "main",
-      std::vector<std::pair<std::string, std::string>> definitions = {},
-      const TBuiltInResource& resources = Shader::defaultResource);
-};
-
-}
-#endif // DKOMPUTE_DISABLE_SHADER_UTILS

test/CMakeLists.txt

@@ -20,6 +20,7 @@ target_include_directories(
     test_kompute PUBLIC
     $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/single_include>
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/compiled_shaders_include>
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/utils>
 )
 
 if(KOMPUTE_OPT_REPO_SUBMODULE_BUILD)
@@ -27,17 +28,18 @@ if(KOMPUTE_OPT_REPO_SUBMODULE_BUILD)
         test_kompute PRIVATE
         ${gtest_SOURCE_DIR}/include)
 
-    target_link_libraries(test_kompute PRIVATE
+    target_link_libraries(test_kompute
         gtest_main)
 else()
-    target_link_libraries(test_kompute PRIVATE 
+    target_link_libraries(test_kompute
         GTest::gtest)
 endif()
 
-target_link_libraries(test_kompute PRIVATE kompute)
+target_link_libraries(test_kompute kompute)
 
 add_test(NAME test_kompute COMMAND test_kompute)
 
+
 #####################################################
 #################### CODECOV  #######################
 #####################################################

test/TestAsyncOperations.cpp

@@ -5,6 +5,8 @@
 
 #include "kompute/Kompute.hpp"
 
+#include "kompute_test/Shader.hpp"
+
 TEST(TestAsyncOperations, TestManagerParallelExecution)
 {
     // This test is built for NVIDIA 1650. It assumes:
@@ -37,7 +39,7 @@ TEST(TestAsyncOperations, TestManagerParallelExecution)
         }
     )");
 
-    std::vector<uint32_t> spirv = kp::Shader::compileSource(shader);
+    std::vector<uint32_t> spirv = compileSource(shader);
 
     std::vector<float> data(size, 0.0);
     std::vector<float> resultSync(size, 100000000);
@@ -145,7 +147,7 @@ TEST(TestAsyncOperations, TestManagerAsyncExecution)
         }
     )");
 
-    std::vector<uint32_t> spirv = kp::Shader::compileSource(shader);
+    std::vector<uint32_t> spirv = compileSource(shader);
 
     std::vector<float> data(size, 0.0);
     std::vector<float> resultAsync(size, 100000000);

test/TestDestroy.cpp

@@ -3,6 +3,8 @@
 
 #include "kompute/Kompute.hpp"
 
+#include "kompute_test/Shader.hpp"
+
 TEST(TestDestroy, TestDestroyTensorSingle)
 {
     std::shared_ptr<kp::TensorT<float>> tensorA = nullptr;
@@ -16,7 +18,7 @@ TEST(TestDestroy, TestDestroyTensorSingle)
           pa[index] = pa[index] + 1;
       })");
 
-    std::vector<uint32_t> spirv = kp::Shader::compileSource(shader);
+    std::vector<uint32_t> spirv = compileSource(shader);
 
     {
         std::shared_ptr<kp::Sequence> sq = nullptr;
@@ -58,7 +60,7 @@ TEST(TestDestroy, TestDestroyTensorVector)
           pa[index] = pa[index] + 1;
           pb[index] = pb[index] + 2;
       })");
-    std::vector<uint32_t> spirv = kp::Shader::compileSource(shader);
+    std::vector<uint32_t> spirv = compileSource(shader);
 
     {
         std::shared_ptr<kp::Sequence> sq = nullptr;
@@ -103,7 +105,7 @@ TEST(TestDestroy, TestDestroySequenceSingle)
           pa[index] = pa[index] + 1;
       })");
 
-    std::vector<uint32_t> spirv = kp::Shader::compileSource(shader);
+    std::vector<uint32_t> spirv = compileSource(shader);
 
     {
         std::shared_ptr<kp::Sequence> sq = nullptr;

test/TestMultipleAlgoExecutions.cpp

@@ -3,6 +3,8 @@
 
 #include "kompute/Kompute.hpp"
 
+#include "kompute_test/Shader.hpp"
+
 TEST(TestMultipleAlgoExecutions, TestEndToEndFunctionality)
 {
 
@@ -51,7 +53,7 @@ TEST(TestMultipleAlgoExecutions, TestEndToEndFunctionality)
     kp::Constants pushConstsB({ 3.0 });
 
     auto algorithm = mgr.algorithm(params,
-                                   kp::Shader::compileSource(shader),
+                                   compileSource(shader),
                                    workgroup,
                                    specConsts,
                                    pushConstsA);
@@ -89,7 +91,7 @@ TEST(TestMultipleAlgoExecutions, SingleSequenceRecord)
           pa[index] = pa[index] + 1;
       })");
 
-    std::vector<uint32_t> spirv = kp::Shader::compileSource(shader);
+    std::vector<uint32_t> spirv = compileSource(shader);
 
     {
         // A sharedMemoryBarrier is required as the shader is not thread-safe:w
@@ -130,7 +132,7 @@ TEST(TestMultipleAlgoExecutions, MultipleCmdBufRecords)
           pa[index] = pa[index] + 1;
       })");
 
-    std::vector<uint32_t> spirv = kp::Shader::compileSource(shader);
+    std::vector<uint32_t> spirv = compileSource(shader);
 
     std::shared_ptr<kp::Algorithm> algorithm =
       mgr.algorithm({ tensorA }, spirv);
@@ -166,7 +168,7 @@ TEST(TestMultipleAlgoExecutions, MultipleSequences)
           pa[index] = pa[index] + 1;
       })");
 
-    std::vector<uint32_t> spirv = kp::Shader::compileSource(shader);
+    std::vector<uint32_t> spirv = compileSource(shader);
 
     std::shared_ptr<kp::Algorithm> algorithm =
       mgr.algorithm({ tensorA }, spirv);
@@ -201,7 +203,7 @@ TEST(TestMultipleAlgoExecutions, SingleRecordMultipleEval)
           pa[index] = pa[index] + 1;
       })");
 
-    std::vector<uint32_t> spirv = kp::Shader::compileSource(shader);
+    std::vector<uint32_t> spirv = compileSource(shader);
 
     std::shared_ptr<kp::Algorithm> algorithm =
       mgr.algorithm({ tensorA }, spirv);

test/TestOpShadersFromStringAndFile.cpp

@@ -5,6 +5,8 @@
 
 #include "kompute_test/shaders/shadertest_op_custom_shader.hpp"
 
+#include "kompute_test/Shader.hpp"
+
 TEST(TestOpAlgoCreate, ShaderRawDataFromConstructor)
 {
     kp::Manager mgr;
@@ -27,7 +29,7 @@ TEST(TestOpAlgoCreate, ShaderRawDataFromConstructor)
         }
     )");
 
-    std::vector<uint32_t> spirv = kp::Shader::compileSource(shader);
+    std::vector<uint32_t> spirv = compileSource(shader);
 
     std::vector<std::shared_ptr<kp::Tensor>> params = { tensorA, tensorB };
 

test/TestOpTensorCopy.cpp

@@ -3,6 +3,8 @@
 
 #include "kompute/Kompute.hpp"
 
+#include "kompute_test/Shader.hpp"
+
 TEST(TestOpTensorCopy, CopyDeviceToDeviceTensor)
 {
 

test/TestPushConstant.cpp

@@ -2,6 +2,8 @@
 
 #include "kompute/Kompute.hpp"
 
+#include "kompute_test/Shader.hpp"
+
 #include "fmt/ranges.h"
 
 TEST(TestPushConstants, TestConstantsAlgoDispatchOverride)
@@ -22,7 +24,7 @@ TEST(TestPushConstants, TestConstantsAlgoDispatchOverride)
               pa[2] += pcs.z;
           })");
 
-        std::vector<uint32_t> spirv = kp::Shader::compileSource(shader);
+        std::vector<uint32_t> spirv = compileSource(shader);
 
         std::shared_ptr<kp::Sequence> sq = nullptr;
 
@@ -67,7 +69,7 @@ TEST(TestPushConstants, TestConstantsAlgoDispatchNoOverride)
               pa[2] += pcs.z;
           })");
 
-        std::vector<uint32_t> spirv = kp::Shader::compileSource(shader);
+        std::vector<uint32_t> spirv = compileSource(shader);
 
         std::shared_ptr<kp::Sequence> sq = nullptr;
 
@@ -112,7 +114,7 @@ TEST(TestPushConstants, TestConstantsWrongSize)
               pa[2] += pcs.z;
           })");
 
-        std::vector<uint32_t> spirv = kp::Shader::compileSource(shader);
+        std::vector<uint32_t> spirv = compileSource(shader);
 
         std::shared_ptr<kp::Sequence> sq = nullptr;
 

test/TestSequence.cpp

@@ -3,6 +3,8 @@
 
 #include "kompute/Kompute.hpp"
 
+#include "kompute_test/Shader.hpp"
+
 TEST(TestSequence, SequenceDestructorViaManager)
 {
     std::shared_ptr<kp::Sequence> sq = nullptr;
@@ -66,7 +68,7 @@ TEST(TestSequence, RerecordSequence)
 
     sq->eval<kp::OpTensorSyncDevice>({ tensorA, tensorB, tensorOut });
 
-    std::vector<uint32_t> spirv = kp::Shader::compileSource(R"(
+    std::vector<uint32_t> spirv = compileSource(R"(
         #version 450
 
         layout (local_size_x = 1) in;
@@ -116,7 +118,7 @@ TEST(TestSequence, SequenceTimestamps)
           pa[index] = pa[index] + 1;
       })");
 
-    std::vector<uint32_t> spirv = kp::Shader::compileSource(shader);
+    std::vector<uint32_t> spirv = compileSource(shader);
 
     auto seq = mgr.sequence(0, 100); // 100 timestamps
     seq->record<kp::OpTensorSyncDevice>({ tensorA })

test/TestShaderResources.cpp

@@ -1,66 +0,0 @@
-#include "gtest/gtest.h"
-
-#include "kompute/Kompute.hpp"
-
-static const std::string shaderString = (R"(
-    #version 450
-
-    layout (local_size_x = 1) in;
-
-    // The input tensors bind index is relative to index in parameter passed
-    layout(set = 0, binding = 0) buffer bina { float tina[]; };
-    layout(set = 0, binding = 1) buffer binb { float tinb[]; };
-    layout(set = 0, binding = 2) buffer bout { float tout[]; };
-
-    void main() {
-        uint index = gl_GlobalInvocationID.x;
-        
-        int i = 1;
-        
-        while(i < 200) {
-            tout[index] += tina[index] * tinb[index];
-            i++;
-        }
-    }
-)");
-
-void
-compileShaderWithGivenResources(const std::string shaderString,
-                                const TBuiltInResource resources)
-{
-    kp::Shader::compileSource(
-      shaderString,
-      std::string("main"),
-      std::vector<std::pair<std::string, std::string>>({}),
-      resources);
-}
-
-TEST(TestShaderResources, TestNoMaxLight)
-{
-    TBuiltInResource noMaxLightResources = kp::Shader::defaultResource;
-    noMaxLightResources.maxLights = 0;
-
-    EXPECT_NO_THROW(
-      compileShaderWithGivenResources(shaderString, noMaxLightResources));
-}
-
-TEST(TestShaderResources, TestSmallComputeWorkGroupSizeX)
-{
-    TBuiltInResource smallComputeWorkGroupSizeXResources =
-      kp::Shader::defaultResource;
-    smallComputeWorkGroupSizeXResources.maxComputeWorkGroupSizeX = 0;
-
-    ASSERT_THROW(compileShaderWithGivenResources(
-                   shaderString, smallComputeWorkGroupSizeXResources),
-                 std::runtime_error);
-}
-
-TEST(TestShaderResources, TestNoWhileLoopLimit)
-{
-    TBuiltInResource noWhileLoopLimitResources = kp::Shader::defaultResource;
-    noWhileLoopLimitResources.limits.whileLoops = 0;
-
-    ASSERT_THROW(
-      compileShaderWithGivenResources(shaderString, noWhileLoopLimitResources),
-      std::runtime_error);
-}

test/TestSpecializationConstant.cpp

@@ -2,6 +2,8 @@
 
 #include "kompute/Kompute.hpp"
 
+#include "kompute_test/Shader.hpp"
+
 TEST(TestSpecializationConstants, TestTwoConstants)
 {
     {
@@ -18,7 +20,7 @@ TEST(TestSpecializationConstants, TestTwoConstants)
               pb[index] = cTwo;
           })");
 
-        std::vector<uint32_t> spirv = kp::Shader::compileSource(shader);
+        std::vector<uint32_t> spirv = compileSource(shader);
 
         std::shared_ptr<kp::Sequence> sq = nullptr;
 

test/utils/kompute_test/Shader.hpp

@@ -0,0 +1,27 @@
+#pragma once
+
+#include <iostream>
+#include <vector>
+#include <fstream>
+
+/**
+ * Compile a single glslang source from string value. This is only meant
+ * to be used for testing as it's non threadsafe, and it had to be removed
+ * from the glslang dependency and now can only run the CLI directly due to 
+ * license issues: see https://github.com/EthicalML/vulkan-kompute/pull/235
+ *
+ * @param source An individual raw glsl shader in string format
+ * @return The compiled SPIR-V binary in unsigned int32 format
+ */
+static
+std::vector<uint32_t>
+compileSource(
+  const std::string& source)
+{
+    if (system(std::string("glslangValidator --stdin -S comp -V -o tmp_kp_shader.comp.spv << END\n" + source + "\nEND").c_str()))
+        throw std::runtime_error("Error running glslangValidator command");
+    std::ifstream fileStream("tmp_kp_shader.comp.spv", std::ios::binary);
+    std::vector<char> buffer;
+    buffer.insert(buffer.begin(), std::istreambuf_iterator<char>(fileStream), {});
+    return {(uint32_t*)buffer.data(), (uint32_t*)(buffer.data() + buffer.size())};
+}