UPdated examples and documentations to not require glsl deps

README.md

@@ -74,7 +74,8 @@ void kompute(const std::string& shader) {
     kp::Constants pushConstsB({ 3.0 });
 
     auto algorithm = mgr.algorithm(params,
-                                   kp_test_utils::Shader::compileSource(shader),
+                                   // See documentation shader section for compileSource
+                                   compileSource(shader),
                                    workgroup,
                                    specConsts,
                                    pushConstsA);
@@ -165,6 +166,7 @@ def kompute(shader):
     push_consts_a = [2]
     push_consts_b = [3]
 
+    # See documentation shader section for compile_source
     spirv = kp.Shader.compile_source(shader)
 
     algo = mgr.algorithm(params, spirv, workgroup, spec_consts, push_consts_a)
@@ -372,7 +374,7 @@ You can also access the <a href="https://github.com/EthicalML/vulkan-kompute/tre
 
 ### Simple examples
 
-* [Pass shader as raw string](https://kompute.cc/overview/advanced-examples.html#simple-shader-example)
+* [Simple multiplication example](https://kompute.cc/overview/advanced-examples.html#simple-shader-example)
 * [Record batch commands with a Kompute Sequence](https://kompute.cc/overview/advanced-examples.html#record-batch-commands)
 * [Run Asynchronous Operations](https://kompute.cc/overview/advanced-examples.html#asynchronous-operations)
 * [Run Parallel Operations Across Multiple GPU Queues](https://kompute.cc/overview/advanced-examples.html#parallel-operations)

docs/overview/advanced-examples.rst

@@ -55,7 +55,8 @@ The example below shows how you can enable the "VK_EXT_shader_atomic_float" exte
                  atomicAdd(pa[2], pcs.z);
              })");
 
-       std::vector<uint32_t> spirv = kp_test_utils::Shader::compileSource(shader);
+       // See shader documentation section for compileSource
+       std::vector<uint32_t> spirv = compileSource(shader);
 
        std::shared_ptr<kp::Sequence> sq = nullptr;
 
@@ -102,7 +103,8 @@ We also provide tools that allow you to `convert shaders into C++ headers <https
                 throw std::runtime_error("Kompute OpMult expected 3 tensors but got " + tensors.size());
             }
 
-            std::vector<uint32_t> spirv = kp_test_utils::Shader::compileSource(R"(
+           // See shader documentation section for compileSource
+            std::vector<uint32_t> spirv = compileSource(R"(
                 #version 450
 
                 layout(set = 0, binding = 0) buffer tensorLhs {
@@ -215,7 +217,8 @@ In this case we create a shader that should take a couple of milliseconds to run
         }
     )");
 
-    auto algo = mgr.algorithm({tensor}, kp_test_utils::Shader::compileSource(shader));
+    // See shader documentation section for compileSource
+    auto algo = mgr.algorithm({tensor}, compileSource(shader));
 
 Now we are able to run the await function on the default sequence. 
 
@@ -361,7 +364,8 @@ Similar to the asyncrhonous usecase above, we can still run synchronous commands
            }
        )");
 
-       std::vector<uint32_t> spirv = kp_test_utils::Shader::compileSource(shader);
+       // See shader documentation section for compileSource
+       std::vector<uint32_t> spirv = compileSource(shader);
 
        std::shared_ptr<kp::Algorithm> algo = mgr.algorithm({tensorA, tenssorB}, spirv);
 

docs/overview/build-system.rst

@@ -69,8 +69,6 @@ Compile Flags
      - Enable debug build including debug flags (enabled by cmake debug build)
    * - -DKOMPUTE_DISABLE_VK_DEBUG_LAYERS
      - Disable the debug Vulkan SDK Layers, mainly used for android builds
-   * - -DKOMPUTE_DISABLE_SHADER_UTILS
-     - Disable the shader utils and skip adding glslang as dependency
 
 Other CMake Flags
 ~~~~~~~~~~~~~~~~~

docs/overview/ci-tests.rst

@@ -38,10 +38,7 @@ Running on the CPU
 
 We use `Swiftshader <https://github.com/google/swiftshader>`_ to enable us to run the Kompute framework directly on the CPU for the CI tests.
 
-Even though Swiftshader is optimized to function as a high-performance CPU backend for the Vulkan SDK, there are several limitations, the most notable in context of Kompute are:
-
-* Loading files (spirv or text) leads to segfault
-* Loading raw text string shaders leads to segfault
+Even though Swiftshader is optimized to function as a high-performance CPU backend for the Vulkan SDK, there are several limitations, the most notable are limitations in extensions.
 
 This is one of the main reason why only a subset of the tests are run in the CI.
 

docs/overview/reference.rst

@@ -119,13 +119,4 @@ The :class:`kp::OpMemoryBarrier` is a tensor only operation which adds memory ba
 .. doxygenclass:: kp::OpTensorSyncDevice
    :members:
 
-Shader
---------
-
-The :class:`kp_test_utils::Shader` class contains a set of utilities to compile and process shaders.
-
-.. doxygenclass:: kp_test_utils::Shader
-   :members:
-
-
 

docs/overview/shaders-to-headers.rst

@@ -3,6 +3,29 @@
 Processing Shaders with Kompute
 =====================
 
+Demo / testing function to compile shaders
+----------------------------------
+
+GLSLANG was initially integrated as part of the framework but it now has been removed due to the license of the glslang pre-processor being under a custom NVIDIA license which explicitly excludes grant of any licenses to NVIDIA's patents in the preprocessor.
+
+For users that are looking to quickly test the processors it is possible to use the function that is provided in the examples which provides a (non-thread-safe / non-robust) implementation that compiles a shader string into spirv bytes. It is not recommended to use in production but it does enable for faster iteration cycles during development.
+
+.. code-block:: cpp
+    :linenos:
+    static std::vector<uint32_t>
+    compileSource(
+      const std::string& source)
+    {
+        if (system(std::string("glslangValidator --stdin -S comp -V -o tmp_kp_shader.comp.spv << END
+    " + source + "
+    END").c_str()))
+            throw std::runtime_error("Error running glslangValidator command");
+        std::ifstream fileStream("tmp_kp_shader.comp.spv", std::ios::binary);
+        std::vector<char> buffer;
+        buffer.insert(buffer.begin(), std::istreambuf_iterator<char>(fileStream), {});
+        return {(uint32_t*)buffer.data(), (uint32_t*)(buffer.data() + buffer.size())};
+    }
+
 Converting Shaders into C / C++ Header Files
 ----------------------------------
 

examples/array_multiplication/src/Main.cpp

@@ -5,6 +5,18 @@
 
 #include "kompute/Kompute.hpp"
 
+static std::vector<uint32_t>
+compileSource(
+  const std::string& source)
+{
+    if (system(std::string("glslangValidator --stdin -S comp -V -o tmp_kp_shader.comp.spv << END\n" + source + "\nEND").c_str()))
+        throw std::runtime_error("Error running glslangValidator command");
+    std::ifstream fileStream("tmp_kp_shader.comp.spv", std::ios::binary);
+    std::vector<char> buffer;
+    buffer.insert(buffer.begin(), std::istreambuf_iterator<char>(fileStream), {});
+    return {(uint32_t*)buffer.data(), (uint32_t*)(buffer.data() + buffer.size())};
+}
+
 int main()
 {
 #if KOMPUTE_ENABLE_SPDLOG
@@ -39,7 +51,7 @@ int main()
 
     std::vector<std::shared_ptr<kp::Tensor>> params = { tensorInA, tensorInB, tensorOut };
 
-    std::shared_ptr<kp::Algorithm> algo = mgr.algorithm(params, kp_test_utils::Shader::compileSource(shader));
+    std::shared_ptr<kp::Algorithm> algo = mgr.algorithm(params, kp_test_utils::compileSource(shader));
 
     mgr.sequence()
         ->record<kp::OpTensorSyncDevice>(params)

examples/godot_examples/custom_module/kompute_summator/KomputeSummatorNode.cpp

@@ -4,6 +4,18 @@
 
 #include "KomputeSummatorNode.h"
 
+static std::vector<uint32_t>
+compileSource(
+  const std::string& source)
+{
+    if (system(std::string("glslangValidator --stdin -S comp -V -o tmp_kp_shader.comp.spv << END\n" + source + "\nEND").c_str()))
+        throw std::runtime_error("Error running glslangValidator command");
+    std::ifstream fileStream("tmp_kp_shader.comp.spv", std::ios::binary);
+    std::vector<char> buffer;
+    buffer.insert(buffer.begin(), std::istreambuf_iterator<char>(fileStream), {});
+    return {(uint32_t*)buffer.data(), (uint32_t*)(buffer.data() + buffer.size())};
+}
+
 KomputeSummatorNode::KomputeSummatorNode() {
     this->_init();
 }
@@ -52,9 +64,9 @@ void KomputeSummatorNode::_init() {
         )");
 
         std::shared_ptr<kp::Algorithm> algo =
-          mgr.algorithm(
+          this->mManager.algorithm(
                 { this->mPrimaryTensor, this->mSecondaryTensor },
-                kp_test_utils::Shader::compileSource(shader));
+                compileSource(shader));
 
 
         // First we ensure secondary tensor loads to GPU
@@ -63,7 +75,7 @@ void KomputeSummatorNode::_init() {
                 { this->mSecondaryTensor });
 
         // Then we run the operation with both tensors
-        sq->record<kp::OpAlgoDispatch>(algo)
+        sq->record<kp::OpAlgoDispatch>(algo);
 
         // We map the result back to local 
         sq->record<kp::OpTensorSyncLocal>(

examples/godot_examples/gdnative_shared/src/KomputeSummator.cpp

@@ -5,6 +5,18 @@
 
 #include "KomputeSummator.hpp"
 
+static std::vector<uint32_t>
+compileSource(
+  const std::string& source)
+{
+    if (system(std::string("glslangValidator --stdin -S comp -V -o tmp_kp_shader.comp.spv << END\n" + source + "\nEND").c_str()))
+        throw std::runtime_error("Error running glslangValidator command");
+    std::ifstream fileStream("tmp_kp_shader.comp.spv", std::ios::binary);
+    std::vector<char> buffer;
+    buffer.insert(buffer.begin(), std::istreambuf_iterator<char>(fileStream), {});
+    return {(uint32_t*)buffer.data(), (uint32_t*)(buffer.data() + buffer.size())};
+}
+
 namespace godot {
 
 KomputeSummator::KomputeSummator() {
@@ -58,7 +70,7 @@ void KomputeSummator::_init() {
         // Then we run the operation with both tensors
         this->mSequence->record<kp::OpAlgoCreate>(
             { this->mPrimaryTensor, this->mSecondaryTensor }, 
-            kp_test_utils::Shader::compileSource(shader));
+            compileSource(shader));
 
         // We map the result back to local 
         this->mSequence->record<kp::OpTensorSyncLocal>(

examples/neural_network_vgg7/sh_conv.py

@@ -1,8 +1,15 @@
+import os
+
 import kp
 
+def compile_source(source):
+    os.system("glslangValidator --stdin -S comp -V -o tmp_kp_shader.comp.spv << END\n" + source + "\nEND")
+    return open("tmp_kp_shader.comp.spv", "rb").read()
+
+
 # This is the convolution & leakyrelu shader.
 global conv_shader
-conv_shader = kp.Shader.compile_source("""
+conv_shader = compile_source("""
 #version 450
 
 layout (local_size_x = 8, local_size_y = 2) in;