Merge pull request #154 from EthicalML/glslang_implementation

Glslang implementation for online shader compilation

.ccls

@@ -17,6 +17,7 @@
 -I./python/pybind11/include/
 -I./external/Vulkan-Headers/include/
 -I./external/googletest/googletest/include/
+-I./external/glslang/
 -I./external/spdlog/include/
 -I./src/include/
 -I./single_include/

.github/workflows/cpp_tests.yml

@@ -19,16 +19,16 @@ jobs:
     - name: configure-cpp
       run: |
         cmake -Bbuild/ \
+                  -DCMAKE_BUILD_TYPE=Debug                   \
                   -DKOMPUTE_OPT_INSTALL=0                    \
                   -DKOMPUTE_OPT_REPO_SUBMODULE_BUILD=1       \
                   -DKOMPUTE_OPT_BUILD_TESTS=1                \
-                  -DKOMPUTE_OPT_ENABLE_SPDLOG=1              \
+                  -DKOMPUTE_OPT_ENABLE_SPDLOG=1
-                  -DSPDLOG_INSTALL=1
     - name: build-cpp
       run: |
         make mk_build_tests
     - name: test-cpp
       run: |
         export VK_ICD_FILENAMES=/swiftshader/vk_swiftshader_icd.json
-        make mk_run_tests_cpu_only
+        make mk_run_tests
 

.gitmodules

@@ -14,3 +14,7 @@
 	path = python/pybind11
 	url = https://github.com/pybind/pybind11
     branch = v2.6.1
+[submodule "external/glslang"]
+	path = external/glslang
+	url = https://github.com/KhronosGroup/glslang/
+    branch = 11.1.0

CMakeLists.txt

@@ -19,12 +19,16 @@ option(KOMPUTE_OPT_ENABLE_SPDLOG "Extra compile flags for Kompute, see docs for
 option(KOMPUTE_OPT_REPO_SUBMODULE_BUILD, "Use the submodule repos instead of external package manager" 0)
 option(KOMPUTE_OPT_ANDOID_BUILD "Enable android compilation flags required" 0)
 option(KOMPUTE_OPT_DISABLE_VK_DEBUG_LAYERS "Explicitly disable debug layers even on debug" 0)
+option(KOMPUTE_OPT_DISABLE_SHADER_UTILS "Remove shader util code and dependencies including glslang" 0)
 # Build flags
 set(KOMPUTE_EXTRA_CXX_FLAGS "" CACHE STRING "Extra compile flags for Kompute, see docs for full list")
 
 if(KOMPUTE_OPT_ENABLE_SPDLOG)
     set(KOMPUTE_EXTRA_CXX_FLAGS "${KOMPUTE_EXTRA_CXX_FLAGS} -DKOMPUTE_ENABLE_SPDLOG=1")
-    set(SPDLOG_INSTALL, 1)
+    if(KOMPUTE_OPT_INSTALL)
+        # Enable install parameters for spdlog (overrides parameters passed)
+        set(SPDLOG_INSTALL ON CACHE BOOL "Enables install of glslang" FORCE)
+    endif()
 endif()
 
 if(KOMPUTE_OPT_ANDOID_BUILD)
@@ -39,6 +43,17 @@ if(KOMPUTE_OPT_DISABLE_VK_DEBUG_LAYERS)
     set(KOMPUTE_EXTRA_CXX_FLAGS "${KOMPUTE_EXTRA_CXX_FLAGS} -DKOMPUTE_DISABLE_VK_DEBUG_LAYERS=1")
 endif()
 
+if(NOT KOMPUTE_OPT_DISABLE_SHADER_UTILS)
+    if(KOMPUTE_OPT_INSTALL)
+        # Enable install parameters for glslang (overrides parameters passed)
+        # When install is enabled the glslang libraries become shared
+        set(ENABLE_GLSLANG_INSTALL ON CACHE BOOL "Enables install of glslang" FORCE)
+        set(BUILD_SHARED_LIBS ON CACHE BOOL "Enables build of shared libraries" FORCE)
+    endif()
+else()
+    set(KOMPUTE_EXTRA_CXX_FLAGS "${KOMPUTE_EXTRA_CXX_FLAGS} -DKOMPUTE_DISABLE_SHADER_UTILS=1")
+endif()
+
 set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DDEBUG=1 ${KOMPUTE_EXTRA_CXX_FLAGS} -DUSE_DEBUG_EXTENTIONS")
 set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -DRELEASE=1 ${KOMPUTE_EXTRA_CXX_FLAGS}")
 

Makefile

@@ -13,7 +13,7 @@ VCPKG_WIN_PATH ?= "C:\\Users\\axsau\\Programming\\lib\\vcpkg\\scripts\\buildsyst
 VCPKG_UNIX_PATH ?= "/c/Users/axsau/Programming/lib/vcpkg/scripts/buildsystems/vcpkg.cmake"
 
 # Regext to pass to catch2 to filter tests
-FILTER_TESTS ?= "*"
+FILTER_TESTS ?= "-TestAsyncOperations.TestManagerParallelExecution"
 
 ifeq ($(OS),Windows_NT)     # is Windows_NT on XP, 2000, 7, Vista, 10...
 	CMAKE_BIN ?= "C:\Program Files\CMake\bin\cmake.exe"
@@ -68,7 +68,6 @@ mk_cmake:
 		-DKOMPUTE_OPT_BUILD_SHADERS=1 \
 		-DKOMPUTE_OPT_BUILD_SINGLE_HEADER=1 \
 		-DKOMPUTE_OPT_ENABLE_SPDLOG=1 \
-		-DSPDLOG_INSTALL=1 \
 		-DKOMPUTE_OPT_CODE_COVERAGE=1 \
 		-G "Unix Makefiles"
 
@@ -88,7 +87,7 @@ mk_run_docs: mk_build_docs
 	(cd build/docs/sphinx && python2.7 -m SimpleHTTPServer)
 
 mk_run_tests: mk_build_tests
-	./build/test/test_kompute $(FILTER_TESTS)
+	./build/test/test_kompute --gtest_filter=$(FILTER_TESTS)
 
 mk_build_swiftshader_library:
 	git clone https://github.com/google/swiftshader || echo "Assuming already cloned"
@@ -99,16 +98,6 @@ mk_build_swiftshader_library:
 mk_run_tests_cpu: export VK_ICD_FILENAMES=$(PWD)/swiftshader/build/vk_swiftshader_icd.json
 mk_run_tests_cpu: mk_build_swiftshader_library mk_build_tests mk_run_tests_cpu_only
 
-mk_run_tests_cpu_only:
-	./build/test/test_kompute --gtest_filter="TestLogisticRegressionAlgorithm.*"
-	./build/test/test_kompute --gtest_filter="TestManager.*"
-	./build/test/test_kompute --gtest_filter="TestOpAlgoBase.ShaderCompiledDataFromConstructor"
-	./build/test/test_kompute --gtest_filter="TestOpTensorCopy.*"
-	./build/test/test_kompute --gtest_filter="TestOpTensorCreate.*"
-	./build/test/test_kompute --gtest_filter="TestOpTensorSync.*"
-	./build/test/test_kompute --gtest_filter="TestSequence.*"
-	./build/test/test_kompute --gtest_filter="TestTensor.*"
-
 
 ####### Visual studio build shortcut commands #######
 
@@ -132,7 +121,6 @@ vs_cmake:
 		-DKOMPUTE_OPT_BUILD_SHADERS=1 \
 		-DKOMPUTE_OPT_BUILD_SINGLE_HEADER=1 \
 		-DKOMPUTE_OPT_ENABLE_SPDLOG=1 \
-		-DSPDLOG_INSTALL=1 \
 		-G "Visual Studio 16 2019"
 
 vs_build_all:

README.md

@@ -78,7 +78,7 @@ int main() {
     // 3. Run operation with string shader synchronously
     mgr.evalOpDefault<kp::OpAlgoBase>(
         { tensorInA, tensorInB, tensorOut },
-        std::vector<char>(shaderString.begin(), shaderString.end()));
+        std::vector<uint32_t>(shaderString.begin(), shaderString.end()));
 
     // 4. Map results back from GPU memory to print the results
     mgr.evalOpDefault<kp::OpTensorSyncLocal>({ tensorInA, tensorInB, tensorOut });

docs/index.rst

@@ -15,12 +15,12 @@ Documentation Index (as per sidebar)
     :caption: C++ Documentation:
 
     C++ Examples <overview/advanced-examples>
-    C++ Memory Management Principles <overview/memory-management>
+    Memory Management Principles <overview/memory-management>
-    C++ Build System Deep Dive <overview/build-system>
+    Build System Deep Dive <overview/build-system>
-    C++ Converting GLSL/HLSL Shaders to Cpp Headers <overview/shaders-to-headers>
+    Processing Shaders (Online & Offline) <overview/shaders-to-headers>
-    C++ Extending Kompute with Custom Operations <overview/custom-operations>
+    Extending Kompute with Custom Operations <overview/custom-operations>
     C++ Class Documentation & Reference <overview/reference>
-    C++ Code Coverage <https://kompute.cc/codecov/>
+    Test Code Coverage <https://kompute.cc/codecov/>
 
 .. toctree::
     :titlesonly:

docs/overview/async-parallel.rst

@@ -27,7 +27,7 @@ This basically provides further granularity on Vulkan Fences, which is its means
 
 It is important that submitting tasks asynchronously, does not mean that these will be executed in parallel. Parallel execution of operations will be covered in the following section.
 
-Asynchronous operation submission can be achieved through the kp::Manager, or directly through the kp::Sequence. Below is an example using the Kompute manager.
+Asynchronous operation submission can be achieved through the :class:`kp::Manager`, or directly through the :class:`kp::Sequence`. Below is an example using the Kompute manager.
 
 Conceptual Overview
 ^^^^^^^^^^^^^^^^^^^^^

docs/overview/build-system.rst

@@ -65,6 +65,8 @@ Compile Flags
      - Enable debug build including debug flags (enabled by cmake debug build)
    * - -DKOMPUTE_DISABLE_VK_DEBUG_LAYERS
      - Disable the debug Vulkan layers, mainly used for android builds
+   * - -DKOMPUTE_DISABLE_SHADER_UTILS
+     - Disable the shader utils and skip adding glslang as dependency
 
 
 Dependencies

docs/overview/custom-operations.rst

@@ -11,7 +11,7 @@ These nuances are important for more advanced users of Kompute, as this will pro
 Flow of Function Calls
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-The top level operation which all operations inherit from is the `kp::OpBase` class. Some of the "Core Native Operations" like `kp::OpTensorCopy`, `kp::OpTensorCreate`, etc all inherit from the base operation class.
+The top level operation which all operations inherit from is the :class:`kp::OpBase` class. Some of the "Core Native Operations" like :class:`kp::OpTensorCopy`, :class:`kp::OpTensorCreate`, etc all inherit from the base operation class.
 
 The `kp::OpAlgoBase` is another base operation that is specifically built to enable users to create their own operations that contain custom shader logic (i.e. requiring Vulkan Compute Pipelines, DescriptorSets, etc). The next section contains an example which shows how to extend the OpAlgoBase class.
 

docs/overview/memory-management.rst

@@ -21,7 +21,7 @@ The memory ownership is hierarchically outlined in the component architecture -
 Optional Memory Management
 -------------
 
-As outlined above, resource memory is only managed by Kompute if the resources are created by Kompute. Each of the Kompute components can also be initialised with externally managed resources. The kp::Manager for example can be initialized with an external Vulkan Device. The first principle ensures that all memory ownership is explicitly defined when managing and creating Kompute resources.
+As outlined above, resource memory is only managed by Kompute if the resources are created by Kompute. Each of the Kompute components can also be initialised with externally managed resources. The :class:`kp::Manager` for example can be initialized with an external Vulkan Device. The first principle ensures that all memory ownership is explicitly defined when managing and creating Kompute resources.
 
 
 

docs/overview/reference.rst

@@ -12,7 +12,7 @@ Below is a diagram that provides insights on the relationship between Vulkan Kom
 Manager
 -------
 
-The Kompute Manager provides a high level interface to simplify interaction with underlying kp::Sequences of kp::Operations.
+The Kompute Manager provides a high level interface to simplify interaction with underlying :class:`kp::Sequences` of :class:`kp::Operations`.
 
 .. image:: ../images/kompute-vulkan-architecture-manager.jpg
    :width: 100%
@@ -23,7 +23,7 @@ The Kompute Manager provides a high level interface to simplify interaction with
 Sequence
 -------
 
-The Kompute Sequence consists of batches of kp::Operations, which are executed on a respective GPU queue. The execution of sequences can be synchronous or asynchronous, and it can be coordinated through its respective vk::Fence.
+The Kompute Sequence consists of batches of :class:`kp::Operations`, which are executed on a respective GPU queue. The execution of sequences can be synchronous or asynchronous, and it can be coordinated through its respective vk::Fence.
 
 .. image:: ../images/kompute-vulkan-architecture-sequence.jpg
    :width: 100%
@@ -34,7 +34,7 @@ The Kompute Sequence consists of batches of kp::Operations, which are executed o
 Tensor
 -------
 
-The kp::Tensor is the atomic unit in Kompute, and it is used primarily for handling Host and GPU Device data.
+The :class:`kp::Tensor` is the atomic unit in Kompute, and it is used primarily for handling Host and GPU Device data.
 
 .. image:: ../images/kompute-vulkan-architecture-tensor.jpg
    :width: 100%
@@ -45,7 +45,7 @@ The kp::Tensor is the atomic unit in Kompute, and it is used primarily for handl
 Algorithm
 -------
 
-The kp::Algorithm consists primarily of the components required for shader code execution, including the relevant vk::DescriptorSet relatedresources as well as vk::Pipeline and all the relevant Vulkan resources as outlined in the architectural diagram.
+The :class:`kp::Algorithm` consists primarily of the components required for shader code execution, including the relevant vk::DescriptorSet relatedresources as well as vk::Pipeline and all the relevant Vulkan resources as outlined in the architectural diagram.
 
 .. image:: ../images/kompute-vulkan-architecture-algorithm.jpg
    :width: 100%
@@ -56,7 +56,7 @@ The kp::Algorithm consists primarily of the components required for shader code
 OpBase
 -------
 
-The kp::OpBase provides a top level class for an operation in Kompute, which is the step that is executed on a GPU submission. The Kompute operations can consist of one or more kp::Tensor.
+The :class:`kp::OpBase` provides a top level class for an operation in Kompute, which is the step that is executed on a GPU submission. The Kompute operations can consist of one or more :class:`kp::Tensor`.
 
 .. image:: ../images/kompute-vulkan-architecture-operations.jpg
    :width: 100%
@@ -78,7 +78,7 @@ The vk::OpAlgoBase extends the vk::OpBase class, and provides the base for shade
 OpMult
 -------
 
-The kp::OpMult operation is a sample implementation of the kp::OpAlgoBase class. This class shows how it is possible to create a custom vk::OpAlgoBase that can compile as part of the binary. The kp::OpMult operation uses the shader-to-cpp-header-file script to convert the script into cpp header files.
+The :class:`kp::OpMult` operation is a sample implementation of the :class:`kp::OpAlgoBase` class. This class shows how it is possible to create a custom vk::OpAlgoBase that can compile as part of the binary. The :class:`kp::OpMult` operation uses the shader-to-cpp-header-file script to convert the script into cpp header files.
 
 .. image:: ../images/kompute-vulkan-architecture-opmult.jpg
    :width: 100%
@@ -90,7 +90,7 @@ The kp::OpMult operation is a sample implementation of the kp::OpAlgoBase class.
 OpTensorCopy
 -------
 
-The kp::OpTensorCopy is a tensor only operation that copies the GPU memory buffer data from one kp::Tensor to one or more subsequent tensors.
+The :class:`kp::OpTensorCopy` is a tensor only operation that copies the GPU memory buffer data from one :class:`kp::Tensor` to one or more subsequent tensors.
 
 .. doxygenclass:: kp::OpTensorCopy
    :members:
@@ -98,7 +98,7 @@ The kp::OpTensorCopy is a tensor only operation that copies the GPU memory buffe
 OpTensorSyncLocal
 -------
 
-The kp::OpTensorSyncLocal is a tensor only operation that maps the data from the GPU device memory into the local host vector.
+The :class:`kp::OpTensorSyncLocal` is a tensor only operation that maps the data from the GPU device memory into the local host vector.
 
 .. doxygenclass:: kp::OpTensorSyncLocal
    :members:
@@ -106,11 +106,19 @@ The kp::OpTensorSyncLocal is a tensor only operation that maps the data from the
 OpTensorSyncDevice
 -------
 
-The kp::OpTensorSyncDevice is a tensor only operation that maps the data from the local host vector into the GPU device memory.
+The :class:`kp::OpTensorSyncDevice` is a tensor only operation that maps the data from the local host vector into the GPU device memory.
 
 .. doxygenclass:: kp::OpTensorSyncDevice
    :members:
 
 
+Shader
+--------
+
+The :class:`kp::Shader` class contains a set of utilities to compile and process shaders.
+
+.. doxygenclass:: kp::Shader
+   :members:
+
 
 

docs/overview/shaders-to-headers.rst

@@ -1,9 +1,28 @@
 
 
-Converting Shaders to C++ Headers
+Processing Shaders with Kompute
 =====================
 
-Kompute allows for shaders to be loaded directly through the kp::OpAlgoBase as either raw strings (through shaderc) or compiled SPIRV bytes. For this latter, the traditional method of including the SPIRV bytes is by loading the SPIRV file directly and passing the contents.
+Kompute allows for two main ways of interacting with shaders - namely:
+
+* Integration with [glslang](https://github.com/KhronosGroup/glslang) for online/runtime shader compilation
+* A CLI that coverts shaders into C++ header files
+
+Processing Shaders Online via Kompute Shader Utils
+---------------
+
+Kompute provides a set of helper functions that expose the C++ functionality of the glslang Khronos framework to process shader sources online during runtime.
+
+It's worth emphasising that the suggested approach is to process shaders offline, so the section below is suggested to convert shaders to either their respective SPV format, or convert them into C++ sources that would be embedded as part of the resulting binary.
+
+The Shader utility function can be skipped on build time through compiler flags - for more information on this you should read the `build section <build-system.rst>`_.
+
+More details on the shader utils can be found in the :class:`kp::Shader` section of the `C++ reference page <reference.rst>`_.
+
+Converting Shaders into C / C++ Header Files
+----------------------------------
+
+Kompute allows for shaders to be loaded directly through the :class:`kp::OpAlgoBase` as either raw strings (through shaderc) or compiled SPIRV bytes. For this latter, the traditional method of including the SPIRV bytes is by loading the SPIRV file directly and passing the contents.
 
 The Kompute codebase has a utility that allows you to convert shader files into C++ header files containing the SPIRV header data. This is useful as it enables developers to compile the SPIRV shaders into the final binary, which avoids the need for multiple files being required.
 

examples/android/android-simple/app/build.gradle

@@ -18,8 +18,10 @@ android {
                 arguments '-DANDROID_TOOLCHAIN=clang',
                         '-DANDROID_STL=c++_static',
                         '-DKOMPUTE_OPT_ANDOID_BUILD=1',
+                        '-DKOMPUTE_OPT_REPO_SUBMODULE_BUILD=1',
                         '-DKOMPUTE_OPT_INSTALL=0',
-                        '-DKOMPUTE_OPT_BUILD_SINGLE_HEADER=1',
+                        '-DKOMPUTE_OPT_ENABLE_SPDLOG=0',
+                        '-DKOMPUTE_OPT_BUILD_SINGLE_HEADER=0',
                         '-DKOMPUTE_OPT_DISABLE_VK_DEBUG_LAYERS=1',
                         '-DKOMPUTE_EXTRA_CXX_FLAGS=-DKOMPUTE_VK_API_MINOR_VERSION=0'
             }

examples/android/android-simple/app/src/main/cpp/KomputeModelML.cpp

@@ -51,19 +51,9 @@ void KomputeModelML::train(std::vector<float> yData, std::vector<float> xIData,
 
             sq->record<kp::OpTensorSyncDevice>({ wIn, bIn });
 
-#ifdef KOMPUTE_ANDROID_SHADER_FROM_STRING
             // Newer versions of Android are able to use shaderc to read raw string
             sq->record<kp::OpAlgoBase>(
-                    params, std::vector<char>(LR_SHADER.begin(), LR_SHADER.end()));
+                    params, kp::Shader::compile_source(LR_SHADER));
-#else
-            // Older versions of Android require the SPIRV binary directly
-            sq->record<kp::OpAlgoBase>(
-                    params, std::vector<char>(
-                            kp::shader_data::shaders_glsl_logisticregression_comp_spv,
-                            kp::shader_data::shaders_glsl_logisticregression_comp_spv
-                                + kp::shader_data::shaders_glsl_logisticregression_comp_spv_len
-                    ));
-#endif
 
             sq->record<kp::OpTensorSyncLocal>({ wOutI, wOutJ, bOut, lOut });
 

examples/array_multiplication/src/Main.cpp

@@ -18,7 +18,6 @@ int main()
     auto tensorInB = mgr.tensor({ 0.0, 1.0, 2.0 });
     auto tensorOut = mgr.tensor({ 0.0, 0.0, 0.0 });
 
-#ifdef KOMPUTE_ANDROID_SHADER_FROM_STRING
     std::string shader(R"(
         // The version to use 
         #version 450
@@ -40,15 +39,7 @@ int main()
 
     mgr.evalOpDefault<kp::OpAlgoBase>(
             { tensorInA, tensorInB, tensorOut },
-            std::vector<char>(shader.begin(), shader.end()));
+            kp::Shader::compile_source(shader));
-#else
-    mgr.evalOpDefault<kp::OpAlgoBase>(
-            { tensorInA, tensorInB, tensorOut },
-            std::vector<char>(
-            kp::shader_data::shaders_glsl_opmult_comp_spv,
-            kp::shader_data::shaders_glsl_opmult_comp_spv
-                + kp::shader_data::shaders_glsl_opmult_comp_spv_len));
-#endif
 
     mgr.evalOpDefault<kp::OpTensorSyncLocal>({tensorOut});
 

examples/godot_examples/custom_module/kompute_summator/KomputeSummatorNode.cpp

@@ -61,7 +61,7 @@ void KomputeSummatorNode::_init() {
         // Then we run the operation with both tensors
         sq->record<kp::OpAlgoBase>(
             { this->mPrimaryTensor, this->mSecondaryTensor }, 
-            std::vector<char>(shader.begin(), shader.end()));
+            kp::Shader::compile_source(shader));
 
         // We map the result back to local 
         sq->record<kp::OpTensorSyncLocal>(

examples/godot_examples/gdnative_shared/src/KomputeSummator.cpp

@@ -58,7 +58,7 @@ void KomputeSummator::_init() {
         // Then we run the operation with both tensors
         this->mSequence->record<kp::OpAlgoBase>(
             { this->mPrimaryTensor, this->mSecondaryTensor }, 
-            std::vector<char>(shader.begin(), shader.end()));
+            kp::Shader::compile_source(shader));
 
         // We map the result back to local 
         this->mSequence->record<kp::OpTensorSyncLocal>(

examples/logistic_regression/src/Main.cpp

@@ -44,16 +44,11 @@ int main()
 
     sq->record<kp::OpTensorSyncDevice>({ wIn, bIn });
 
-#ifdef KOMPUTE_ANDROID_SHADER_FROM_STRING
     sq->record<kp::OpAlgoBase>(
-      params, "shaders/glsl/logistic_regression.comp");
+        params, std::vector<uint32_t>(
-#else
+                (uint32_t*)kp::shader_data::shaders_glsl_logisticregression_comp_spv,
-    sq->record<kp::OpAlgoBase>(
+                (uint32_t*)(kp::shader_data::shaders_glsl_logisticregression_comp_spv
-        params, std::vector<char>(
+                    + kp::shader_data::shaders_glsl_logisticregression_comp_spv_len)));
-                kp::shader_data::shaders_glsl_logisticregression_comp_spv,
-                kp::shader_data::shaders_glsl_logisticregression_comp_spv
-                    + kp::shader_data::shaders_glsl_logisticregression_comp_spv_len));
-#endif
 
     sq->record<kp::OpTensorSyncLocal>({ wOutI, wOutJ, bOut, lOut });
 

examples/python/README.md

@@ -1,17 +0,0 @@
-# Kompute Python Example
-
-This folder contains the accompanying code for the article "High Performance Python for GPU Accelerated Machine Learning in Cross-Vendor GPUs".
-
-The easiest way to try this example is by using the [Google Binder Notebook](https://colab.research.google.com/drive/15uQ7qMZuOyk8JcXF-3SB2R5yNFW21I4P), which will allow you to use a GPU for free and runs without much setup.
-
-<a href="https://colab.research.google.com/drive/15uQ7qMZuOyk8JcXF-3SB2R5yNFW21I4P">
-<img src="https://raw.githubusercontent.com/EthicalML/vulkan-kompute/python_extensions/docs/images/binder-python.jpg">
-</a>
-
-Alternatively if you want to test the example yourself locally, you can get setup and started through the following links:
-
-1. Install the [Kompute Python Package](https://kompute.cc/overview/python-package.html#package-installation)
-2. Run the [Array Multiplication Code](https://github.com/EthicalML/vulkan-kompute/blob/python_extensions/python/test/test_array_multiplication.py)
-3. Run the [Logistic Regression Code](https://github.com/EthicalML/vulkan-kompute/blob/python_extensions/python/test/test_logistic_regression.py)
-
-

python/src/main.cpp

@@ -30,6 +30,28 @@ PYBIND11_MODULE(kp, m) {
         .value("storage", kp::Tensor::TensorTypes::eStorage, "Tensor with host visible gpu memory.")
         .export_values();
 
+#if !defined(KOMPUTE_DISABLE_SHADER_UTILS) || !KOMPUTE_DISABLE_SHADER_UTILS
+    py::class_<kp::Shader>(m, "Shader", "Shader class")
+        .def_static("compile_source", [](
+                                    const std::string& source,
+                                    const std::string& entryPoint,
+                                    const std::vector<std::pair<std::string,std::string>>& definitions) {
+                std::vector<uint32_t> spirv = kp::Shader::compile_source(source, entryPoint, definitions);
+                return py::bytes((const char*)spirv.data(), spirv.size() * sizeof(uint32_t));
+            },
+            "Compiles string source provided and returns the value in bytes",
+            py::arg("source"), py::arg("entryPoint") = "main", py::arg("definitions") = std::vector<std::pair<std::string,std::string>>() )
+        .def_static("compile_sources", [](
+                                    const std::vector<std::string>& source,
+                                    const std::vector<std::string>& files,
+                                    const std::string& entryPoint,
+                                    const std::vector<std::pair<std::string,std::string>>& definitions) {
+                std::vector<uint32_t> spirv = kp::Shader::compile_sources(source, files, entryPoint, definitions);
+                return py::bytes((const char*)spirv.data(), spirv.size() * sizeof(uint32_t));
+            },
+            "Compiles sources provided with file names and returns the value in bytes",
+            py::arg("sources"), py::arg("files") = std::vector<std::string>(), py::arg("entryPoint") = "main", py::arg("definitions") = std::vector<std::pair<std::string,std::string>>() );
+#endif // KOMPUTE_DISABLE_SHADER_UTILS
 
     py::class_<kp::Tensor, std::shared_ptr<kp::Tensor>>(m, "Tensor", DOC(kp, Tensor))
         .def(py::init(
@@ -127,7 +149,7 @@ PYBIND11_MODULE(kp, m) {
                 const char *data = reinterpret_cast<const char *>(info.ptr);
                 size_t length = static_cast<size_t>(info.size);
                 return self.record<kp::OpAlgoBase>(
-                    tensors, std::vector<char>(data, data + length), workgroup, constants);
+                    tensors, std::vector<uint32_t>((uint32_t*)data, (uint32_t*)(data + length)), workgroup, constants);
             },
             "Records an operation using a custom shader provided as spirv bytes",
             py::arg("tensors"), py::arg("bytes"), py::arg("workgroup") = kp::Workgroup(), py::arg("constants") = kp::Constants() );
@@ -205,7 +227,7 @@ PYBIND11_MODULE(kp, m) {
                 const char *data = reinterpret_cast<const char *>(info.ptr);
                 size_t length = static_cast<size_t>(info.size);
                 self.evalOpDefault<kp::OpAlgoBase>(
-                    tensors, std::vector<char>(data, data + length), workgroup, constants);
+                    tensors, std::vector<uint32_t>((uint32_t*)data, (uint32_t*)(data + length)), workgroup, constants);
             },
             "Evaluates an operation using a custom shader provided as spirv bytes with new anonymous Sequence",
             py::arg("tensors"), py::arg("bytes"), py::arg("workgroup") = kp::Workgroup(), py::arg("constants") = kp::Constants() )
@@ -235,7 +257,7 @@ PYBIND11_MODULE(kp, m) {
                 const char *data = reinterpret_cast<const char *>(info.ptr);
                 size_t length = static_cast<size_t>(info.size);
                 self.evalOp<kp::OpAlgoBase>(
-                    tensors, sequenceName, std::vector<char>(data, data + length), workgroup, constants);
+                    tensors, sequenceName, std::vector<uint32_t>((uint32_t*)data, (uint32_t*)(data + length)), workgroup, constants);
             },
             "Evaluates an operation using a custom shader provided as spirv bytes with explicitly named Sequence",
             py::arg("tensors"), py::arg("sequence_name"), py::arg("bytes"), py::arg("workgroup") = kp::Workgroup(), py::arg("constants") = kp::Constants() )
@@ -264,7 +286,7 @@ PYBIND11_MODULE(kp, m) {
                 const char *data = reinterpret_cast<const char *>(info.ptr);
                 size_t length = static_cast<size_t>(info.size);
                 self.evalOpAsyncDefault<kp::OpAlgoBase>(
-                    tensors, std::vector<char>(data, data + length), workgroup, constants);
+                    tensors, std::vector<uint32_t>((uint32_t*)data, (uint32_t*)(data + length)), workgroup, constants);
             },
             "Evaluates asynchronously an operation using a custom shader provided as raw string or spirv bytes with anonymous Sequence",
             py::arg("tensors"), py::arg("bytes"), py::arg("workgroup") = kp::Workgroup(), py::arg("constants") = kp::Constants() )
@@ -294,7 +316,7 @@ PYBIND11_MODULE(kp, m) {
                 const char *data = reinterpret_cast<const char *>(info.ptr);
                 size_t length = static_cast<size_t>(info.size);
                 self.evalOpAsync<kp::OpAlgoBase>(
-                    tensors, sequenceName, std::vector<char>(data, data + length), workgroup, constants);
+                    tensors, sequenceName, std::vector<uint32_t>((uint32_t*)data, (uint32_t*)(data + length)), workgroup, constants);
             },
             "Evaluates asynchronously an operation using a custom shader provided as raw string or spirv bytes with explicitly named Sequence",
             py::arg("tensors"), py::arg("sequence_name"), py::arg("bytes"), py::arg("workgroup") = kp::Workgroup(), py::arg("constants") = kp::Constants() );

python/test/test_kompute.py

@@ -17,23 +17,36 @@ def test_opalgobase_file():
     tensor_out = kp.Tensor([0, 0, 0])
 
     mgr = kp.Manager()
-
     mgr.rebuild([tensor_in_a, tensor_in_b, tensor_out])
 
-    shader_path = os.path.abspath(os.path.join(DIRNAME, "../../shaders/glsl/opmult.comp.spv"))
+    shader_path = os.path.join(DIRNAME, "../../shaders/glsl/opmult.comp.spv")
-    mgr.eval_async_algo_file_def([tensor_in_a, tensor_in_b, tensor_out], shader_path)
+
+    mgr.eval_algo_file_def([tensor_in_a, tensor_in_b, tensor_out], shader_path)
 
     mgr.eval_tensor_sync_local_def([tensor_out])
 
     assert tensor_out.data() == [2.0, 4.0, 6.0]
-    assert np.all(tensor_out.numpy() == [2.0, 4.0, 6.0])
 
 
-def test_opalgobase_file():
+def test_shader_str():
     """
     Test basic OpAlgoBase operation
     """
 
+    shader = """
+#version 450
+layout(set = 0, binding = 0) buffer tensorLhs {float valuesLhs[];};
+layout(set = 0, binding = 1) buffer tensorRhs {float valuesRhs[];};
+layout(set = 0, binding = 2) buffer tensorOutput { float valuesOutput[];};
+layout (local_size_x = 1, local_size_y = 1, local_size_z = 1) in;
+
+void main()
+{
+    uint index = gl_GlobalInvocationID.x;
+    valuesOutput[index] = valuesLhs[index] * valuesRhs[index];
+}
+    """
+
     tensor_in_a = kp.Tensor([2, 2, 2])
     tensor_in_b = kp.Tensor([1, 2, 3])
     tensor_out = kp.Tensor([0, 0, 0])
@@ -41,9 +54,9 @@ def test_opalgobase_file():
     mgr = kp.Manager()
     mgr.rebuild([tensor_in_a, tensor_in_b, tensor_out])
 
-    shader_path = os.path.join(DIRNAME, "../../shaders/glsl/opmult.comp.spv")
+    spirv = kp.Shader.compile_source(shader)
 
-    mgr.eval_algo_file_def([tensor_in_a, tensor_in_b, tensor_out], shader_path)
+    mgr.eval_algo_data_def([tensor_in_a, tensor_in_b, tensor_out], spirv)
 
     mgr.eval_tensor_sync_local_def([tensor_out])
 

single_include/AggregateHeaders.cpp

@@ -1,5 +1,6 @@
 #pragma once
 #include "kompute/Core.hpp"
+#include "kompute/Shader.hpp"
 #include "kompute/shaders/shaderopmult.hpp"
 #include "kompute/shaders/shaderlogisticregression.hpp"
 #include "kompute/Manager.hpp"

single_include/kompute/Kompute.hpp

@@ -107,6 +107,61 @@ extern py::object kp_debug, kp_info, kp_warning, kp_error;
 #endif // KOMPUTE_SPDLOG_ENABLED
 #endif // KOMPUTE_LOG_OVERRIDE
 
+#if !defined(KOMPUTE_DISABLE_SHADER_UTILS) || !KOMPUTE_DISABLE_SHADER_UTILS
+#include <iostream>
+#include <vector>
+
+#include <glslang/Public/ShaderLang.h>
+#include <StandAlone/ResourceLimits.h>
+#include <SPIRV/GlslangToSpv.h>
+
+namespace kp {
+
+/**
+    Shader utily class with functions to compile and process glsl files.
+*/
+class Shader {
+public:
+    /**
+     * Compile multiple sources with optional filenames. Currently this function
+     * uses the glslang C++ interface which is not thread safe so this funciton
+     * should not be called from multiple threads concurrently. If you have a
+     * online shader processing multithreading use-case that can't use offline 
+     * compilation please open an issue.
+     *
+     * @param sources A list of raw glsl shaders in string format
+     * @param files A list of file names respective to each of the sources
+     * @param entryPoint The function name to use as entry point
+     * @param definitions List of pairs containing key value definitions
+     * @return The compiled SPIR-V binary in unsigned int32 format
+     */
+    static std::vector<uint32_t> compile_sources(
+            const std::vector<std::string>& sources,
+            const std::vector<std::string>& files = {},
+            const std::string& entryPoint = "main",
+            std::vector<std::pair<std::string,std::string>> definitions = {});
+
+    /**
+     * Compile a single glslang source from string value. Currently this function
+     * uses the glslang C++ interface which is not thread safe so this funciton
+     * should not be called from multiple threads concurrently. If you have a
+     * online shader processing multithreading use-case that can't use offline 
+     * compilation please open an issue.
+     *
+     * @param source An individual raw glsl shader in string format
+     * @param entryPoint The function name to use as entry point
+     * @param definitions List of pairs containing key value definitions
+     * @return The compiled SPIR-V binary in unsigned int32 format
+     */
+    static std::vector<uint32_t> compile_source(
+            const std::string& source,
+            const std::string& entryPoint = "main",
+            std::vector<std::pair<std::string,std::string>> definitions = {});
+
+};
+}
+#endif // DKOMPUTE_DISABLE_SHADER_UTILS
+
 /*
     THIS FILE HAS BEEN AUTOMATICALLY GENERATED - DO NOT EDIT
 
@@ -133,7 +188,7 @@ extern py::object kp_debug, kp_info, kp_warning, kp_error;
 namespace kp {
 namespace shader_data {
 static const unsigned char shaders_glsl_opmult_comp_spv[] = {
-  0x03, 0x02, 0x23, 0x07, 0x00, 0x00, 0x01, 0x00, 0x08, 0x00, 0x08, 0x00,
+  0x03, 0x02, 0x23, 0x07, 0x00, 0x00, 0x01, 0x00, 0x0a, 0x00, 0x08, 0x00,
   0x2e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x02, 0x00,
   0x01, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x06, 0x00, 0x01, 0x00, 0x00, 0x00,
   0x47, 0x4c, 0x53, 0x4c, 0x2e, 0x73, 0x74, 0x64, 0x2e, 0x34, 0x35, 0x30,
@@ -287,7 +342,7 @@ static const unsigned int shaders_glsl_opmult_comp_spv_len = 1464;
 namespace kp {
 namespace shader_data {
 static const unsigned char shaders_glsl_logisticregression_comp_spv[] = {
-  0x03, 0x02, 0x23, 0x07, 0x00, 0x00, 0x01, 0x00, 0x08, 0x00, 0x08, 0x00,
+  0x03, 0x02, 0x23, 0x07, 0x00, 0x00, 0x01, 0x00, 0x0a, 0x00, 0x08, 0x00,
   0xae, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x02, 0x00,
   0x01, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x06, 0x00, 0x01, 0x00, 0x00, 0x00,
   0x47, 0x4c, 0x53, 0x4c, 0x2e, 0x73, 0x74, 0x64, 0x2e, 0x34, 0x35, 0x30,
@@ -1658,7 +1713,7 @@ public:
      * @specalizationInstalces The specialization parameters to pass to the function
      * processing
      */
-    void init(const std::vector<char>& shaderFileData,
+    void init(const std::vector<uint32_t>& shaderFileData,
               std::vector<std::shared_ptr<Tensor>> tensorParams);
 
     /**
@@ -1702,7 +1757,7 @@ private:
     Constants mSpecializationConstants;
 
     // Create util functions
-    void createShaderModule(const std::vector<char>& shaderFileData);
+    void createShaderModule(const std::vector<uint32_t>& shaderFileData);
     void createPipeline();
 
     // Parameters
@@ -1783,7 +1838,7 @@ class OpAlgoBase : public OpBase
            std::shared_ptr<vk::Device> device,
            std::shared_ptr<vk::CommandBuffer> commandBuffer,
            std::vector<std::shared_ptr<Tensor>>& tensors,
-           const std::vector<char>& shaderDataRaw,
+           const std::vector<uint32_t>& shaderDataRaw,
            const Workgroup& komputeWorkgroup = {},
            const Constants& specializationConstants = {});
 
@@ -1835,9 +1890,9 @@ class OpAlgoBase : public OpBase
     Workgroup mKomputeWorkgroup;
 
     std::string mShaderFilePath; ///< Optional member variable which can be provided for the OpAlgoBase to find the data automatically and load for processing
-    std::vector<char> mShaderDataRaw; ///< Optional member variable which can be provided to contain either the raw shader content or the spirv binary content
+    std::vector<uint32_t> mShaderDataRaw; ///< Optional member variable which can be provided to contain either the raw shader content or the spirv binary content
 
-    virtual std::vector<char> fetchSpirvBinaryData();
+    virtual std::vector<uint32_t> fetchSpirvBinaryData();
 };
 
 } // End namespace kp
@@ -1960,7 +2015,7 @@ class OpMult : public OpAlgoBase
         SPDLOG_DEBUG("Kompute OpMult constructor with params");
 
 #ifndef RELEASE
-        this->mShaderFilePath = "shaders/glsl/opmult.comp";
+        this->mShaderFilePath = "shaders/glsl/opmult.comp.spv";
 #endif
     }
 
@@ -1969,15 +2024,15 @@ class OpMult : public OpAlgoBase
      * If RELEASE=1 it will be using the static version of the shader which is 
      * loaded using this file directly. Otherwise it should not override the function.
      */
-    std::vector<char> fetchSpirvBinaryData() override
+    std::vector<uint32_t> fetchSpirvBinaryData() override
     {
         SPDLOG_WARN(
           "Kompute OpMult Running shaders directly from header");
 
-        return std::vector<char>(
+        return std::vector<uint32_t>(
-          shader_data::shaders_glsl_opmult_comp_spv,
+          (uint32_t*)shader_data::shaders_glsl_opmult_comp_spv,
-          shader_data::shaders_glsl_opmult_comp_spv +
+          (uint32_t*)(shader_data::shaders_glsl_opmult_comp_spv +
-            kp::shader_data::shaders_glsl_opmult_comp_spv_len);
+            kp::shader_data::shaders_glsl_opmult_comp_spv_len));
 
     }
 #endif

src/Algorithm.cpp

@@ -108,7 +108,7 @@ Algorithm::~Algorithm()
 }
 
 void
-Algorithm::init(const std::vector<char>& shaderFileData,
+Algorithm::init(const std::vector<uint32_t>& shaderFileData,
                 std::vector<std::shared_ptr<Tensor>> tensorParams)
 {
     SPDLOG_DEBUG("Kompute Algorithm init started");
@@ -149,6 +149,7 @@ Algorithm::createParameters(std::vector<std::shared_ptr<Tensor>>& tensorParams)
     this->mDescriptorPool = std::make_shared<vk::DescriptorPool>();
     this->mDevice->createDescriptorPool(
       &descriptorPoolInfo, nullptr, this->mDescriptorPool.get());
+    this->mFreeDescriptorPool = true;
 
     std::vector<vk::DescriptorSetLayoutBinding> descriptorSetBindings;
     for (size_t i = 0; i < tensorParams.size(); i++) {
@@ -206,14 +207,14 @@ Algorithm::createParameters(std::vector<std::shared_ptr<Tensor>>& tensorParams)
 }
 
 void
-Algorithm::createShaderModule(const std::vector<char>& shaderFileData)
+Algorithm::createShaderModule(const std::vector<uint32_t>& shaderFileData)
 {
     SPDLOG_DEBUG("Kompute Algorithm createShaderModule started");
 
     vk::ShaderModuleCreateInfo shaderModuleInfo(
       vk::ShaderModuleCreateFlags(),
-      shaderFileData.size(),
+      sizeof(uint32_t) * shaderFileData.size(),
-      (uint32_t*)shaderFileData.data());
+      shaderFileData.data());
 
     SPDLOG_DEBUG("Kompute Algorithm Creating shader module. ShaderFileSize: {}",
                  shaderFileData.size());

src/CMakeLists.txt

@@ -105,6 +105,44 @@ if(KOMPUTE_OPT_BUILD_SINGLE_HEADER)
         build_single_header)
 endif()
 
+#####################################################
+#################### GLSLANG #######################
+#####################################################
+
+if(NOT KOMPUTE_OPT_DISABLE_SHADER_UTILS)
+    if(KOMPUTE_OPT_REPO_SUBMODULE_BUILD)
+        add_subdirectory(${PROJECT_SOURCE_DIR}/external/glslang
+            ${CMAKE_CURRENT_BINARY_DIR}/kompute_glslang)
+
+        target_include_directories(
+            kompute PRIVATE
+            ${PROJECT_SOURCE_DIR}/external/glslang)
+
+        target_link_libraries(kompute
+            # Not including hlsl support
+            # HLSL
+            # glslang includes OGLCompiler, OSDependent, MachineIndependent
+            glslang
+            SPIRV
+            glslang-default-resource-limits)
+    else()
+        find_package(glslang CONFIG REQUIRED)
+
+        target_include_directories(
+            kompute PRIVATE
+            ${GLSLANG_GENERATED_INCLUDEDIR})
+
+        target_link_libraries(kompute
+            # Not including hlsl support
+            # glslang::HLSL
+            # Adding explicit dependencies to match above
+            glslang
+            SPIRV
+            glslang-default-resource-limits)
+    endif()
+endif()
+
+
 add_library(kompute::kompute ALIAS kompute)
 
 if(KOMPUTE_OPT_INSTALL)

src/OpAlgoBase.cpp

@@ -61,7 +61,7 @@ OpAlgoBase::OpAlgoBase(std::shared_ptr<vk::PhysicalDevice> physicalDevice,
                        std::shared_ptr<vk::Device> device,
                        std::shared_ptr<vk::CommandBuffer> commandBuffer,
                        std::vector<std::shared_ptr<Tensor>>& tensors,
-                       const std::vector<char>& shaderDataRaw,
+                       const std::vector<uint32_t>& shaderDataRaw,
                        const Workgroup& komputeWorkgroup,
                        const Constants& specializationConstants)
   : OpAlgoBase(physicalDevice, device, commandBuffer, tensors, komputeWorkgroup, specializationConstants)
@@ -98,7 +98,7 @@ OpAlgoBase::init()
 
     SPDLOG_DEBUG("Kompute OpAlgoBase fetching spirv data");
 
-    std::vector<char> shaderFileData = this->fetchSpirvBinaryData();
+    std::vector<uint32_t> shaderFileData = this->fetchSpirvBinaryData();
 
     SPDLOG_DEBUG("Kompute OpAlgoBase Initialising algorithm component");
 
@@ -137,7 +137,7 @@ OpAlgoBase::postEval()
     SPDLOG_DEBUG("Kompute OpAlgoBase postSubmit called");
 }
 
-std::vector<char>
+std::vector<uint32_t>
 OpAlgoBase::fetchSpirvBinaryData()
 {
     SPDLOG_DEBUG("Kompute OpAlgoBase Running fetchSpirvBinaryData");
@@ -162,7 +162,7 @@ OpAlgoBase::fetchSpirvBinaryData()
 
         SPDLOG_WARN("Kompute OpAlgoBase fetched {} bytes", shaderFileSize);
 
-        return std::vector<char>(shaderDataRaw, shaderDataRaw + shaderFileSize);
+        return std::vector<uint32_t>((uint32_t*)shaderDataRaw, (uint32_t*)(shaderDataRaw + shaderFileSize));
     } else if (this->mShaderDataRaw.size()) {
         SPDLOG_DEBUG("Kompute OpAlgoBase Reading data from data provided");
         return this->mShaderDataRaw;

src/OpAlgoLhsRhsOut.cpp

@@ -67,7 +67,7 @@ OpAlgoLhsRhsOut::init()
 
     SPDLOG_DEBUG("Kompute OpAlgoLhsRhsOut fetching spirv data");
 
-    std::vector<char> shaderFileData = this->fetchSpirvBinaryData();
+    std::vector<uint32_t> shaderFileData = this->fetchSpirvBinaryData();
 
     SPDLOG_DEBUG("Kompute OpAlgoLhsRhsOut Initialising algorithm component");
 

src/Shader.cpp

@@ -0,0 +1,96 @@
+
+#if !defined(KOMPUTE_DISABLE_SHADER_UTILS) || !KOMPUTE_DISABLE_SHADER_UTILS
+#include "kompute/Shader.hpp"
+
+namespace kp {
+
+std::vector<uint32_t>
+Shader::compile_sources(const std::vector<std::string>& sources,
+                                   const std::vector<std::string>& files,
+                                   const std::string& entryPoint,
+                                   std::vector<std::pair<std::string,std::string>> definitions) {
+
+    // Initialize glslang library.
+    glslang::InitializeProcess();
+
+    // Currently we don't support other shader types nor plan to
+    const EShLanguage language = EShLangCompute;
+    glslang::TShader shader(language);
+
+    std::vector<const char*> filesCStr(files.size()), sourcesCStr(sources.size());
+    for (size_t i = 0; i < sources.size(); i++) sourcesCStr[i] = sources[i].c_str();
+
+    if (files.size() > 1) {
+        assert(files.size() == sources.size());
+        for (size_t i = 0; i < files.size(); i++) filesCStr[i] = files[i].c_str();
+        shader.setStringsWithLengthsAndNames(sourcesCStr.data(), nullptr, filesCStr.data(), filesCStr.size());
+    }
+    else {
+        filesCStr = {""};
+        shader.setStringsWithLengthsAndNames(sourcesCStr.data(), nullptr, filesCStr.data(), sourcesCStr.size());
+    }
+
+    shader.setEntryPoint(entryPoint.c_str());
+    shader.setSourceEntryPoint(entryPoint.c_str());
+
+    std::string info_log = "";
+    const EShMessages messages = static_cast<EShMessages>(EShMsgDefault | EShMsgVulkanRules | EShMsgSpvRules);
+    if (!shader.parse(&glslang::DefaultTBuiltInResource, 100, false, messages))
+    {
+        info_log = std::string(shader.getInfoLog()) + "\n" + std::string(shader.getInfoDebugLog());
+        SPDLOG_ERROR("Kompute Shader Error: {}", info_log);
+        throw std::runtime_error(info_log);
+    }
+
+    // Add shader to new program object.
+    glslang::TProgram program;
+    program.addShader(&shader);
+    // Link program.
+    if (!program.link(messages))
+    {
+        info_log = std::string(program.getInfoLog()) + "\n" + std::string(program.getInfoDebugLog());
+        SPDLOG_ERROR("Kompute Shader Error: {}", info_log);
+        throw std::runtime_error(info_log);
+    }
+
+    // Save any info log that was generated.
+    if (shader.getInfoLog())
+    {
+        info_log += std::string(shader.getInfoLog()) + "\n" + std::string(shader.getInfoDebugLog()) + "\n";
+        SPDLOG_INFO("Kompute Shader Information: {}", info_log);
+    }
+
+    glslang::TIntermediate *intermediate = program.getIntermediate(language);
+    // Translate to SPIRV.
+    if (!intermediate)
+    {
+        info_log += "Failed to get shared intermediate code.\n";
+        SPDLOG_ERROR("Kompute Shader Error: {}", info_log);
+        throw std::runtime_error(info_log);
+    }
+
+    spv::SpvBuildLogger logger;
+    std::vector<std::uint32_t> spirv;
+    glslang::GlslangToSpv(*intermediate, spirv, &logger);
+
+    if (shader.getInfoLog())
+    {
+        info_log += logger.getAllMessages() + "\n";
+        SPDLOG_DEBUG("Kompute Shader all result messages: {}", info_log);
+    }
+
+    // Shutdown glslang library.
+    glslang::FinalizeProcess();
+
+    return spirv;
+}
+
+std::vector<uint32_t>
+Shader::compile_source(const std::string& source,
+        const std::string& entryPoint,
+        std::vector<std::pair<std::string,std::string>> definitions) {
+    return compile_sources({source});
+}
+
+}
+#endif // DKOMPUTE_DISABLE_SHADER_UTILS

src/include/kompute/Algorithm.hpp

@@ -39,7 +39,7 @@ public:
      * @specalizationInstalces The specialization parameters to pass to the function
      * processing
      */
-    void init(const std::vector<char>& shaderFileData,
+    void init(const std::vector<uint32_t>& shaderFileData,
               std::vector<std::shared_ptr<Tensor>> tensorParams);
 
     /**
@@ -83,7 +83,7 @@ private:
     Constants mSpecializationConstants;
 
     // Create util functions
-    void createShaderModule(const std::vector<char>& shaderFileData);
+    void createShaderModule(const std::vector<uint32_t>& shaderFileData);
     void createPipeline();
 
     // Parameters

src/include/kompute/Core.hpp

@@ -1,6 +1,6 @@
 #pragma once
 
-#ifdef VK_USE_PLATFORM_ANDROID_KHR
+#if VK_USE_PLATFORM_ANDROID_KHR
 #include <android/log.h>
 #include <kompute_vk_ndk_wrapper.hpp>
 // VK_NO_PROTOTYPES required before vulkan import but after wrapper.hpp
@@ -82,7 +82,7 @@ extern py::object kp_debug, kp_info, kp_warning, kp_error;
 #else
 #if defined(VK_USE_PLATFORM_ANDROID_KHR)
 #define SPDLOG_WARN(message, ...)                                              \
-    ((void)__android_log_print(ANDROID_LOG_INFO, KOMPUTE_LOG_TAG, message))
+    ((void)__android_log_print(ANDROID_LOG_WARN, KOMPUTE_LOG_TAG, message))
 #elif defined(KOMPUTE_BUILD_PYTHON)
 #define SPDLOG_WARN(message, ...) kp_warning(message);
 #else
@@ -96,7 +96,7 @@ extern py::object kp_debug, kp_info, kp_warning, kp_error;
 #else
 #if defined(VK_USE_PLATFORM_ANDROID_KHR)
 #define SPDLOG_ERROR(message, ...)                                             \
-    ((void)__android_log_print(ANDROID_LOG_INFO, KOMPUTE_LOG_TAG, message))
+    ((void)__android_log_print(ANDROID_LOG_ERROR, KOMPUTE_LOG_TAG, message))
 #elif defined(KOMPUTE_BUILD_PYTHON)
 #define SPDLOG_ERROR(message, ...) kp_error(message);
 #else

src/include/kompute/Shader.hpp

@@ -0,0 +1,59 @@
+#pragma once
+
+#if !defined(KOMPUTE_DISABLE_SHADER_UTILS) || !KOMPUTE_DISABLE_SHADER_UTILS
+#include <iostream>
+#include <vector>
+
+#include <glslang/Public/ShaderLang.h>
+#include <StandAlone/ResourceLimits.h>
+#include <SPIRV/GlslangToSpv.h>
+
+#include "kompute/Core.hpp"
+
+namespace kp {
+
+/**
+    Shader utily class with functions to compile and process glsl files.
+*/
+class Shader {
+public:
+    /**
+     * Compile multiple sources with optional filenames. Currently this function
+     * uses the glslang C++ interface which is not thread safe so this funciton
+     * should not be called from multiple threads concurrently. If you have a
+     * online shader processing multithreading use-case that can't use offline 
+     * compilation please open an issue.
+     *
+     * @param sources A list of raw glsl shaders in string format
+     * @param files A list of file names respective to each of the sources
+     * @param entryPoint The function name to use as entry point
+     * @param definitions List of pairs containing key value definitions
+     * @return The compiled SPIR-V binary in unsigned int32 format
+     */
+    static std::vector<uint32_t> compile_sources(
+            const std::vector<std::string>& sources,
+            const std::vector<std::string>& files = {},
+            const std::string& entryPoint = "main",
+            std::vector<std::pair<std::string,std::string>> definitions = {});
+
+    /**
+     * Compile a single glslang source from string value. Currently this function
+     * uses the glslang C++ interface which is not thread safe so this funciton
+     * should not be called from multiple threads concurrently. If you have a
+     * online shader processing multithreading use-case that can't use offline 
+     * compilation please open an issue.
+     *
+     * @param source An individual raw glsl shader in string format
+     * @param entryPoint The function name to use as entry point
+     * @param definitions List of pairs containing key value definitions
+     * @return The compiled SPIR-V binary in unsigned int32 format
+     */
+    static std::vector<uint32_t> compile_source(
+            const std::string& source,
+            const std::string& entryPoint = "main",
+            std::vector<std::pair<std::string,std::string>> definitions = {});
+
+};
+}
+#endif // DKOMPUTE_DISABLE_SHADER_UTILS
+

src/include/kompute/operations/OpAlgoBase.hpp

@@ -82,7 +82,7 @@ class OpAlgoBase : public OpBase
            std::shared_ptr<vk::Device> device,
            std::shared_ptr<vk::CommandBuffer> commandBuffer,
            std::vector<std::shared_ptr<Tensor>>& tensors,
-           const std::vector<char>& shaderDataRaw,
+           const std::vector<uint32_t>& shaderDataRaw,
            const Workgroup& komputeWorkgroup = {},
            const Constants& specializationConstants = {});
 
@@ -135,9 +135,9 @@ class OpAlgoBase : public OpBase
     Workgroup mKomputeWorkgroup;
 
     std::string mShaderFilePath; ///< Optional member variable which can be provided for the OpAlgoBase to find the data automatically and load for processing
-    std::vector<char> mShaderDataRaw; ///< Optional member variable which can be provided to contain either the raw shader content or the spirv binary content
+    std::vector<uint32_t> mShaderDataRaw; ///< Optional member variable which can be provided to contain either the raw shader content or the spirv binary content
 
-    virtual std::vector<char> fetchSpirvBinaryData();
+    virtual std::vector<uint32_t> fetchSpirvBinaryData();
 };
 
 } // End namespace kp

src/include/kompute/operations/OpMult.hpp

@@ -50,7 +50,7 @@ class OpMult : public OpAlgoBase
         SPDLOG_DEBUG("Kompute OpMult constructor with params");
 
 #ifndef RELEASE
-        this->mShaderFilePath = "shaders/glsl/opmult.comp";
+        this->mShaderFilePath = "shaders/glsl/opmult.comp.spv";
 #endif
     }
 
@@ -59,15 +59,15 @@ class OpMult : public OpAlgoBase
      * If RELEASE=1 it will be using the static version of the shader which is 
      * loaded using this file directly. Otherwise it should not override the function.
      */
-    std::vector<char> fetchSpirvBinaryData() override
+    std::vector<uint32_t> fetchSpirvBinaryData() override
     {
         SPDLOG_WARN(
           "Kompute OpMult Running shaders directly from header");
 
-        return std::vector<char>(
+        return std::vector<uint32_t>(
-          shader_data::shaders_glsl_opmult_comp_spv,
+          (uint32_t*)shader_data::shaders_glsl_opmult_comp_spv,
-          shader_data::shaders_glsl_opmult_comp_spv +
+          (uint32_t*)(shader_data::shaders_glsl_opmult_comp_spv +
-            kp::shader_data::shaders_glsl_opmult_comp_spv_len);
+            kp::shader_data::shaders_glsl_opmult_comp_spv_len));
 
     }
 #endif

src/include/kompute/shaders/shaderlogisticregression.hpp

@@ -24,7 +24,7 @@
 namespace kp {
 namespace shader_data {
 static const unsigned char shaders_glsl_logisticregression_comp_spv[] = {
-  0x03, 0x02, 0x23, 0x07, 0x00, 0x00, 0x01, 0x00, 0x08, 0x00, 0x08, 0x00,
+  0x03, 0x02, 0x23, 0x07, 0x00, 0x00, 0x01, 0x00, 0x0a, 0x00, 0x08, 0x00,
   0xae, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x02, 0x00,
   0x01, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x06, 0x00, 0x01, 0x00, 0x00, 0x00,
   0x47, 0x4c, 0x53, 0x4c, 0x2e, 0x73, 0x74, 0x64, 0x2e, 0x34, 0x35, 0x30,

src/include/kompute/shaders/shaderopmult.hpp

@@ -24,7 +24,7 @@
 namespace kp {
 namespace shader_data {
 static const unsigned char shaders_glsl_opmult_comp_spv[] = {
-  0x03, 0x02, 0x23, 0x07, 0x00, 0x00, 0x01, 0x00, 0x08, 0x00, 0x08, 0x00,
+  0x03, 0x02, 0x23, 0x07, 0x00, 0x00, 0x01, 0x00, 0x0a, 0x00, 0x08, 0x00,
   0x2e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x02, 0x00,
   0x01, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x06, 0x00, 0x01, 0x00, 0x00, 0x00,
   0x47, 0x4c, 0x53, 0x4c, 0x2e, 0x73, 0x74, 0x64, 0x2e, 0x34, 0x35, 0x30,

test/CMakeLists.txt

@@ -1,4 +1,7 @@
 
+#####################################################
+#################### GETEST #######################
+#####################################################
 enable_testing()
 if(KOMPUTE_OPT_REPO_SUBMODULE_BUILD)
     add_subdirectory(${PROJECT_SOURCE_DIR}/external/googletest EXCLUDE_FROM_ALL
@@ -67,25 +70,28 @@ if (KOMPUTE_OPT_CODE_COVERAGE)
         WORKING_DIRECTORY ${CODECOV_DIR}
         DEPENDS codecov_copy_files)
 
-    add_custom_target(codecov_lcov
+    add_custom_target(codecov_lcov_capture
         COMMAND lcov
             --capture
             -o ${CODECOV_FILENAME_LCOV_INFO_FULL}
             -d .
+        WORKING_DIRECTORY ${CODECOV_DIR}
+        DEPENDS codecov_gcov)
+    add_custom_target(codecov_lcov_extract
         COMMAND lcov
             --extract
             ${CODECOV_FILENAME_LCOV_INFO_FULL}
             -o ${CODECOV_FILENAME_LCOV_INFO}
             -d .
-            "*/src/*" "*/test/*"
+            "*/src/*"
         WORKING_DIRECTORY ${CODECOV_DIR}
-        DEPENDS codecov_gcov)
+        DEPENDS codecov_lcov_capture)
 
     add_custom_target(codecov_genhtml
         COMMAND genhtml
             ${CODECOV_FILENAME_LCOV_INFO}
             --output-directory ${CODECOV_DIR_HTML}
         WORKING_DIRECTORY ${CODECOV_DIR}
-        DEPENDS codecov_lcov)
+        DEPENDS codecov_lcov_extract)
 endif()
 

test/TestAsyncOperations.cpp

@@ -55,7 +55,7 @@ TEST(TestAsyncOperations, TestManagerParallelExecution)
 
     for (uint32_t i = 0; i < numParallel; i++) {
         mgr.evalOpDefault<kp::OpAlgoBase>(
-          { inputsSyncB[i] }, std::vector<char>(shader.begin(), shader.end()));
+          { inputsSyncB[i] }, kp::Shader::compile_source(shader));
     }
 
     auto endSync = std::chrono::high_resolution_clock::now();
@@ -89,7 +89,7 @@ TEST(TestAsyncOperations, TestManagerParallelExecution)
         mgrAsync.evalOpAsync<kp::OpAlgoBase>(
           { inputsAsyncB[i] },
           "async" + std::to_string(i),
-          std::vector<char>(shader.begin(), shader.end()));
+          kp::Shader::compile_source(shader));
     }
 
     for (uint32_t i = 0; i < numParallel; i++) {
@@ -151,11 +151,13 @@ TEST(TestAsyncOperations, TestManagerAsyncExecution)
 
     mgr.rebuild({ tensorA, tensorB });
 
-    mgr.evalOpAsync<kp::OpAlgoBase>(
+    std::vector<uint32_t> result = kp::Shader::compile_source(shader);
-      { tensorA }, "asyncOne", std::vector<char>(shader.begin(), shader.end()));
 
     mgr.evalOpAsync<kp::OpAlgoBase>(
-      { tensorB }, "asyncTwo", std::vector<char>(shader.begin(), shader.end()));
+      { tensorA }, "asyncOne", kp::Shader::compile_source(shader));
+
+    mgr.evalOpAsync<kp::OpAlgoBase>(
+      { tensorB }, "asyncTwo", kp::Shader::compile_source(shader));
 
     mgr.evalOpAwait("asyncOne");
     mgr.evalOpAwait("asyncTwo");

test/TestDestroy.cpp

@@ -28,7 +28,7 @@ TEST(TestDestroy, TestDestroyTensorSingle)
 
             sq->begin();
             sq->record<kp::OpAlgoBase>(
-              { tensorA }, std::vector<char>(shader.begin(), shader.end()));
+              { tensorA }, kp::Shader::compile_source(shader));
             sq->end();
 
             sq->eval();
@@ -71,7 +71,7 @@ TEST(TestDestroy, TestDestroyTensorVector)
 
             sq->begin();
             sq->record<kp::OpAlgoBase>(
-              { tensorA, tensorB }, std::vector<char>(shader.begin(), shader.end()));
+              { tensorA, tensorB }, kp::Shader::compile_source(shader));
             sq->end();
 
             sq->eval();
@@ -136,7 +136,7 @@ TEST(TestDestroy, TestDestroySequenceSingle)
 
             sq->begin();
             sq->record<kp::OpAlgoBase>(
-              { tensorA }, std::vector<char>(shader.begin(), shader.end()));
+              { tensorA }, kp::Shader::compile_source(shader));
             sq->end();
 
             sq->eval();
@@ -176,14 +176,14 @@ TEST(TestDestroy, TestDestroySequenceVector)
             sq1 = mgr.sequence("One");
             sq1->begin();
             sq1->record<kp::OpAlgoBase>(
-              { tensorA }, std::vector<char>(shader.begin(), shader.end()));
+              { tensorA }, kp::Shader::compile_source(shader));
             sq1->end();
             sq1->eval();
 
             sq2 = mgr.sequence("Two");
             sq2->begin();
             sq2->record<kp::OpAlgoBase>(
-              { tensorA }, std::vector<char>(shader.begin(), shader.end()));
+              { tensorA }, kp::Shader::compile_source(shader));
             sq2->end();
             sq2->eval();
 
@@ -218,11 +218,11 @@ TEST(TestDestroy, TestDestroySequenceNameSingleInsideManager)
 
             mgr.evalOp<kp::OpAlgoBase>(
               { tensorA }, "one",
-              std::vector<char>(shader.begin(), shader.end()));
+              kp::Shader::compile_source(shader));
 
             mgr.evalOp<kp::OpAlgoBase>(
               { tensorA }, "two",
-              std::vector<char>(shader.begin(), shader.end()));
+              kp::Shader::compile_source(shader));
 
             mgr.evalOpDefault<kp::OpTensorSyncLocal>({ tensorA });
 
@@ -257,7 +257,7 @@ TEST(TestDestroy, TestDestroySequenceNameSingleOutsideManager)
             sq1 = mgr.sequence("One");
             sq1->begin();
             sq1->record<kp::OpAlgoBase>(
-              { tensorA }, std::vector<char>(shader.begin(), shader.end()));
+              { tensorA }, kp::Shader::compile_source(shader));
             sq1->end();
             sq1->eval();
 
@@ -291,11 +291,11 @@ TEST(TestDestroy, TestDestroySequenceNameVectorInsideManager)
 
             mgr.evalOp<kp::OpAlgoBase>(
               { tensorA }, "one",
-              std::vector<char>(shader.begin(), shader.end()));
+              kp::Shader::compile_source(shader));
 
             mgr.evalOp<kp::OpAlgoBase>(
               { tensorA }, "two",
-              std::vector<char>(shader.begin(), shader.end()));
+              kp::Shader::compile_source(shader));
 
             mgr.evalOpDefault<kp::OpTensorSyncLocal>({ tensorA });
 
@@ -325,11 +325,11 @@ TEST(TestDestroy, TestDestroySequenceNameVectorOutsideManager)
 
             mgr.evalOp<kp::OpAlgoBase>(
               { tensorA }, "one",
-              std::vector<char>(shader.begin(), shader.end()));
+              kp::Shader::compile_source(shader));
 
             mgr.evalOp<kp::OpAlgoBase>(
               { tensorA }, "two",
-              std::vector<char>(shader.begin(), shader.end()));
+              kp::Shader::compile_source(shader));
 
             mgr.evalOpDefault<kp::OpTensorSyncLocal>({ tensorA });
 
@@ -359,7 +359,7 @@ TEST(TestDestroy, TestDestroySequenceNameDefaultOutsideManager)
 
             mgr.evalOpDefault<kp::OpAlgoBase>(
               { tensorA },
-              std::vector<char>(shader.begin(), shader.end()));
+              kp::Shader::compile_source(shader));
 
             mgr.evalOpDefault<kp::OpTensorSyncLocal>({ tensorA });
 

test/TestLogisticRegression.cpp

@@ -5,7 +5,7 @@
 
 #include "kompute_test/shaders/shadertest_logistic_regression.hpp"
 
-TEST(TestLogisticRegressionAlgorithm, TestMainLogisticRegression)
+TEST(TestLogisticRegression, TestMainLogisticRegression)
 {
 
     uint32_t ITERATIONS = 100;
@@ -41,19 +41,13 @@ TEST(TestLogisticRegressionAlgorithm, TestMainLogisticRegression)
 
         sq->record<kp::OpTensorSyncDevice>({ wIn, bIn });
 
-#ifdef KOMPUTE_SHADER_FROM_STRING
-        sq->record<kp::OpAlgoBase>(
-          params, "test/shaders/glsl/test_logistic_regression.comp",
-          kp::Workgroup(), kp::Constants({5.0}));
-#else
         sq->record<kp::OpAlgoBase>(
           params,
-          std::vector<char>(
+          std::vector<uint32_t>(
-            kp::shader_data::shaders_glsl_logisticregression_comp_spv,
+            (uint32_t*)kp::shader_data::shaders_glsl_logisticregression_comp_spv,
-            kp::shader_data::shaders_glsl_logisticregression_comp_spv +
+            (uint32_t*)(kp::shader_data::shaders_glsl_logisticregression_comp_spv +
-              kp::shader_data::shaders_glsl_logisticregression_comp_spv_len),
+              kp::shader_data::shaders_glsl_logisticregression_comp_spv_len)),
           kp::Workgroup(), kp::Constants({5.0}));
-#endif
 
         sq->record<kp::OpTensorSyncLocal>({ wOutI, wOutJ, bOut, lOut });
 
@@ -87,7 +81,7 @@ TEST(TestLogisticRegressionAlgorithm, TestMainLogisticRegression)
                 bIn->data()[0]);
 }
 
-TEST(TestLogisticRegressionAlgorithm, TestMainLogisticRegressionManualCopy)
+TEST(TestLogisticRegression, TestMainLogisticRegressionManualCopy)
 {
 
     uint32_t ITERATIONS = 100;
@@ -126,19 +120,13 @@ TEST(TestLogisticRegressionAlgorithm, TestMainLogisticRegressionManualCopy)
         // Record op algo base
         sq->begin();
 
-#ifdef KOMPUTE_SHADER_FROM_STRING
-        sq->record<kp::OpAlgoBase>(
-          params, "test/shaders/glsl/test_logistic_regression.comp.spv",
-           kp::Workgroup(), kp::Algorithm::SpecializationContainer{{(uint32_t)5}});
-#else
         sq->record<kp::OpAlgoBase>(
           params,
-          std::vector<char>(
+          std::vector<uint32_t>(
-            kp::shader_data::shaders_glsl_logisticregression_comp_spv,
+            (uint32_t*)kp::shader_data::shaders_glsl_logisticregression_comp_spv,
-            kp::shader_data::shaders_glsl_logisticregression_comp_spv +
+            (uint32_t*)(kp::shader_data::shaders_glsl_logisticregression_comp_spv +
-              kp::shader_data::shaders_glsl_logisticregression_comp_spv_len),
+              kp::shader_data::shaders_glsl_logisticregression_comp_spv_len)),
-           kp::Workgroup(), kp::Constants({5.0}));
+          kp::Workgroup(), kp::Constants({5.0}));
-#endif
 
         sq->record<kp::OpTensorSyncLocal>({ wOutI, wOutJ, bOut, lOut });
 

test/TestMultipleAlgoExecutions.cpp

@@ -28,11 +28,11 @@ TEST(TestMultipleAlgoExecutions, SingleSequenceRecord)
         sq->begin();
 
         sq->record<kp::OpAlgoBase>(
-          { tensorA }, std::vector<char>(shader.begin(), shader.end()));
+          { tensorA }, kp::Shader::compile_source(shader));
         sq->record<kp::OpAlgoBase>(
-          { tensorA }, std::vector<char>(shader.begin(), shader.end()));
+          { tensorA }, kp::Shader::compile_source(shader));
         sq->record<kp::OpAlgoBase>(
-          { tensorA }, std::vector<char>(shader.begin(), shader.end()));
+          { tensorA }, kp::Shader::compile_source(shader));
 
         sq->record<kp::OpTensorSyncLocal>({ tensorA });
 
@@ -73,19 +73,19 @@ TEST(TestMultipleAlgoExecutions, MultipleCmdBufRecords)
     // Then perform the computations
     sq->begin();
     sq->record<kp::OpAlgoBase>({ tensorA },
-                               std::vector<char>(shader.begin(), shader.end()));
+                               kp::Shader::compile_source(shader));
     sq->end();
     sq->eval();
 
     sq->begin();
     sq->record<kp::OpAlgoBase>({ tensorA },
-                               std::vector<char>(shader.begin(), shader.end()));
+                               kp::Shader::compile_source(shader));
     sq->end();
     sq->eval();
 
     sq->begin();
     sq->record<kp::OpAlgoBase>({ tensorA },
-                               std::vector<char>(shader.begin(), shader.end()));
+                               kp::Shader::compile_source(shader));
     sq->end();
     sq->eval();
 
@@ -122,7 +122,7 @@ TEST(TestMultipleAlgoExecutions, MultipleSequences)
         sq->begin();
 
         sq->record<kp::OpAlgoBase>(
-          { tensorA }, std::vector<char>(shader.begin(), shader.end()));
+          { tensorA }, kp::Shader::compile_source(shader));
 
         sq->end();
         sq->eval();
@@ -135,7 +135,7 @@ TEST(TestMultipleAlgoExecutions, MultipleSequences)
         sq->begin();
 
         sq->record<kp::OpAlgoBase>(
-          { tensorA }, std::vector<char>(shader.begin(), shader.end()));
+          { tensorA }, kp::Shader::compile_source(shader));
 
         sq->end();
         sq->eval();
@@ -148,7 +148,7 @@ TEST(TestMultipleAlgoExecutions, MultipleSequences)
         sq->begin();
 
         sq->record<kp::OpAlgoBase>(
-          { tensorA }, std::vector<char>(shader.begin(), shader.end()));
+          { tensorA }, kp::Shader::compile_source(shader));
 
         sq->end();
         sq->eval();
@@ -206,7 +206,7 @@ TEST(TestMultipleAlgoExecutions, SingleRecordMultipleEval)
         sq->begin();
 
         sq->record<kp::OpAlgoBase>(
-          { tensorA }, std::vector<char>(shader.begin(), shader.end()));
+          { tensorA }, kp::Shader::compile_source(shader));
 
         sq->end();
 
@@ -265,7 +265,7 @@ TEST(TestMultipleAlgoExecutions, ManagerEvalMultSourceStrOpCreate)
 
     mgr.evalOpDefault<kp::OpAlgoBase>(
       { tensorInA, tensorInB, tensorOut },
-      std::vector<char>(shader.begin(), shader.end()));
+      kp::Shader::compile_source(shader));
 
     mgr.evalOpDefault<kp::OpTensorSyncLocal>({ tensorOut });
 
@@ -308,7 +308,7 @@ TEST(TestMultipleAlgoExecutions, ManagerEvalMultSourceStrMgrCreate)
 
     mgr.evalOpDefault<kp::OpAlgoBase>(
       { tensorInA, tensorInB, tensorOut },
-      std::vector<char>(shader.begin(), shader.end()));
+      kp::Shader::compile_source(shader));
 
     mgr.evalOpDefault<kp::OpTensorSyncLocal>({ tensorOut });
 
@@ -340,7 +340,7 @@ TEST(TestMultipleAlgoExecutions, SequenceAlgoDestroyOutsideManagerScope)
 
             sq->begin();
             sq->record<kp::OpAlgoBase>(
-              { tensorA }, std::vector<char>(shader.begin(), shader.end()));
+              { tensorA }, kp::Shader::compile_source(shader));
             sq->end();
 
             sq->eval();

test/TestOpAlgoLoopsPassingData.cpp

@@ -53,7 +53,7 @@ TEST(TestProcessingIterations, IterateThroughMultipleSumAndCopies)
 
         sq->record<kp::OpAlgoBase>(
           { tensorA, tensorB },
-          std::vector<char>(shader.begin(), shader.end()));
+          kp::Shader::compile_source(shader));
 
         sq->record<kp::OpTensorCopy>({ tensorB, tensorA });
         sq->end();

test/TestOpShadersFromStringAndFile.cpp

@@ -29,7 +29,7 @@ TEST(TestOpAlgoBase, ShaderRawDataFromConstructor)
     )");
 
     mgr.evalOpDefault<kp::OpAlgoBase>(
-      { tensorA, tensorB }, std::vector<char>(shader.begin(), shader.end()));
+      { tensorA, tensorB }, kp::Shader::compile_source(shader));
 
     mgr.evalOpDefault<kp::OpTensorSyncLocal>({ tensorA, tensorB });
 
@@ -47,28 +47,11 @@ TEST(TestOpAlgoBase, ShaderCompiledDataFromConstructor)
 
     mgr.evalOpDefault<kp::OpAlgoBase>(
       { tensorA, tensorB },
-      std::vector<char>(
+      std::vector<uint32_t>(
-        kp::shader_data::test_shaders_glsl_test_op_custom_shader_comp_spv,
+        (uint32_t*)kp::shader_data::test_shaders_glsl_test_op_custom_shader_comp_spv,
-        kp::shader_data::test_shaders_glsl_test_op_custom_shader_comp_spv +
+        (uint32_t*)(kp::shader_data::test_shaders_glsl_test_op_custom_shader_comp_spv +
           kp::shader_data::
-            test_shaders_glsl_test_op_custom_shader_comp_spv_len));
+            test_shaders_glsl_test_op_custom_shader_comp_spv_len)));
-
-    mgr.evalOpDefault<kp::OpTensorSyncLocal>({ tensorA, tensorB });
-
-    EXPECT_EQ(tensorA->data(), std::vector<float>({ 0, 1, 2 }));
-    EXPECT_EQ(tensorB->data(), std::vector<float>({ 3, 4, 5 }));
-}
-
-TEST(TestOpAlgoBase, ShaderRawDataFromFile)
-{
-    kp::Manager mgr;
-
-    std::shared_ptr<kp::Tensor> tensorA{ new kp::Tensor({ 3, 4, 5 }) };
-    std::shared_ptr<kp::Tensor> tensorB{ new kp::Tensor({ 0, 0, 0 }) };
-    mgr.rebuild({ tensorA, tensorB });
-
-    mgr.evalOpDefault<kp::OpAlgoBase>(
-      { tensorA, tensorB }, "test/shaders/glsl/test_op_custom_shader.comp");
 
     mgr.evalOpDefault<kp::OpTensorSyncLocal>({ tensorA, tensorB });
 

test/TestSpecializationConstant.cpp

@@ -1,9 +1,7 @@
-
 #include "gtest/gtest.h"
 
 #include "kompute/Kompute.hpp"
 
-
 TEST(TestSpecializationConstants, TestTwoConstants)
 {
     std::shared_ptr<kp::Tensor> tensorA{ new kp::Tensor({ 0, 0, 0 }) };
@@ -37,7 +35,7 @@ TEST(TestSpecializationConstants, TestTwoConstants)
             sq->begin();
             sq->record<kp::OpAlgoBase>(
               { tensorA, tensorB },
-              std::vector<char>(shader.begin(), shader.end()),
+              kp::Shader::compile_source(shader),
               kp::Workgroup(), spec);
             sq->end();
 

test/TestWorkgroup.cpp

@@ -25,10 +25,10 @@ TEST(TestWorkgroup, TestSimpleWorkgroup)
             sq->begin();
             sq->record<kp::OpAlgoBase>(
               { tensorA, tensorB },
-              std::vector<char>(
+              std::vector<uint32_t>(
-                kp::shader_data::test_shaders_glsl_test_workgroup_comp_spv,
+                (uint32_t*)kp::shader_data::test_shaders_glsl_test_workgroup_comp_spv,
-                kp::shader_data::test_shaders_glsl_test_workgroup_comp_spv +
+                (uint32_t*)(kp::shader_data::test_shaders_glsl_test_workgroup_comp_spv +
-                  kp::shader_data::test_shaders_glsl_test_workgroup_comp_spv_len),
+                  kp::shader_data::test_shaders_glsl_test_workgroup_comp_spv_len)),
                 workgroup);
             sq->end();
 

test/compiled_shaders_include/kompute_test/shaders/shadertest_logistic_regression.hpp

@@ -24,7 +24,7 @@
 namespace kp {
 namespace shader_data {
 static const unsigned char test_shaders_glsl_test_logistic_regression_comp_spv[] = {
-  0x03, 0x02, 0x23, 0x07, 0x00, 0x00, 0x01, 0x00, 0x08, 0x00, 0x08, 0x00,
+  0x03, 0x02, 0x23, 0x07, 0x00, 0x00, 0x01, 0x00, 0x0a, 0x00, 0x08, 0x00,
   0xae, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x02, 0x00,
   0x01, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x06, 0x00, 0x01, 0x00, 0x00, 0x00,
   0x47, 0x4c, 0x53, 0x4c, 0x2e, 0x73, 0x74, 0x64, 0x2e, 0x34, 0x35, 0x30,

test/compiled_shaders_include/kompute_test/shaders/shadertest_op_custom_shader.hpp

@@ -24,7 +24,7 @@
 namespace kp {
 namespace shader_data {
 static const unsigned char test_shaders_glsl_test_op_custom_shader_comp_spv[] = {
-  0x03, 0x02, 0x23, 0x07, 0x00, 0x00, 0x01, 0x00, 0x08, 0x00, 0x08, 0x00,
+  0x03, 0x02, 0x23, 0x07, 0x00, 0x00, 0x01, 0x00, 0x0a, 0x00, 0x08, 0x00,
   0x27, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x02, 0x00,
   0x01, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x06, 0x00, 0x01, 0x00, 0x00, 0x00,
   0x47, 0x4c, 0x53, 0x4c, 0x2e, 0x73, 0x74, 0x64, 0x2e, 0x34, 0x35, 0x30,

test/compiled_shaders_include/kompute_test/shaders/shadertest_workgroup.hpp

@@ -24,7 +24,7 @@
 namespace kp {
 namespace shader_data {
 static const unsigned char test_shaders_glsl_test_workgroup_comp_spv[] = {
-  0x03, 0x02, 0x23, 0x07, 0x00, 0x00, 0x01, 0x00, 0x08, 0x00, 0x08, 0x00,
+  0x03, 0x02, 0x23, 0x07, 0x00, 0x00, 0x01, 0x00, 0x0a, 0x00, 0x08, 0x00,
   0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x02, 0x00,
   0x01, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x06, 0x00, 0x01, 0x00, 0x00, 0x00,
   0x47, 0x4c, 0x53, 0x4c, 0x2e, 0x73, 0x74, 0x64, 0x2e, 0x34, 0x35, 0x30,