Merge pull request #89 from EthicalML/enhanced_python_build

Enhanced python build

CMakeLists.txt

@@ -23,6 +23,7 @@ set(KOMPUTE_EXTRA_CXX_FLAGS "" CACHE STRING "Extra compile flags for Kompute, se
 
 if(KOMPUTE_OPT_ENABLE_SPDLOG)
     set(KOMPUTE_EXTRA_CXX_FLAGS "${KOMPUTE_EXTRA_CXX_FLAGS} -DKOMPUTE_ENABLE_SPDLOG=1")
+    set(SPDLOG_INSTALL, 1)
 endif()
 
 if(KOMPUTE_OPT_ANDOID_BUILD)

MANIFEST.in

@@ -0,0 +1,6 @@
+include CMakeLists.txt
+recursive-include src *
+recursive-include python *
+recursive-include single_include *
+recursive-include external *
+

Makefile

@@ -86,7 +86,7 @@ mk_run_tests: mk_build_tests
 VS_BUILD_TYPE ?= "Debug"
 # Run with multiprocessin / parallel build by default
 VS_CMAKE_EXTRA_FLAGS ?= ""
-VS_KOMPUTE_EXTRA_CXX_FLAGS ?= "/MP" # Adding multiprocessing by default. You should add "/MT" for submodule builds for compatibility with gtest
+VS_KOMPUTE_EXTRA_CXX_FLAGS ?= "/MP" # /MP is for faster multiprocessing builds. You should add "/MT" for submodule builds for compatibility with gtest
 VS_INSTALL_PATH ?= "build/src/CMakeFiles/Export/" # Set to "" if prefer default
 
 vs_cmake:

README.md

@@ -179,6 +179,25 @@ static std::string shaderData = "path/to/shader.glsl";
 static std::string shaderData = "path/to/shader.glsl.spv";
 ```
 
+## More examples
+
+### Simple examples
+
+* [Pass shader as raw string](https://kompute.cc/overview/advanced-examples.html#simple-shader-example)
+* [Record batch commands with a Kompute Sequence](https://kompute.cc/overview/advanced-examples.html#record-batch-commands)
+* [Run Asynchronous Operations](https://kompute.cc/overview/advanced-examples.html#asynchronous-operations)
+* [Run Parallel Operations Across Multiple GPU Queues](https://kompute.cc/overview/advanced-examples.html#parallel-operations)
+* [Create your custom Kompute Operations](https://kompute.cc/overview/advanced-examples.html#your-custom-kompute-operation)
+* [Implementing logistic regression from scratch](https://kompute.cc/overview/advanced-examples.html#logistic-regression-example)
+
+### End-to-end examples
+
+* [Machine Learning Logistic Regression Implementation](https://towardsdatascience.com/machine-learning-and-data-processing-in-the-gpu-with-vulkan-kompute-c9350e5e5d3a)
+* [Parallelizing GPU-intensive Workloads via Multi-Queue Operations](https://towardsdatascience.com/parallelizing-heavy-gpu-workloads-via-multi-queue-operations-50a38b15a1dc)
+* [Android NDK Mobile Kompute ML Application](https://towardsdatascience.com/gpu-accelerated-machine-learning-in-your-mobile-applications-using-the-android-ndk-vulkan-kompute-1e9da37b7617)
+* [Game Development Kompute ML in Godot Engine](https://towardsdatascience.com/supercharging-game-development-with-gpu-accelerated-ml-using-vulkan-kompute-the-godot-game-engine-4e75a84ea9f0)
+
+
 ## Architectural Overview
 
 The core architecture of Kompute includes the following:
@@ -259,10 +278,16 @@ You can also access the <a href="https://github.com/EthicalML/vulkan-kompute/tre
 
 Besides the C++ core SDK you can also use the Python package of Kompute, which exposes the same core functionality, and supports interoperability with Python objects like Lists, Numpy Arrays, etc.
 
-You can install from the repository by running:
+The only dependencies are Python 3.5+ and Cmake 3.4.1+. You can install Kompute from the [Python pypi package](https://pypi.org/project/kp/) using the following command.
 
 ```
-pip install .
+pip install kp
+```
+
+You can also install from master branch using:
+
+```
+pip install git+git://github.com/EthicalML/vulkan-kompute.git@master
 ```
 
 For further details you can read the [Python Package documentation](https://kompute.cc/overview/python-package.html) or the [Python Class Reference documentation](https://kompute.cc/overview/python-reference.html).
@@ -329,25 +354,6 @@ assert tensor_out.data() == [2.0, 4.0, 6.0]
 
 For further details you can read the [Python Package documentation](https://kompute.cc/overview/python-package.html) or the [Python Class Reference documentation](https://kompute.cc/overview/python-reference.html).
 
-## More examples
-
-### Simple examples
-
-* [Pass shader as raw string](https://kompute.cc/overview/advanced-examples.html#simple-shader-example)
-* [Record batch commands with a Kompute Sequence](https://kompute.cc/overview/advanced-examples.html#record-batch-commands)
-* [Run Asynchronous Operations](https://kompute.cc/overview/advanced-examples.html#asynchronous-operations)
-* [Run Parallel Operations Across Multiple GPU Queues](https://kompute.cc/overview/advanced-examples.html#parallel-operations)
-* [Create your custom Kompute Operations](https://kompute.cc/overview/advanced-examples.html#your-custom-kompute-operation)
-* [Implementing logistic regression from scratch](https://kompute.cc/overview/advanced-examples.html#logistic-regression-example)
-
-### End-to-end examples
-
-* [Machine Learning Logistic Regression Implementation](https://towardsdatascience.com/machine-learning-and-data-processing-in-the-gpu-with-vulkan-kompute-c9350e5e5d3a)
-* [Parallelizing GPU-intensive Workloads via Multi-Queue Operations](https://towardsdatascience.com/parallelizing-heavy-gpu-workloads-via-multi-queue-operations-50a38b15a1dc)
-* [Android NDK Mobile Kompute ML Application](https://towardsdatascience.com/gpu-accelerated-machine-learning-in-your-mobile-applications-using-the-android-ndk-vulkan-kompute-1e9da37b7617)
-* [Game Development Kompute ML in Godot Engine](https://towardsdatascience.com/supercharging-game-development-with-gpu-accelerated-ml-using-vulkan-kompute-the-godot-game-engine-4e75a84ea9f0)
-
-
 ## Build Overview
 
 The build system provided uses `cmake`, which allows for cross platform builds.

docs/overview/advanced-examples.rst

@@ -23,7 +23,7 @@ End-to-end examples
 
 
 * `Machine Learning Logistic Regression Implementation <https://towardsdatascience.com/machine-learning-and-data-processing-in-the-gpu-with-vulkan-kompute-c9350e5e5d3a>`_
-* `Parallelizing GPU-intensive Workloads via Multi-Queue Operations [https://towardsdatascience.com/parallelizing-heavy-gpu-workloads-via-multi-queue-operations-50a38b15a1dc>`_
+* `Parallelizing GPU-intensive Workloads via Multi-Queue Operations <https://towardsdatascience.com/parallelizing-heavy-gpu-workloads-via-multi-queue-operations-50a38b15a1dc>`_
 * `Android NDK Mobile Kompute ML Application <https://towardsdatascience.com/gpu-accelerated-machine-learning-in-your-mobile-applications-using-the-android-ndk-vulkan-kompute-1e9da37b7617>`_
 * `Game Development Kompute ML in Godot Engine <https://towardsdatascience.com/supercharging-game-development-with-gpu-accelerated-ml-using-vulkan-kompute-the-godot-game-engine-4e75a84ea9f0>`_
 

docs/overview/python-package.rst

@@ -84,8 +84,22 @@ Package Installation
 The package can be installed through the top level `setup.py` by running:
 
 ```
-pip install .
+pip install kp
+```
+
+You can also install from master branch using:
+
+```
+pip install git+git://github.com/EthicalML/vulkan-kompute.git@master
+```
+
+You can configure log level with the function `kp.log_level` as outlined below.
+
+The values are TRACE=0, DEBUG=1, INFO=2, WARN=3, ERROR=4. Kompute defaults to INFO.
+
+```
+import kp
+kp.log_level(1)
 ```
 
 
-

python/src/main.cpp

@@ -7,6 +7,20 @@ namespace py = pybind11;
 
 PYBIND11_MODULE(kp, m) {
 
+#if KOMPUTE_ENABLE_SPDLOG
+    spdlog::set_level(
+      static_cast<spdlog::level::level_enum>(SPDLOG_ACTIVE_LEVEL));
+#endif
+
+    m.def("log_level", [](uint8_t logLevel) {
+#if KOMPUTE_ENABLE_SPDLOG
+            spdlog::set_level(
+              static_cast<spdlog::level::level_enum>(SPDLOG_LEVEL_INFO));
+#else
+            SPDLOG_WARN("SPDLOG not enabled so log level config function not supported");
+#endif
+        });
+
     py::enum_<kp::Tensor::TensorTypes>(m, "TensorTypes", "Enum with GPU memory types for Tensor.")
         .value("device", kp::Tensor::TensorTypes::eDevice, "Tensor holding data in GPU memory.")
         .value("staging", kp::Tensor::TensorTypes::eStaging, "Tensor used for transfer of data to device.")

setup.py

@@ -8,6 +8,9 @@ from setuptools import setup, Extension
 from setuptools.command.build_ext import build_ext
 from distutils.version import LooseVersion
 
+curr_dir = os.path.abspath(os.path.dirname(__file__))
+with open(os.path.join(curr_dir, 'README.md'), encoding='utf-8') as f:
+    long_description = f.read()
 
 class CMakeExtension(Extension):
     def __init__(self, name, sourcedir=''):
@@ -23,10 +26,9 @@ class CMakeBuild(build_ext):
             raise RuntimeError("CMake must be installed to build the following extensions: " +
                                ", ".join(e.name for e in self.extensions))
 
-        if platform.system() == "Windows":
-            cmake_version = LooseVersion(re.search(r'version\s*([\d.]+)', out.decode()).group(1))
-            if cmake_version < '3.1.0':
-                raise RuntimeError("CMake >= 3.1.0 is required on Windows")
+        cmake_version = LooseVersion(re.search(r'version\s*([\d.]+)', out.decode()).group(1))
+        if cmake_version < '3.4.1':
+            raise RuntimeError("CMake >= 3.4.1 is required")
 
         for ext in self.extensions:
             self.build_extension(ext)
@@ -39,13 +41,15 @@ class CMakeBuild(build_ext):
 
         cmake_args = ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir,
                       '-DKOMPUTE_OPT_BUILD_PYTHON=1',
-                      '-DKOMPUTE_OPT_BUILD_SINGLE_HEADER=1',
+                      '-DKOMPUTE_OPT_ENABLE_SPDLOG=1',
+                      '-DKOMPUTE_OPT_REPO_SUBMODULE_BUILD=1',
                       '-DPYTHON_EXECUTABLE=' + sys.executable]
 
         cfg = 'Debug' if self.debug else 'Release'
         build_args = ['--config', cfg]
 
         if platform.system() == "Windows":
+            cmake_args += ['-DKOMPUTE_EXTRA_CXX_FLAGS=""']
             cmake_args += ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}'.format(cfg.upper(), extdir)]
             if sys.maxsize > 2**32:
                 cmake_args += ['-A', 'x64']
@@ -66,11 +70,13 @@ class CMakeBuild(build_ext):
 
 setup(
     name='kp',
-    version='0.0.1',
+    version='0.4.1',
     author='Alejandro Saucedo',
     description='Vulkan Kompute: Blazing fast, mobile-enabled, asynchronous, and optimized for advanced GPU processing usecases.',
-    long_description='',
+    long_description=long_description,
+    long_description_content_type='text/markdown',
     ext_modules=[CMakeExtension('kp')],
     cmdclass=dict(build_ext=CMakeBuild),
     zip_safe=False,
+    include_package_data=True,
 )

single_include/kompute/Kompute.hpp

@@ -24,7 +24,7 @@ static const char* KOMPUTE_LOG_TAG = "KomputeLog";
 #endif // KOMPUTE_VK_API_VERSION
 
 // SPDLOG_ACTIVE_LEVEL must be defined before spdlog.h import
-#if !defined(SPDLOG_ACTIVE_LEVEL)
+#ifndef SPDLOG_ACTIVE_LEVEL
 #if DEBUG
 #define SPDLOG_ACTIVE_LEVEL SPDLOG_LEVEL_DEBUG
 #else

src/Sequence.cpp

@@ -244,6 +244,11 @@ Sequence::freeMemoryDestroyGPUResources()
         SPDLOG_DEBUG("Kompute Sequence Destroyed CommandPool");
     }
 
+    if (this->mOperations.size()) {
+        SPDLOG_INFO("Kompute Sequence clearing operations buffer");
+        this->mOperations.clear();
+    }
+
     this->mIsInit = false;
 
 }

src/include/kompute/Core.hpp

@@ -24,7 +24,7 @@ static const char* KOMPUTE_LOG_TAG = "KomputeLog";
 #endif // KOMPUTE_VK_API_VERSION
 
 // SPDLOG_ACTIVE_LEVEL must be defined before spdlog.h import
-#if !defined(SPDLOG_ACTIVE_LEVEL)
+#ifndef SPDLOG_ACTIVE_LEVEL
 #if DEBUG
 #define SPDLOG_ACTIVE_LEVEL SPDLOG_LEVEL_DEBUG
 #else