From ca28898b8473355beb9e0f0932efbc89e186c1ef Mon Sep 17 00:00:00 2001 From: TheK0tYaRa Date: Mon, 16 Feb 2026 00:40:38 +0200 Subject: [PATCH] I must fuck the tree https://www.youtube.com/watch?v=fZ1DGCtwL9k --- custom/pkgs/by-category/default.nix | 37 +- .../as/ascii_colors/default.nix | 54 --- .../li/lightrag-hku/default.nix | 83 ----- .../na/nano-vectordb/default.nix | 53 --- .../pythonPackages/pi/pipmaster/default.nix | 54 --- .../pythonPackages/python-packages.nix | 342 ++++++++++++++++++ .../pythonPackages/ra/raganything/default.nix | 56 --- custom/pkgs/lib/parse-github-url.nix | 42 +++ flake.lock | 6 +- 9 files changed, 414 insertions(+), 313 deletions(-) delete mode 100644 custom/pkgs/by-category/pythonPackages/as/ascii_colors/default.nix delete mode 100644 custom/pkgs/by-category/pythonPackages/li/lightrag-hku/default.nix delete mode 100644 custom/pkgs/by-category/pythonPackages/na/nano-vectordb/default.nix delete mode 100644 custom/pkgs/by-category/pythonPackages/pi/pipmaster/default.nix create mode 100644 custom/pkgs/by-category/pythonPackages/python-packages.nix delete mode 100644 custom/pkgs/by-category/pythonPackages/ra/raganything/default.nix create mode 100644 custom/pkgs/lib/parse-github-url.nix diff --git a/custom/pkgs/by-category/default.nix b/custom/pkgs/by-category/default.nix index 664063b..24da888 100644 --- a/custom/pkgs/by-category/default.nix +++ b/custom/pkgs/by-category/default.nix @@ -5,21 +5,38 @@ packagesFromTree, }: let - categoryFolders = { - pythonPackages = ./pythonPackages; - }; + pythonPackagesDir = ./pythonPackages; - pyExts = - if categoryFolders ? pythonPackages && builtins.pathExists categoryFolders.pythonPackages then + treeExts = + if builtins.pathExists pythonPackagesDir then [ - (pyFinal: _pyPrev: packagesFromTree categoryFolders.pythonPackages pyFinal.callPackage) + ( + python-final: python-prev: + removeAttrs (packagesFromTree pythonPackagesDir python-final.callPackage) [ + "python-packages" + ] + ) + ] + else + [ ]; + + fileExts = + if builtins.pathExists (pythonPackagesDir + "/python-packages.nix") then + [ + ( + python-final: python-prev: + prev.lib.customisation.callPackagesWith ( + python-prev + // { + lib = prev.lib; + fetchFromGitHub = prev.fetchFromGitHub; + } + ) (pythonPackagesDir + "/python-packages.nix") { } + ) ] else [ ]; in { - pythonPackagesExtensions = (prev.pythonPackagesExtensions or [ ]) ++ pyExts; - - # optional: expose mapping for debugging - # inherit categoryFolders; + pythonPackagesExtensions = (prev.pythonPackagesExtensions or [ ]) ++ treeExts ++ fileExts; } diff --git a/custom/pkgs/by-category/pythonPackages/as/ascii_colors/default.nix b/custom/pkgs/by-category/pythonPackages/as/ascii_colors/default.nix deleted file mode 100644 index 2150e23..0000000 --- a/custom/pkgs/by-category/pythonPackages/as/ascii_colors/default.nix +++ /dev/null @@ -1,54 +0,0 @@ -# derivation.nix -{ - lib, - buildPythonPackage, - fetchFromGitHub, - pythonOlder, - setuptools, - wheel, - - python, - - wcwidth, -}: - -buildPythonPackage rec { - pname = "ascii_colors"; - version = "0.11.11"; - - src = fetchFromGitHub { - owner = "ParisNeo"; - repo = "ascii_colors"; - rev = "0011470"; - hash = "sha256-jiolJzsBQMp9/DHdHTixJGvXoS19vAp/S7+8PyLARoo="; - }; - - disabled = pythonOlder "3.8"; - format = "pyproject"; - - nativeBuildInputs = [ - setuptools - wheel - ]; - - propagatedBuildInputs = [ - wcwidth - ]; - - doCheck = false; - - doInstallCheck = true; - pythonImportsCheck = [ "ascii_colors" ]; - - installCheckPhase = '' - runHook preInstallCheck - ${python.interpreter} -c "import ascii_colors; print(getattr(ascii_colors,'__version__','unknown'))" - runHook postInstallCheck - ''; - - meta = with lib; { - description = "A Python library for rich terminal output with advanced logging features."; - homepage = "https://github.com/ParisNeo/ascii_colors"; - license = licenses.asl20; - }; -} diff --git a/custom/pkgs/by-category/pythonPackages/li/lightrag-hku/default.nix b/custom/pkgs/by-category/pythonPackages/li/lightrag-hku/default.nix deleted file mode 100644 index 9550204..0000000 --- a/custom/pkgs/by-category/pythonPackages/li/lightrag-hku/default.nix +++ /dev/null @@ -1,83 +0,0 @@ -{ - lib, - buildPythonPackage, - fetchFromGitHub, - pythonOlder, - setuptools, - wheel, - - python, - - aiohttp, - configparser, - google-api-core, - google-genai, - json-repair, - nano-vectordb, - networkx, - numpy, - pandas, - pipmaster, - pydantic, - pypinyin, - python-dotenv, - tenacity, - tiktoken, - xlsxwriter, -}: - -buildPythonPackage rec { - pname = "lightrag-hku"; - version = "1.4.9.11"; - - src = fetchFromGitHub { - owner = "HKUDS"; - repo = "LightRAG"; - rev = "v${version}"; - hash = "sha256-TX/HSF2ZqoDo2SRlFzM+bkUxQXiUpnrl6kwI+lljjdo="; - }; - - disabled = pythonOlder "3.10"; - format = "pyproject"; - - nativeBuildInputs = [ - setuptools - wheel - ]; - - propagatedBuildInputs = [ - aiohttp - configparser - google-api-core - google-genai - json-repair - nano-vectordb - networkx - numpy - pandas - pipmaster - pydantic - pypinyin - python-dotenv - tenacity - tiktoken - xlsxwriter - ]; - - doCheck = false; - - doInstallCheck = true; - pythonImportsCheck = [ "lightrag" ]; - - installCheckPhase = '' - runHook preInstallCheck - ${python.interpreter} -c "import lightrag; print(getattr(lightrag,'__version__','unknown'))" - runHook postInstallCheck - ''; - - meta = with lib; { - description = "LightRAG: Simple and Fast Retrieval-Augmented Generation"; - homepage = "https://github.com/HKUDS/LightRAG"; - license = licenses.mit; - }; -} diff --git a/custom/pkgs/by-category/pythonPackages/na/nano-vectordb/default.nix b/custom/pkgs/by-category/pythonPackages/na/nano-vectordb/default.nix deleted file mode 100644 index b3293d9..0000000 --- a/custom/pkgs/by-category/pythonPackages/na/nano-vectordb/default.nix +++ /dev/null @@ -1,53 +0,0 @@ -{ - lib, - buildPythonPackage, - fetchFromGitHub, - pythonOlder, - setuptools, - wheel, - - python, - - numpy, -}: - -buildPythonPackage rec { - pname = "nano-vectordb"; - version = "0.0.4.3"; - - src = fetchFromGitHub { - owner = "gusye1234"; - repo = "nano-vectordb"; - rev = "03f8213"; - hash = "sha256-jLco+1TAncF8Ep+VGd3DhsoiZTW7G/H8fAEwnNsovTY="; - }; - - disabled = pythonOlder "3.9"; - format = "setuptools"; - - nativeBuildInputs = [ - setuptools - wheel - ]; - - propagatedBuildInputs = [ - numpy - ]; - - doCheck = false; - - doInstallCheck = true; - pythonImportsCheck = [ "nano_vectordb" ]; - - installCheckPhase = '' - runHook preInstallCheck - ${python.interpreter} -c "import nano_vectordb; print(getattr(nano_vectordb,'__version__','unknown'))" - runHook postInstallCheck - ''; - - meta = with lib; { - description = "A simple, easy-to-hack Vector Database implementation"; - homepage = "https://github.com/gusye1234/nano-vectordb"; - license = licenses.mit; - }; -} diff --git a/custom/pkgs/by-category/pythonPackages/pi/pipmaster/default.nix b/custom/pkgs/by-category/pythonPackages/pi/pipmaster/default.nix deleted file mode 100644 index ef3fcbf..0000000 --- a/custom/pkgs/by-category/pythonPackages/pi/pipmaster/default.nix +++ /dev/null @@ -1,54 +0,0 @@ -# derivation.nix -{ - lib, - buildPythonPackage, - fetchFromGitHub, - pythonOlder, - setuptools, - - python, - - packaging, - ascii_colors, -}: - -buildPythonPackage rec { - pname = "pipmaster"; - version = "1.1.0"; - - src = fetchFromGitHub { - owner = "ParisNeo"; - repo = "pipmaster"; - rev = "e2f4d55"; - hash = "sha256-IJ6VG4WvoEm584vbZDLTtRcEZekZh7HJto1hnABoodo="; - }; - - disabled = pythonOlder "3.8"; - format = "pyproject"; - - nativeBuildInputs = [ - setuptools - ]; - - propagatedBuildInputs = [ - packaging - ascii_colors - ]; - - doCheck = false; - - doInstallCheck = true; - pythonImportsCheck = [ "pipmaster" ]; - - installCheckPhase = '' - runHook preInstallCheck - ${python.interpreter} -c "import pipmaster; print(getattr(pipmaster,'__version__','unknown'))" - runHook postInstallCheck - ''; - - meta = with lib; { - description = "A versatile Python package manager utility for simplifying package installation, updates, checks, and environment management."; - homepage = "https://github.com/ParisNeo/pipmaster"; - license = licenses.asl20; - }; -} diff --git a/custom/pkgs/by-category/pythonPackages/python-packages.nix b/custom/pkgs/by-category/pythonPackages/python-packages.nix new file mode 100644 index 0000000..e79aeef --- /dev/null +++ b/custom/pkgs/by-category/pythonPackages/python-packages.nix @@ -0,0 +1,342 @@ +{ + lib, + fetchFromGitHub, + buildPythonPackage, + setuptools, + wheel, + hatchling, + poetry-core, + + python, + + huggingface-hub, + tqdm, + + aiohttp, + configparser, + google-api-core, + google-genai, + json-repair, + networkx, + numpy, + pandas, + pydantic, + pypinyin, + python-dotenv, + tenacity, + tiktoken, + xlsxwriter, + + boto3, + click, + loguru, + pdfminer-six, + requests, + httpx, + pillow, + pypdfium2, + pypdf, + reportlab, + modelscope, + opencv-python, + scikit-image, + openai, + beautifulsoup4, + magika, + + av, + + pdm-backend, + fasttext-predict, + + pydantic-settings, + + aiofiles, + hatch-fancy-pypi-readme, + + colorlog, + torch, + torchvision, + + wcwidth, +}: + +let + prefab = + { + url, + hash, + force ? { }, + meta ? { }, + propagatedBuildDeps ? [ ], + nativeBuildDeps ? [ ], + pythonRelaxDeps ? [ ], + pythonImportsCheck ? null, + }: + let + x = (import ../../lib/parse-github-url.nix { inherit lib; }) url; + pname = x.pname; + rev = force.rev or x.rev; + version = force.version or x.version; + + defaultImports = [ (lib.toLower (lib.replaceStrings [ "-" ] [ "_" ] pname)) ]; + imports = if pythonImportsCheck == null then defaultImports else pythonImportsCheck; + + rawSubdir = force.subdir or x.subdir or null; + subdir = if rawSubdir == null then null else lib.removeSuffix "/" (lib.removePrefix "/" rawSubdir); + + src = fetchFromGitHub { + owner = x.owner; + repo = x.repo; + rev = rev; + hash = hash; + }; + in + buildPythonPackage ( + { + inherit pname version src; + pyproject = true; + + build-system = [ + setuptools + wheel + ] + ++ nativeBuildDeps; + dependencies = propagatedBuildDeps; + pythonRelaxDeps = pythonRelaxDeps; + + pythonImportsCheck = imports; + + installCheckPhase = '' + runHook preInstallCheck + ${builtins.concatStringsSep "\n" ( + map (lib: '' + ${python.interpreter} -c "import ${lib}; print(getattr(${lib},'__version__','unknown'))" + '') imports + )} + runHook postInstallCheck + ''; + + meta = { + homepage = x.homepage; + } + // meta; + } + // lib.optionalAttrs (subdir != null && subdir != "") { + sourceRoot = "${src.name}/${subdir}"; + } + ); +in +rec { + raganything = prefab { + url = "https://github.com/HKUDS/RAG-Anything/tree/v1.2.9"; + hash = "sha256-yepiLYzPD6UcJRbAbovg/BwFE8nh903o/tHypiHGKSw="; + propagatedBuildDeps = [ + huggingface-hub + lightrag-hku + mineru + tqdm + ]; + pythonImportsCheck = [ "raganything" ]; + meta = { + description = "RAGAnything: All-in-One RAG System"; + license = lib.licenses.mit; + }; + }; + lightrag-hku = prefab { + url = "https://github.com/HKUDS/LightRAG/tree/v1.4.9.11"; + hash = "sha256-TX/HSF2ZqoDo2SRlFzM+bkUxQXiUpnrl6kwI+lljjdo="; + propagatedBuildDeps = [ + aiohttp + configparser + google-api-core + google-genai + json-repair + nano-vectordb + networkx + numpy + pandas + pipmaster + pydantic + pypinyin + python-dotenv + tenacity + tiktoken + xlsxwriter + ]; + meta = { + description = "LightRAG: Simple and Fast Retrieval-Augmented Generation"; + license = lib.licenses.mit; + }; + }; + pipmaster = prefab { + url = "https://github.com/ParisNeo/pipmaster/tree/820acdbc9d541443942bd8afd6ba968036bd8452"; + hash = "sha256-IJ6VG4WvoEm584vbZDLTtRcEZekZh7HJto1hnABoodo="; + force.version = "1.1.0"; + propagatedBuildDeps = [ + ascii-colors + ]; + meta = with lib; { + description = "A versatile Python package manager utility for simplifying package installation, updates, checks, and environment management."; + license = licenses.asl20; + }; + }; + nano-vectordb = prefab { + url = "https://github.com/gusye1234/nano-vectordb/tree/03f821348f04a93a9c36bb575faae05b61f4c02e"; + hash = "sha256-jLco+1TAncF8Ep+VGd3DhsoiZTW7G/H8fAEwnNsovTY="; + force.version = "0.0.4.3"; + propagatedBuildDeps = [ + numpy + ]; + meta = { + description = "A simple, easy-to-hack Vector Database implementation"; + license = lib.licenses.mit; + }; + }; + ascii-colors = prefab { + url = "https://github.com/ParisNeo/ascii_colors/tree/817a21485136635e3da89ff08444183254b25aab"; + hash = "sha256-jiolJzsBQMp9/DHdHTixJGvXoS19vAp/S7+8PyLARoo="; + force.version = "0.11.12"; + propagatedBuildDeps = [ + wcwidth + ]; + meta = { + description = "A python library for displaying stuff on the console in a pretty way"; + license = lib.licenses.asl20; + }; + }; + mineru = prefab { + url = "https://github.com/opendatalab/MinerU/tree/mineru-2.7.6-released"; + hash = "sha256-A/nFNQYGEfmWUdpX8N1lbex3mdiF8+eN8s0UKQFc42E="; + propagatedBuildDeps = [ + boto3 + click + loguru + numpy + pdfminer-six + tqdm + requests + httpx + pillow + pypdfium2 + pypdf + reportlab + pdftext + modelscope + huggingface-hub + json-repair + opencv-python + fast-langdetect + scikit-image + openai + beautifulsoup4 + magika + mineru-vl-utils + qwen-vl-utils + ]; + pythonRelaxDeps = [ + "fast-langdetect" + ]; + meta = { + description = "Transforms complex documents like PDFs into LLM-ready markdown/JSON for your Agentic workflows."; + license = lib.licenses.gpl3; + }; + }; + qwen-vl-utils = prefab { + url = "https://github.com/QwenLM/Qwen3-VL/tree/main/qwen-vl-utils"; + hash = "sha256-Vha/Tc4q2v5RCL31hB9U4ZrfIFxfZjgwM6PYlvDeoAQ="; + force = { + version = "0.0.14"; + rev = "fe12058"; + }; + propagatedBuildDeps = [ + hatchling + av + pillow + requests + torch + torchvision + ]; + pythonImportsCheck = [ "qwen_vl_utils" ]; + meta = { + description = "Qwen-VL Utils contains a set of helper functions for processing and integrating visual language information with Qwen-VL Series Model."; + license = lib.licenses.asl20; + }; + }; + mineru-vl-utils = prefab { + url = "https://github.com/opendatalab/mineru-vl-utils/tree/mineru_vl_utils-0.1.22-released"; + hash = "sha256-hpTW/1nwXPxfld4nx0XHZBMerWj+UL1vzDhYpwjezRU="; + propagatedBuildDeps = [ + httpx + httpx-retries + aiofiles + pillow + pydantic + loguru + ]; + meta = { + description = "A Python package for interacting with the MinerU Vision-Language Model."; + license = lib.licenses.gpl3; + }; + }; + httpx-retries = prefab { + url = "https://github.com/will-ockmore/httpx-retries/tree/0.4.5"; + hash = "sha256-zJ3ExSEWxlHFluSdYA8/XZ3zb4KBelU+IOFyUu4ezvo="; + propagatedBuildDeps = [ + hatchling + hatch-fancy-pypi-readme + + httpx + ]; + meta = { + description = "A retry layer for HTTPX."; + license = lib.licenses.mit; + }; + }; + fast-langdetect = prefab { + url = "https://github.com/LlmKira/fast-langdetect/tree/pypi_1.0.0"; + hash = "sha256-pj46gHG9cjkSjnYc88bSctL/1LAUe0jkBuM/GZWMsUI="; + propagatedBuildDeps = [ + pdm-backend + robust-downloader + requests + fasttext-predict + ]; + meta = { + description = "⚡️ 80x faster Fasttext language detection out of the box | Split text by language "; + license = lib.licenses.mit; + }; + }; + robust-downloader = prefab { + url = "https://github.com/fedebotu/robust-downloader/tree/0.0.2"; + hash = "sha256-UmzfEIPiMtUkOG6sIMYgLxc8YwL5wRgMBRlywqKomv0="; + propagatedBuildDeps = [ + tqdm + colorlog + requests + ]; + meta = { + description = "Minimal Python downloader with robustness in mind - resumable downloads, retries, and more"; + license = lib.licenses.asl20; + }; + }; + pdftext = prefab { + url = "https://github.com/datalab-to/pdftext/tree/v0.6.3"; + hash = "sha256-EGVjzjDWtdcEPX//cOm5+xm9FvX0aP+h6fsD25hC8gA="; + propagatedBuildDeps = [ + poetry-core + + click + pydantic + pydantic-settings + pypdfium2 + ]; + pythonRelaxDeps = [ + "pypdfium2" + ]; + meta = { + description = "Extract structured text from pdfs quickly"; + license = lib.licenses.asl20; + }; + }; +} diff --git a/custom/pkgs/by-category/pythonPackages/ra/raganything/default.nix b/custom/pkgs/by-category/pythonPackages/ra/raganything/default.nix deleted file mode 100644 index 587d9de..0000000 --- a/custom/pkgs/by-category/pythonPackages/ra/raganything/default.nix +++ /dev/null @@ -1,56 +0,0 @@ -{ - lib, - buildPythonPackage, - fetchFromGitHub, - pythonOlder, - setuptools, - wheel, - - python, - - python-dotenv, - lightrag-hku, - tqdm, -}: - -buildPythonPackage rec { - pname = "raganything"; - version = "1.2.9"; - - src = fetchFromGitHub { - owner = "HKUDS"; - repo = "RAG-Anything"; - rev = "v${version}"; - hash = "sha256-yepiLYzPD6UcJRbAbovg/BwFE8nh903o/tHypiHGKSw="; - }; - - disabled = pythonOlder "3.10"; - format = "setuptools"; - - nativeBuildInputs = [ - setuptools - wheel - ]; - - propagatedBuildInputs = [ - python-dotenv - lightrag-hku - tqdm - ]; - - doInstallCheck = true; - pythonImportsCheck = [ "raganything" ]; - - installCheckPhase = '' - runHook preInstallCheck - ${python.interpreter} -c "import raganything; print(getattr(raganything,'__version__','unknown'))" - runHook postInstallCheck - ''; - - meta = with lib; { - description = "RAGAnything: All-in-One RAG System"; - homepage = "https://github.com/HKUDS/RAG-Anything"; - license = licenses.mit; - maintainer = [ ]; - }; -} diff --git a/custom/pkgs/lib/parse-github-url.nix b/custom/pkgs/lib/parse-github-url.nix new file mode 100644 index 0000000..7e3ef71 --- /dev/null +++ b/custom/pkgs/lib/parse-github-url.nix @@ -0,0 +1,42 @@ +{ lib }: + +url: +let + # убрать ?query/#fragment и финальный / + u0 = builtins.head (builtins.split "[?#]" url); + u = lib.strings.removeSuffix "/" u0; + + # owner / repo / (tree|tag|commit) / rev / (optional / subdir...) + m = builtins.match "https://github.com/([^/]+)/([^/]+)(/(tree|tag|commit)/([^/]+)(/(.*))?)?" u; +in +assert m != null; +let + owner = builtins.elemAt m 0; + repo = builtins.elemAt m 1; + + kind = builtins.elemAt m 3; # null если нет /tree|tag|commit + rev = builtins.elemAt m 4; # null если нет + + subdir = builtins.elemAt m 6; # null или "qwen-vl-utils" или "a/b/c" +in +{ + inherit + owner + repo + kind + rev + subdir + ; + + version = + if rev == null then + null + else + let + vm = builtins.match "^.*([0-9]+(\\.[0-9]+)+).*$" rev; + in + if vm == null then null else builtins.elemAt vm 0; + + pname = repo; + homepage = "https://github.com/${owner}/${repo}"; +} diff --git a/flake.lock b/flake.lock index c40a35f..99980d5 100644 --- a/flake.lock +++ b/flake.lock @@ -385,11 +385,11 @@ ] }, "locked": { - "lastModified": 1771132481, - "narHash": "sha256-Tc+YqZ/Q1K35vJK4ji4RbLB/qKGcEq6yh7p4CKoZF60=", + "lastModified": 1771188132, + "narHash": "sha256-qLXxN/tPrZtnekaLBQuVtxQfvqqs5cT5WbyH4zZaTGI=", "owner": "nix-community", "repo": "home-manager", - "rev": "1e53254671f36cb7d0e2dcca08730f066d5e69b4", + "rev": "ae8003d8b61d0d373e7ca3da1a48f9c870d15df9", "type": "github" }, "original": {