From 4223c18b84c0bc99985be86bec69edb4123a205a Mon Sep 17 00:00:00 2001 From: TheK0tYaRa Date: Tue, 17 Feb 2026 14:33:50 +0200 Subject: [PATCH] moving to a cleaner python package registry, 1 --- .../pythonPackages/prefabs-specs.nix | 297 ++++++++++++++++++ .../pythonPackages/python-packages.nix | 33 +- custom/pkgs/lib/parse-github-url.nix | 8 +- custom/pkgs/lib/parse-source-ref.nix | 75 +++++ custom/pkgs/lib/prefab.nix | 115 +++++++ 5 files changed, 521 insertions(+), 7 deletions(-) create mode 100644 custom/pkgs/by-category/pythonPackages/prefabs-specs.nix create mode 100644 custom/pkgs/lib/parse-source-ref.nix create mode 100644 custom/pkgs/lib/prefab.nix diff --git a/custom/pkgs/by-category/pythonPackages/prefabs-specs.nix b/custom/pkgs/by-category/pythonPackages/prefabs-specs.nix new file mode 100644 index 0000000..2e973d7 --- /dev/null +++ b/custom/pkgs/by-category/pythonPackages/prefabs-specs.nix @@ -0,0 +1,297 @@ +{ + lib, + hatchling, + poetry-core, + + huggingface-hub, + tqdm, + + aiohttp, + configparser, + google-api-core, + google-genai, + json-repair, + networkx, + numpy, + pandas, + pydantic, + pypinyin, + python-dotenv, + tenacity, + tiktoken, + xlsxwriter, + + boto3, + click, + loguru, + pdfminer-six, + requests, + httpx, + pillow, + pypdfium2, + pypdf, + reportlab, + modelscope, + opencv-python, + scikit-image, + openai, + beautifulsoup4, + magika, + + av, + + pdm-backend, + fasttext-predict, + + pydantic-settings, + + aiofiles, + hatch-fancy-pypi-readme, + + colorlog, + torch, + torchvision, + + wcwidth, +}: + +rec { + raganything = { + url = "https://github.com/HKUDS/RAG-Anything/tree/v1.2.9"; + hash = "sha256-yepiLYzPD6UcJRbAbovg/BwFE8nh903o/tHypiHGKSw="; + propagatedBuildDeps = [ + huggingface-hub + lightrag-hku + tqdm + # mineru + ] + ++ mineru.optional-dependencies.core; + pythonImportsCheck = [ "raganything" ]; + meta = { + description = "RAGAnything: All-in-One RAG System"; + license = lib.licenses.mit; + }; + }; + lightrag-hku = { + url = "https://github.com/HKUDS/LightRAG/tree/v1.4.9.11"; + hash = "sha256-TX/HSF2ZqoDo2SRlFzM+bkUxQXiUpnrl6kwI+lljjdo="; + propagatedBuildDeps = [ + aiohttp + configparser + google-api-core + google-genai + json-repair + nano-vectordb + networkx + numpy + pandas + pipmaster + pydantic + pypinyin + python-dotenv + tenacity + tiktoken + xlsxwriter + ]; + meta = { + description = "LightRAG: Simple and Fast Retrieval-Augmented Generation"; + license = lib.licenses.mit; + }; + }; + pipmaster = { + url = "https://github.com/ParisNeo/pipmaster/tree/820acdbc9d541443942bd8afd6ba968036bd8452"; + hash = "sha256-IJ6VG4WvoEm584vbZDLTtRcEZekZh7HJto1hnABoodo="; + force.version = "1.1.0"; + propagatedBuildDeps = [ + ascii-colors + ]; + meta = with lib; { + description = "A versatile Python package manager utility for simplifying package installation, updates, checks, and environment management."; + license = licenses.asl20; + }; + }; + nano-vectordb = { + url = "https://github.com/gusye1234/nano-vectordb/tree/03f821348f04a93a9c36bb575faae05b61f4c02e"; + hash = "sha256-jLco+1TAncF8Ep+VGd3DhsoiZTW7G/H8fAEwnNsovTY="; + force.version = "0.0.4.3"; + propagatedBuildDeps = [ + numpy + ]; + meta = { + description = "A simple, easy-to-hack Vector Database implementation"; + license = lib.licenses.mit; + }; + }; + ascii-colors = { + url = "https://github.com/ParisNeo/ascii_colors/tree/817a21485136635e3da89ff08444183254b25aab"; + hash = "sha256-jiolJzsBQMp9/DHdHTixJGvXoS19vAp/S7+8PyLARoo="; + force.version = "0.11.12"; + propagatedBuildDeps = [ + wcwidth + ]; + meta = { + description = "A python library for displaying stuff on the console in a pretty way"; + license = lib.licenses.asl20; + }; + }; + mineru = { + url = "https://github.com/opendatalab/MinerU/tree/mineru-2.7.6-released"; + hash = "sha256-A/nFNQYGEfmWUdpX8N1lbex3mdiF8+eN8s0UKQFc42E="; + propagatedBuildDeps = [ + boto3 + click + loguru + numpy + pdfminer-six + tqdm + requests + httpx + pillow + pypdfium2 + pypdf + reportlab + pdftext + modelscope + huggingface-hub + json-repair + opencv-python + fast-langdetect + scikit-image + openai + beautifulsoup4 + magika + mineru-vl-utils + qwen-vl-utils + ]; + optional-dependencies = { + core = [ doclayout-yolo ] + # ++ (with mineru.optional-dependencies; [ + # vlm + # pipeline + # api + # gradio + # ]) + ; + vlm = [ ]; + pipeline = [ + doclayout-yolo + ]; + api = [ ]; + gradio = [ ]; + }; + pythonRelaxDeps = [ + "fast-langdetect" + ]; + meta = { + description = "Transforms complex documents like PDFs into LLM-ready markdown/JSON for your Agentic workflows."; + license = lib.licenses.gpl3; + }; + }; + doclayout-yolo = { + # reason for creating fetcher construct + url = "https://github.com/opendatalab/DocLayout-YOLO/tree/main"; # master is 10 months old and last version was set in Oct 18, 2024 + hash = "sha256-wLCmJuWyw7Gqou7awsHtxqVyr4AN+yJSkF0q3GUSQh8="; + force.version = "0.0.4"; # NOT 0.0.4 on github but fuck it we ball + meta = { + description = "DocLayout-YOLO: Enhancing Document Layout Analysis through Diverse Synthetic Data and Global-to-Local Adaptive Perception"; + license = lib.licenses.agpl3Only; + }; + }; + qwen-vl-utils = { + url = "https://github.com/QwenLM/Qwen3-VL/tree/main/qwen-vl-utils"; + hash = "sha256-Vha/Tc4q2v5RCL31hB9U4ZrfIFxfZjgwM6PYlvDeoAQ="; + force = { + version = "0.0.14"; + rev = "fe12058"; + }; + propagatedBuildDeps = [ + hatchling + av + pillow + requests + torch + torchvision + ]; + pythonImportsCheck = [ "qwen_vl_utils" ]; + meta = { + description = "Qwen-VL Utils contains a set of helper functions for processing and integrating visual language information with Qwen-VL Series Model."; + license = lib.licenses.asl20; + }; + }; + mineru-vl-utils = { + url = "https://github.com/opendatalab/mineru-vl-utils/tree/mineru_vl_utils-0.1.22-released"; + hash = "sha256-hpTW/1nwXPxfld4nx0XHZBMerWj+UL1vzDhYpwjezRU="; + propagatedBuildDeps = [ + httpx + httpx-retries + aiofiles + pillow + pydantic + loguru + ]; + meta = { + description = "A Python package for interacting with the MinerU Vision-Language Model."; + license = lib.licenses.gpl3; + }; + }; + httpx-retries = { + url = "https://github.com/will-ockmore/httpx-retries/tree/0.4.5"; + hash = "sha256-zJ3ExSEWxlHFluSdYA8/XZ3zb4KBelU+IOFyUu4ezvo="; + propagatedBuildDeps = [ + hatchling + hatch-fancy-pypi-readme + + httpx + ]; + meta = { + description = "A retry layer for HTTPX."; + license = lib.licenses.mit; + }; + }; + fast-langdetect = { + url = "https://github.com/LlmKira/fast-langdetect/tree/pypi_1.0.0"; + hash = "sha256-pj46gHG9cjkSjnYc88bSctL/1LAUe0jkBuM/GZWMsUI="; + propagatedBuildDeps = [ + pdm-backend + robust-downloader + requests + fasttext-predict + ]; + meta = { + description = "⚡️ 80x faster Fasttext language detection out of the box | Split text by language "; + license = lib.licenses.mit; + }; + }; + robust-downloader = { + url = "https://github.com/fedebotu/robust-downloader/tree/0.0.2"; + hash = "sha256-UmzfEIPiMtUkOG6sIMYgLxc8YwL5wRgMBRlywqKomv0="; + propagatedBuildDeps = [ + tqdm + colorlog + requests + ]; + meta = { + description = "Minimal Python downloader with robustness in mind - resumable downloads, retries, and more"; + license = lib.licenses.asl20; + }; + }; + pdftext = { + url = "https://github.com/datalab-to/pdftext/tree/v0.6.3"; + hash = "sha256-EGVjzjDWtdcEPX//cOm5+xm9FvX0aP+h6fsD25hC8gA="; + propagatedBuildDeps = [ + poetry-core + + click + pydantic + pydantic-settings + pypdfium2 + ]; + pythonRelaxDeps = [ + "pypdfium2" + ]; + meta = { + description = "Extract structured text from pdfs quickly"; + license = lib.licenses.asl20; + }; + }; +} diff --git a/custom/pkgs/by-category/pythonPackages/python-packages.nix b/custom/pkgs/by-category/pythonPackages/python-packages.nix index e79aeef..4aebac9 100644 --- a/custom/pkgs/by-category/pythonPackages/python-packages.nix +++ b/custom/pkgs/by-category/pythonPackages/python-packages.nix @@ -72,6 +72,7 @@ let nativeBuildDeps ? [ ], pythonRelaxDeps ? [ ], pythonImportsCheck ? null, + optional-dependencies ? null, }: let x = (import ../../lib/parse-github-url.nix { inherit lib; }) url; @@ -103,6 +104,7 @@ let ] ++ nativeBuildDeps; dependencies = propagatedBuildDeps; + optional-dependencies = optional-dependencies; pythonRelaxDeps = pythonRelaxDeps; pythonImportsCheck = imports; @@ -134,9 +136,10 @@ rec { propagatedBuildDeps = [ huggingface-hub lightrag-hku - mineru tqdm - ]; + # mineru + ] + ++ mineru.optional-dependencies.core; pythonImportsCheck = [ "raganything" ]; meta = { description = "RAGAnything: All-in-One RAG System"; @@ -234,6 +237,22 @@ rec { mineru-vl-utils qwen-vl-utils ]; + optional-dependencies = { + core = [ doclayout-yolo ] + # ++ (with mineru.optional-dependencies; [ + # vlm + # pipeline + # api + # gradio + # ]) + ; + vlm = [ ]; + pipeline = [ + doclayout-yolo + ]; + api = [ ]; + gradio = [ ]; + }; pythonRelaxDeps = [ "fast-langdetect" ]; @@ -242,6 +261,16 @@ rec { license = lib.licenses.gpl3; }; }; + doclayout-yolo = prefab { + # reason for creating fetcher construct + url = "https://github.com/opendatalab/DocLayout-YOLO/tree/main"; # master is 10 months old and last version was set in Oct 18, 2024 + hash = "sha256-wLCmJuWyw7Gqou7awsHtxqVyr4AN+yJSkF0q3GUSQh8="; + force.version = "0.0.4"; # NOT 0.0.4 on github but fuck it we ball + meta = { + description = "DocLayout-YOLO: Enhancing Document Layout Analysis through Diverse Synthetic Data and Global-to-Local Adaptive Perception"; + license = lib.licenses.agpl3Only; + }; + }; qwen-vl-utils = prefab { url = "https://github.com/QwenLM/Qwen3-VL/tree/main/qwen-vl-utils"; hash = "sha256-Vha/Tc4q2v5RCL31hB9U4ZrfIFxfZjgwM6PYlvDeoAQ="; diff --git a/custom/pkgs/lib/parse-github-url.nix b/custom/pkgs/lib/parse-github-url.nix index 7e3ef71..4645e78 100644 --- a/custom/pkgs/lib/parse-github-url.nix +++ b/custom/pkgs/lib/parse-github-url.nix @@ -2,11 +2,9 @@ url: let - # убрать ?query/#fragment и финальный / u0 = builtins.head (builtins.split "[?#]" url); u = lib.strings.removeSuffix "/" u0; - # owner / repo / (tree|tag|commit) / rev / (optional / subdir...) m = builtins.match "https://github.com/([^/]+)/([^/]+)(/(tree|tag|commit)/([^/]+)(/(.*))?)?" u; in assert m != null; @@ -14,10 +12,10 @@ let owner = builtins.elemAt m 0; repo = builtins.elemAt m 1; - kind = builtins.elemAt m 3; # null если нет /tree|tag|commit - rev = builtins.elemAt m 4; # null если нет + kind = builtins.elemAt m 3; + rev = builtins.elemAt m 4; - subdir = builtins.elemAt m 6; # null или "qwen-vl-utils" или "a/b/c" + subdir = builtins.elemAt m 6; in { inherit diff --git a/custom/pkgs/lib/parse-source-ref.nix b/custom/pkgs/lib/parse-source-ref.nix new file mode 100644 index 0000000..0f00be8 --- /dev/null +++ b/custom/pkgs/lib/parse-source-ref.nix @@ -0,0 +1,75 @@ +{ lib }: + +url: +let + # strip ?query/#fragment and trailing / + u0 = builtins.head (builtins.split "[?#]" url); + u = lib.removeSuffix "/" u0; + + # GitHub: https://github.com/owner/repo/(tree|tag|commit)/rev/(optional/subdir) + gh = builtins.match "https://github.com/([^/]+)/([^/]+)(/(tree|tag|commit)/([^/]+)(/(.*))?)?" u; + + # PyPI project page: https://pypi.org/project/name/version + pypiProject = builtins.match "https://pypi.org/project/([^/]+)(/([^/]+))?" u; + + # PyPI short form: + # pypi:requests@2.32.3 + # pypi:requests==2.32.3 + # pypi:requests/2.32.3 + pypiShort = builtins.match "^pypi:([^@=/]+)(==|@|/)?([^/]+)?$" u; +in +if gh != null then + let + owner = builtins.elemAt gh 0; + repo = builtins.elemAt gh 1; + + kind = builtins.elemAt gh 3; # null if absent + rev = builtins.elemAt gh 4; # null if absent + subdir = builtins.elemAt gh 6; # null or "a/b" + + version = + if rev == null then + null + else + let + vm = builtins.match "^.*([0-9]+(\\.[0-9]+)+).*$" rev; + in + if vm == null then null else builtins.elemAt vm 0; + in + { + type = "github"; + pname = repo; + homepage = "https://github.com/${owner}/${repo}"; + inherit + owner + repo + kind + rev + subdir + version + ; + } +else if pypiShort != null then + let + pname = builtins.elemAt pypiShort 0; + version = builtins.elemAt pypiShort 2; # may be null + in + { + type = "pypi"; + homepage = "https://pypi.org/project/${pname}/"; + inherit pname version; + subdir = null; + } +else if pypiProject != null then + let + pname = builtins.elemAt pypiProject 0; + version = builtins.elemAt pypiProject 2; # may be null + in + { + type = "pypi"; + homepage = "https://pypi.org/project/${pname}/"; + inherit pname version; + subdir = null; + } +else + throw "parse-source-ref: unsupported url: ${url}" diff --git a/custom/pkgs/lib/prefab.nix b/custom/pkgs/lib/prefab.nix new file mode 100644 index 0000000..25b2dc9 --- /dev/null +++ b/custom/pkgs/lib/prefab.nix @@ -0,0 +1,115 @@ +{ + lib, + fetchFromGitHub, + fetchPypi, + buildPythonPackage, + python, + setuptools, + wheel, +}: + +let + parseRef = import ./parse-source-ref.nix { inherit lib; }; + + mkPrefab = + { + url, + hash, + force ? { }, # overrides: pname/version/rev/subdir/sourceRootBase/homepage/src + meta ? { }, + propagatedBuildDeps ? [ ], + nativeBuildDeps ? [ ], + pythonRelaxDeps ? [ ], + pythonImportsCheck ? null, + optional-dependencies ? null, + }: + let + ref = parseRef url; + + pname = force.pname or ref.pname; + + version0 = force.version or ref.version or null; + version = if version0 != null then version0 else "unstable"; + + rawSubdir = force.subdir or ref.subdir or null; + subdir = if rawSubdir == null then null else lib.removeSuffix "/" (lib.removePrefix "/" rawSubdir); + + rev = force.rev or ref.rev or null; + + src = + if (force.src or null) != null then + force.src + else if ref.type == "github" then + ( + assert rev != null; + fetchFromGitHub { + owner = ref.owner; + repo = ref.repo; + inherit rev hash; + } + ) + else if ref.type == "pypi" then + ( + assert (force.version or ref.version or null) != null; + fetchPypi { + inherit pname version; + inherit hash; + } + ) + else + throw "mkPrefab: unsupported ref.type=${ref.type}"; + + sourceRootBase = + force.sourceRootBase or (if ref.type == "github" then src.name else "${pname}-${version}"); + + defaultImports = [ (lib.toLower (lib.replaceStrings [ "-" ] [ "_" ] pname)) ]; + imports = if pythonImportsCheck == null then defaultImports else pythonImportsCheck; + + homepage = force.homepage or ref.homepage; + in + buildPythonPackage ( + ( + { + inherit pname version src; + pyproject = true; + + build-system = [ + setuptools + wheel + ] + ++ nativeBuildDeps; + dependencies = propagatedBuildDeps; + pythonRelaxDeps = pythonRelaxDeps; + pythonImportsCheck = imports; + + installCheckPhase = '' + runHook preInstallCheck + ${builtins.concatStringsSep "\n" ( + map (mod: '' + ${python.interpreter} -c "import ${mod}; print(getattr(${mod},'__version__','unknown'))" + '') imports + )} + runHook postInstallCheck + ''; + + meta = { + inherit homepage; + } + // meta; + } + // lib.optionalAttrs (optional-dependencies != null) { + optional-dependencies = optional-dependencies; + } + ) + // lib.optionalAttrs (subdir != null && subdir != "") { + sourceRoot = "${sourceRootBase}/${subdir}"; + } + ); + + # 핵: specs = { pkgA = { url=...; hash=...; ...; }; ...; } + # -> { pkgA = mkPrefab specs.pkgA; ...; } + mkPrefabs = specs: lib.mapAttrs (_: spec: mkPrefab spec) specs; +in +{ + inherit mkPrefab mkPrefabs; +}