diff --git a/custom/pkgs/by-category/pythonPackages/prefab-specs.nix b/custom/pkgs/by-category/pythonPackages/prefab-specs.nix index 648c6dc..54d7ce9 100644 --- a/custom/pkgs/by-category/pythonPackages/prefab-specs.nix +++ b/custom/pkgs/by-category/pythonPackages/prefab-specs.nix @@ -6,7 +6,6 @@ huggingface-hub, tqdm, - aiohttp, configparser, google-api-core, @@ -21,7 +20,6 @@ tenacity, tiktoken, xlsxwriter, - boto3, click, loguru, @@ -38,23 +36,16 @@ openai, beautifulsoup4, magika, - av, - pdm-backend, fasttext-predict, - pydantic-settings, - aiofiles, hatch-fancy-pypi-readme, - colorlog, torch, torchvision, - wcwidth, - matplotlib, pyyaml, scipy, @@ -62,9 +53,22 @@ py-cpuinfo, seaborn, albumentations, - distutils, - ninja, + transformers, + accelerate, + ultralytics, + dill, + ftfy, + shapely, + pyclipper, + omegaconf, + onnxruntime, + fastapi, + python-multipart, + uvicorn, + gradio, + gradio-pdf, + ultralytics-thop, }: self: @@ -79,10 +83,11 @@ let fast-langdetect mineru-vl-utils qwen-vl-utils - thop + # thop httpx-retries robust-downloader doclayout-yolo + decord ; in { @@ -92,10 +97,10 @@ in propagatedBuildDeps = [ huggingface-hub lightrag-hku - mineru + # mineru + (mineru.optionalDeps [ "core" ]) tqdm - ] - ++ mineru.passthru.optional-dependencies.core; + ]; pythonImportsCheck = [ "raganything" ]; meta = { description = "RAGAnything: All-in-One RAG System"; @@ -130,7 +135,7 @@ in }; pipmaster = { url = "https://github.com/ParisNeo/pipmaster/tree/820acdbc9d541443942bd8afd6ba968036bd8452"; - hash = "sha256-IJ6VG4WvoEm584vbZDLTtRcEZekZh7HJto1hnABoodo="; + hash = "sha256-H1R/hXPyjn1r6Dal0QMEQeBR5xUlgacxFsuDUZVwV+s="; force.version = "1.1.0"; propagatedBuildDeps = [ ascii-colors @@ -157,7 +162,7 @@ in hash = "sha256-zK4vM2sOfEFDJBpFjcJmQXljg3pgprP+VFuuk5JNWnE="; force.version = "0.11.12"; propagatedBuildDeps = [ - # wcwidth + wcwidth ]; meta = { description = "A python library for displaying stuff on the console in a pretty way"; @@ -193,24 +198,52 @@ in mineru-vl-utils qwen-vl-utils ]; - optional-dependencies = { - core = [ - doclayout-yolo - ] - # ++ (with mineru.optional-dependencies; [ - # vlm - # pipeline - # api - # gradio - # ]) - ; - # vlm = [ ]; - # pipeline = [ - # doclayout-yolo - # ]; - # api = [ ]; - # gradio = [ ]; - }; + optional-dependencies = + let + _gradio = gradio; + in + rec { + vlm = [ + torch + transformers + accelerate + ]; + # vllm = [ + # vllm + # ]; + # lmdeploy = [ + # lmdeploy + # ]; + # mlx = [ + # mlx-vlm + # ]; + pipeline = [ + matplotlib + ultralytics + doclayout-yolo + dill + pyyaml + ftfy + shapely + pyclipper + omegaconf + torch + torchvision + transformers + onnxruntime + ]; + api = [ + fastapi + python-multipart + uvicorn + ]; + gradio = [ + _gradio + gradio-pdf + ]; + core = vlm ++ pipeline ++ api ++ gradio; + # all = core ++ mlx ++ vllm ++ lmdeploy; + }; pythonRelaxDeps = [ "fast-langdetect" ]; @@ -234,32 +267,45 @@ in tqdm psutil py-cpuinfo - thop + ultralytics-thop pandas seaborn albumentations + huggingface-hub + ]; + postInstall = '' + # avoid buildEnv collisions with ultralytics' YOLO CLI wrappers + rm -f "$out/bin/yolo" + ''; + pythonRelaxDeps = [ + # satisfied by ultralytics-thop (module), but dist name differs + "thop" + ]; + pythonRemoveDeps = [ + "thop" ]; meta = { description = "DocLayout-YOLO: Enhancing Document Layout Analysis through Diverse Synthetic Data and Global-to-Local Adaptive Perception"; license = lib.licenses.agpl3Only; }; }; - thop = { - url = "https://pypi.org/project/thop/0.1.1.post2209072238/"; - force = { - format = "wheel"; - python = "py3"; - dist = "py3"; - }; - hash = "sha256-AUc8IlIxkn0q1xg1H3jr98/+avO+1GTE8boe8PfN2ic="; - propagatedBuildDeps = [ - distutils - ]; - meta = { - description = "Count the MACs / FLOPs of your PyTorch model."; - license = lib.licenses.mit; - }; - }; + # thop = { + # url = "https://pypi.org/project/thop/0.1.1.post2209072238/"; + # force = { + # format = "wheel"; + # python = "py3"; + # dist = "py3"; + # }; + # hash = "sha256-AUc8IlIxkn0q1xg1H3jr98/+avO+1GTE8boe8PfN2ic="; + # propagatedBuildDeps = [ + # distutils + # torch + # ]; + # meta = { + # description = "Count the MACs / FLOPs of your PyTorch model."; + # license = lib.licenses.mit; + # }; + # }; qwen-vl-utils = { url = "https://github.com/QwenLM/Qwen3-VL/tree/fe12058/qwen-vl-utils"; hash = "sha256-Vha/Tc4q2v5RCL31hB9U4ZrfIFxfZjgwM6PYlvDeoAQ="; @@ -275,11 +321,23 @@ in torchvision ]; pythonImportsCheck = [ "qwen_vl_utils" ]; + optional-dependencies = { + decord = [ + decord + ]; + }; meta = { description = "Qwen-VL Utils contains a set of helper functions for processing and integrating visual language information with Qwen-VL Series Model."; license = lib.licenses.asl20; }; }; + decord = { + url = "https://github.com/dmlc/decord/tree/v0.6.0/python"; + meta = { + description = "An efficient video loader for deep learning with smart shuffling that's super easy to digest"; + license = lib.licenses.asl20; + }; + }; mineru-vl-utils = { url = "https://github.com/opendatalab/mineru-vl-utils/tree/mineru_vl_utils-0.1.22-released"; hash = "sha256-hpTW/1nwXPxfld4nx0XHZBMerWj+UL1vzDhYpwjezRU="; @@ -291,6 +349,24 @@ in pydantic loguru ]; + optional-dependencies = { + transformers = [ + torch + transformers + accelerate + torchvision + ]; + # vllm = [ + # vllm + # ]; + # mlx = [ + # mlx-vlm + # ]; + # lmdeploy = [ + # lmdeploy + # qwen-vl-utils + # ]; + }; meta = { description = "A Python package for interacting with the MinerU Vision-Language Model."; license = lib.licenses.gpl3; diff --git a/custom/pkgs/lib/prefab.nix b/custom/pkgs/lib/prefab.nix index 2786b5a..e2445a7 100644 --- a/custom/pkgs/lib/prefab.nix +++ b/custom/pkgs/lib/prefab.nix @@ -25,6 +25,8 @@ let propagatedBuildDeps ? [ ], nativeBuildDeps ? [ ], pythonRelaxDeps ? [ ], + pythonRemoveDeps ? [ ], + postInstall ? null, pythonImportsCheck ? null, optional-dependencies ? { }, }: @@ -77,49 +79,80 @@ let passthru' = passthru // lib.optionalAttrs (optional-dependencies != { }) { - passthru.optional-dependencies = optional-dependencies; + optional-dependencies = optional-dependencies; }; in - buildPythonPackage ( - { - inherit pname version src; - dependencies = propagatedBuildDeps; - pythonRelaxDeps = pythonRelaxDeps; - pythonImportsCheck = imports; - dontBuild = (force.format or null) == "wheel"; + let + mkDrv = + extraDeps: + buildPythonPackage ( + { + inherit pname version src; + dependencies = propagatedBuildDeps ++ extraDeps; + pythonRelaxDeps = pythonRelaxDeps; + pythonRemoveDeps = pythonRemoveDeps; + pythonImportsCheck = imports; - installCheckPhase = '' - runHook preInstallCheck - ${builtins.concatStringsSep "\n" ( - map (mod: '' - ${python.interpreter} -c "import ${mod}; print(getattr(${mod},'__version__','unknown'))" - '') imports - )} - runHook postInstallCheck - ''; + dontBuild = (force.format or null) == "wheel"; - passthru = passthru'; + installCheckPhase = '' + runHook preInstallCheck + ${builtins.concatStringsSep "\n" ( + map (mod: '' + ${python.interpreter} -c "import ${mod}; print(getattr(${mod},'__version__','unknown'))" + '') imports + )} + runHook postInstallCheck + ''; - meta = { - inherit homepage; - } - // meta; - sourceRoot = if ref.subdir != null then "source/${ref.subdir}" else null; - } - // lib.optionalAttrs isWheel { - format = "wheel"; - nativeBuildInputs = nativeBuildDeps; - } - // lib.optionalAttrs (!isWheel) { - pyproject = true; - build-system = [ - setuptools - wheel - ] - ++ nativeBuildDeps; - } - ); + passthru = passthru'; + + meta = { + inherit homepage; + } + // meta; + sourceRoot = if ref.subdir != null then "source/${ref.subdir}" else null; + } + // lib.optionalAttrs (postInstall != null) { inherit postInstall; } + // lib.optionalAttrs isWheel { + format = "wheel"; + nativeBuildInputs = nativeBuildDeps; + } + // lib.optionalAttrs (!isWheel) { + pyproject = true; + build-system = [ + setuptools + wheel + ] + ++ nativeBuildDeps; + } + ); + drv = mkDrv [ ]; + + in + if optional-dependencies == { } then + drv + else + let + extraPkgs = + extras: + lib.concatLists ( + map ( + x: + if builtins.isString x then + (optional-dependencies.${x} or (throw "${pname}.optionalDeps: unknown extra '${x}'")) + else if builtins.isList x then + x + else + [ x ] + ) extras + ); + in + drv + // { + optionalDeps = extras: mkDrv (extraPkgs extras); + }; mkPrefabs = specs: lib.mapAttrs (_: spec: mkPrefab spec) specs; mkPrefabsRec = specsFn: lib.fix (self: mkPrefabs (specsFn self));