dependencies with optionals work, nice
This commit is contained in:
parent
2109aceb7b
commit
1b9fefb118
2 changed files with 198 additions and 89 deletions
|
|
@ -6,7 +6,6 @@
|
|||
|
||||
huggingface-hub,
|
||||
tqdm,
|
||||
|
||||
aiohttp,
|
||||
configparser,
|
||||
google-api-core,
|
||||
|
|
@ -21,7 +20,6 @@
|
|||
tenacity,
|
||||
tiktoken,
|
||||
xlsxwriter,
|
||||
|
||||
boto3,
|
||||
click,
|
||||
loguru,
|
||||
|
|
@ -38,23 +36,16 @@
|
|||
openai,
|
||||
beautifulsoup4,
|
||||
magika,
|
||||
|
||||
av,
|
||||
|
||||
pdm-backend,
|
||||
fasttext-predict,
|
||||
|
||||
pydantic-settings,
|
||||
|
||||
aiofiles,
|
||||
hatch-fancy-pypi-readme,
|
||||
|
||||
colorlog,
|
||||
torch,
|
||||
torchvision,
|
||||
|
||||
wcwidth,
|
||||
|
||||
matplotlib,
|
||||
pyyaml,
|
||||
scipy,
|
||||
|
|
@ -62,9 +53,22 @@
|
|||
py-cpuinfo,
|
||||
seaborn,
|
||||
albumentations,
|
||||
|
||||
distutils,
|
||||
ninja,
|
||||
transformers,
|
||||
accelerate,
|
||||
ultralytics,
|
||||
dill,
|
||||
ftfy,
|
||||
shapely,
|
||||
pyclipper,
|
||||
omegaconf,
|
||||
onnxruntime,
|
||||
fastapi,
|
||||
python-multipart,
|
||||
uvicorn,
|
||||
gradio,
|
||||
gradio-pdf,
|
||||
ultralytics-thop,
|
||||
}:
|
||||
|
||||
self:
|
||||
|
|
@ -79,10 +83,11 @@ let
|
|||
fast-langdetect
|
||||
mineru-vl-utils
|
||||
qwen-vl-utils
|
||||
thop
|
||||
# thop
|
||||
httpx-retries
|
||||
robust-downloader
|
||||
doclayout-yolo
|
||||
decord
|
||||
;
|
||||
in
|
||||
{
|
||||
|
|
@ -92,10 +97,10 @@ in
|
|||
propagatedBuildDeps = [
|
||||
huggingface-hub
|
||||
lightrag-hku
|
||||
mineru
|
||||
# mineru
|
||||
(mineru.optionalDeps [ "core" ])
|
||||
tqdm
|
||||
]
|
||||
++ mineru.passthru.optional-dependencies.core;
|
||||
];
|
||||
pythonImportsCheck = [ "raganything" ];
|
||||
meta = {
|
||||
description = "RAGAnything: All-in-One RAG System";
|
||||
|
|
@ -130,7 +135,7 @@ in
|
|||
};
|
||||
pipmaster = {
|
||||
url = "https://github.com/ParisNeo/pipmaster/tree/820acdbc9d541443942bd8afd6ba968036bd8452";
|
||||
hash = "sha256-IJ6VG4WvoEm584vbZDLTtRcEZekZh7HJto1hnABoodo=";
|
||||
hash = "sha256-H1R/hXPyjn1r6Dal0QMEQeBR5xUlgacxFsuDUZVwV+s=";
|
||||
force.version = "1.1.0";
|
||||
propagatedBuildDeps = [
|
||||
ascii-colors
|
||||
|
|
@ -157,7 +162,7 @@ in
|
|||
hash = "sha256-zK4vM2sOfEFDJBpFjcJmQXljg3pgprP+VFuuk5JNWnE=";
|
||||
force.version = "0.11.12";
|
||||
propagatedBuildDeps = [
|
||||
# wcwidth
|
||||
wcwidth
|
||||
];
|
||||
meta = {
|
||||
description = "A python library for displaying stuff on the console in a pretty way";
|
||||
|
|
@ -193,24 +198,52 @@ in
|
|||
mineru-vl-utils
|
||||
qwen-vl-utils
|
||||
];
|
||||
optional-dependencies = {
|
||||
core = [
|
||||
doclayout-yolo
|
||||
]
|
||||
# ++ (with mineru.optional-dependencies; [
|
||||
# vlm
|
||||
# pipeline
|
||||
# api
|
||||
# gradio
|
||||
# ])
|
||||
;
|
||||
# vlm = [ ];
|
||||
# pipeline = [
|
||||
# doclayout-yolo
|
||||
# ];
|
||||
# api = [ ];
|
||||
# gradio = [ ];
|
||||
};
|
||||
optional-dependencies =
|
||||
let
|
||||
_gradio = gradio;
|
||||
in
|
||||
rec {
|
||||
vlm = [
|
||||
torch
|
||||
transformers
|
||||
accelerate
|
||||
];
|
||||
# vllm = [
|
||||
# vllm
|
||||
# ];
|
||||
# lmdeploy = [
|
||||
# lmdeploy
|
||||
# ];
|
||||
# mlx = [
|
||||
# mlx-vlm
|
||||
# ];
|
||||
pipeline = [
|
||||
matplotlib
|
||||
ultralytics
|
||||
doclayout-yolo
|
||||
dill
|
||||
pyyaml
|
||||
ftfy
|
||||
shapely
|
||||
pyclipper
|
||||
omegaconf
|
||||
torch
|
||||
torchvision
|
||||
transformers
|
||||
onnxruntime
|
||||
];
|
||||
api = [
|
||||
fastapi
|
||||
python-multipart
|
||||
uvicorn
|
||||
];
|
||||
gradio = [
|
||||
_gradio
|
||||
gradio-pdf
|
||||
];
|
||||
core = vlm ++ pipeline ++ api ++ gradio;
|
||||
# all = core ++ mlx ++ vllm ++ lmdeploy;
|
||||
};
|
||||
pythonRelaxDeps = [
|
||||
"fast-langdetect"
|
||||
];
|
||||
|
|
@ -234,32 +267,45 @@ in
|
|||
tqdm
|
||||
psutil
|
||||
py-cpuinfo
|
||||
thop
|
||||
ultralytics-thop
|
||||
pandas
|
||||
seaborn
|
||||
albumentations
|
||||
huggingface-hub
|
||||
];
|
||||
postInstall = ''
|
||||
# avoid buildEnv collisions with ultralytics' YOLO CLI wrappers
|
||||
rm -f "$out/bin/yolo"
|
||||
'';
|
||||
pythonRelaxDeps = [
|
||||
# satisfied by ultralytics-thop (module), but dist name differs
|
||||
"thop"
|
||||
];
|
||||
pythonRemoveDeps = [
|
||||
"thop"
|
||||
];
|
||||
meta = {
|
||||
description = "DocLayout-YOLO: Enhancing Document Layout Analysis through Diverse Synthetic Data and Global-to-Local Adaptive Perception";
|
||||
license = lib.licenses.agpl3Only;
|
||||
};
|
||||
};
|
||||
thop = {
|
||||
url = "https://pypi.org/project/thop/0.1.1.post2209072238/";
|
||||
force = {
|
||||
format = "wheel";
|
||||
python = "py3";
|
||||
dist = "py3";
|
||||
};
|
||||
hash = "sha256-AUc8IlIxkn0q1xg1H3jr98/+avO+1GTE8boe8PfN2ic=";
|
||||
propagatedBuildDeps = [
|
||||
distutils
|
||||
];
|
||||
meta = {
|
||||
description = "Count the MACs / FLOPs of your PyTorch model.";
|
||||
license = lib.licenses.mit;
|
||||
};
|
||||
};
|
||||
# thop = {
|
||||
# url = "https://pypi.org/project/thop/0.1.1.post2209072238/";
|
||||
# force = {
|
||||
# format = "wheel";
|
||||
# python = "py3";
|
||||
# dist = "py3";
|
||||
# };
|
||||
# hash = "sha256-AUc8IlIxkn0q1xg1H3jr98/+avO+1GTE8boe8PfN2ic=";
|
||||
# propagatedBuildDeps = [
|
||||
# distutils
|
||||
# torch
|
||||
# ];
|
||||
# meta = {
|
||||
# description = "Count the MACs / FLOPs of your PyTorch model.";
|
||||
# license = lib.licenses.mit;
|
||||
# };
|
||||
# };
|
||||
qwen-vl-utils = {
|
||||
url = "https://github.com/QwenLM/Qwen3-VL/tree/fe12058/qwen-vl-utils";
|
||||
hash = "sha256-Vha/Tc4q2v5RCL31hB9U4ZrfIFxfZjgwM6PYlvDeoAQ=";
|
||||
|
|
@ -275,11 +321,23 @@ in
|
|||
torchvision
|
||||
];
|
||||
pythonImportsCheck = [ "qwen_vl_utils" ];
|
||||
optional-dependencies = {
|
||||
decord = [
|
||||
decord
|
||||
];
|
||||
};
|
||||
meta = {
|
||||
description = "Qwen-VL Utils contains a set of helper functions for processing and integrating visual language information with Qwen-VL Series Model.";
|
||||
license = lib.licenses.asl20;
|
||||
};
|
||||
};
|
||||
decord = {
|
||||
url = "https://github.com/dmlc/decord/tree/v0.6.0/python";
|
||||
meta = {
|
||||
description = "An efficient video loader for deep learning with smart shuffling that's super easy to digest";
|
||||
license = lib.licenses.asl20;
|
||||
};
|
||||
};
|
||||
mineru-vl-utils = {
|
||||
url = "https://github.com/opendatalab/mineru-vl-utils/tree/mineru_vl_utils-0.1.22-released";
|
||||
hash = "sha256-hpTW/1nwXPxfld4nx0XHZBMerWj+UL1vzDhYpwjezRU=";
|
||||
|
|
@ -291,6 +349,24 @@ in
|
|||
pydantic
|
||||
loguru
|
||||
];
|
||||
optional-dependencies = {
|
||||
transformers = [
|
||||
torch
|
||||
transformers
|
||||
accelerate
|
||||
torchvision
|
||||
];
|
||||
# vllm = [
|
||||
# vllm
|
||||
# ];
|
||||
# mlx = [
|
||||
# mlx-vlm
|
||||
# ];
|
||||
# lmdeploy = [
|
||||
# lmdeploy
|
||||
# qwen-vl-utils
|
||||
# ];
|
||||
};
|
||||
meta = {
|
||||
description = "A Python package for interacting with the MinerU Vision-Language Model.";
|
||||
license = lib.licenses.gpl3;
|
||||
|
|
|
|||
|
|
@ -25,6 +25,8 @@ let
|
|||
propagatedBuildDeps ? [ ],
|
||||
nativeBuildDeps ? [ ],
|
||||
pythonRelaxDeps ? [ ],
|
||||
pythonRemoveDeps ? [ ],
|
||||
postInstall ? null,
|
||||
pythonImportsCheck ? null,
|
||||
optional-dependencies ? { },
|
||||
}:
|
||||
|
|
@ -77,49 +79,80 @@ let
|
|||
passthru' =
|
||||
passthru
|
||||
// lib.optionalAttrs (optional-dependencies != { }) {
|
||||
passthru.optional-dependencies = optional-dependencies;
|
||||
optional-dependencies = optional-dependencies;
|
||||
};
|
||||
in
|
||||
buildPythonPackage (
|
||||
{
|
||||
inherit pname version src;
|
||||
dependencies = propagatedBuildDeps;
|
||||
pythonRelaxDeps = pythonRelaxDeps;
|
||||
pythonImportsCheck = imports;
|
||||
|
||||
dontBuild = (force.format or null) == "wheel";
|
||||
let
|
||||
mkDrv =
|
||||
extraDeps:
|
||||
buildPythonPackage (
|
||||
{
|
||||
inherit pname version src;
|
||||
dependencies = propagatedBuildDeps ++ extraDeps;
|
||||
pythonRelaxDeps = pythonRelaxDeps;
|
||||
pythonRemoveDeps = pythonRemoveDeps;
|
||||
pythonImportsCheck = imports;
|
||||
|
||||
installCheckPhase = ''
|
||||
runHook preInstallCheck
|
||||
${builtins.concatStringsSep "\n" (
|
||||
map (mod: ''
|
||||
${python.interpreter} -c "import ${mod}; print(getattr(${mod},'__version__','unknown'))"
|
||||
'') imports
|
||||
)}
|
||||
runHook postInstallCheck
|
||||
'';
|
||||
dontBuild = (force.format or null) == "wheel";
|
||||
|
||||
passthru = passthru';
|
||||
installCheckPhase = ''
|
||||
runHook preInstallCheck
|
||||
${builtins.concatStringsSep "\n" (
|
||||
map (mod: ''
|
||||
${python.interpreter} -c "import ${mod}; print(getattr(${mod},'__version__','unknown'))"
|
||||
'') imports
|
||||
)}
|
||||
runHook postInstallCheck
|
||||
'';
|
||||
|
||||
meta = {
|
||||
inherit homepage;
|
||||
}
|
||||
// meta;
|
||||
sourceRoot = if ref.subdir != null then "source/${ref.subdir}" else null;
|
||||
}
|
||||
// lib.optionalAttrs isWheel {
|
||||
format = "wheel";
|
||||
nativeBuildInputs = nativeBuildDeps;
|
||||
}
|
||||
// lib.optionalAttrs (!isWheel) {
|
||||
pyproject = true;
|
||||
build-system = [
|
||||
setuptools
|
||||
wheel
|
||||
]
|
||||
++ nativeBuildDeps;
|
||||
}
|
||||
);
|
||||
passthru = passthru';
|
||||
|
||||
meta = {
|
||||
inherit homepage;
|
||||
}
|
||||
// meta;
|
||||
sourceRoot = if ref.subdir != null then "source/${ref.subdir}" else null;
|
||||
}
|
||||
// lib.optionalAttrs (postInstall != null) { inherit postInstall; }
|
||||
// lib.optionalAttrs isWheel {
|
||||
format = "wheel";
|
||||
nativeBuildInputs = nativeBuildDeps;
|
||||
}
|
||||
// lib.optionalAttrs (!isWheel) {
|
||||
pyproject = true;
|
||||
build-system = [
|
||||
setuptools
|
||||
wheel
|
||||
]
|
||||
++ nativeBuildDeps;
|
||||
}
|
||||
);
|
||||
drv = mkDrv [ ];
|
||||
|
||||
in
|
||||
if optional-dependencies == { } then
|
||||
drv
|
||||
else
|
||||
let
|
||||
extraPkgs =
|
||||
extras:
|
||||
lib.concatLists (
|
||||
map (
|
||||
x:
|
||||
if builtins.isString x then
|
||||
(optional-dependencies.${x} or (throw "${pname}.optionalDeps: unknown extra '${x}'"))
|
||||
else if builtins.isList x then
|
||||
x
|
||||
else
|
||||
[ x ]
|
||||
) extras
|
||||
);
|
||||
in
|
||||
drv
|
||||
// {
|
||||
optionalDeps = extras: mkDrv (extraPkgs extras);
|
||||
};
|
||||
mkPrefabs = specs: lib.mapAttrs (_: spec: mkPrefab spec) specs;
|
||||
|
||||
mkPrefabsRec = specsFn: lib.fix (self: mkPrefabs (specsFn self));
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue