i broke srcDir at some point... rework part 3
This commit is contained in:
parent
7febb7395f
commit
497b758c25
5 changed files with 471 additions and 427 deletions
|
|
@ -21,7 +21,7 @@ let
|
|||
[ ];
|
||||
|
||||
fileExts =
|
||||
if builtins.pathExists (pythonPackagesDir + "/python-packages.nix") then
|
||||
if builtins.pathExists (pythonPackagesDir + "/prefab-builder.nix") then
|
||||
[
|
||||
(
|
||||
python-final: python-prev:
|
||||
|
|
@ -29,9 +29,13 @@ let
|
|||
python-prev
|
||||
// {
|
||||
lib = prev.lib;
|
||||
pkgs = prev; # чтобы prefab-builder мог звать pkgs.callPackage
|
||||
pyPkgs = python-prev; # явная подача python package set
|
||||
fetchFromGitHub = prev.fetchFromGitHub;
|
||||
# fetchPypi обычно уже есть в python-prev; если нет — добавь:
|
||||
fetchPypi = prev.fetchPypi;
|
||||
}
|
||||
) (pythonPackagesDir + "/python-packages.nix") { }
|
||||
) (pythonPackagesDir + "/prefab-builder.nix") { }
|
||||
)
|
||||
]
|
||||
else
|
||||
|
|
|
|||
|
|
@ -1,8 +1,23 @@
|
|||
{
|
||||
pkgs,
|
||||
lib,
|
||||
pyPkgs,
|
||||
fetchFromGitHub,
|
||||
fetchPypi,
|
||||
}:
|
||||
|
||||
let
|
||||
prefab = import ../../../lib/prefab.nix {
|
||||
inherit lib fetchFromGitHub fetchPypi;
|
||||
inherit (python.pkgs) buildPythonPackage setuptools wheel;
|
||||
python = python;
|
||||
prefab = pkgs.callPackage ../../lib/prefab.nix {
|
||||
inherit
|
||||
lib
|
||||
fetchFromGitHub
|
||||
fetchPypi
|
||||
pyPkgs
|
||||
;
|
||||
};
|
||||
in
|
||||
prefab.mkPrefabsRec (import ./prefab-specs.nix)
|
||||
prefab.mkPrefabsRec (
|
||||
lib.customisation.callPackageWith pyPkgs ./prefab-specs.nix {
|
||||
inherit lib;
|
||||
}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -59,22 +59,39 @@
|
|||
scipy,
|
||||
psutil,
|
||||
py-cpuinfo,
|
||||
thop,
|
||||
seaborn,
|
||||
albumentations,
|
||||
}:
|
||||
|
||||
self: {
|
||||
self:
|
||||
let
|
||||
inherit (self)
|
||||
lightrag-hku
|
||||
mineru
|
||||
nano-vectordb
|
||||
pipmaster
|
||||
ascii-colors
|
||||
pdftext
|
||||
fast-langdetect
|
||||
mineru-vl-utils
|
||||
qwen-vl-utils
|
||||
thop
|
||||
httpx-retries
|
||||
robust-downloader
|
||||
;
|
||||
in
|
||||
{
|
||||
raganything = {
|
||||
url = "https://github.com/HKUDS/RAG-Anything/tree/v1.2.9";
|
||||
hash = "sha256-yepiLYzPD6UcJRbAbovg/BwFE8nh903o/tHypiHGKSw=";
|
||||
propagatedBuildDeps = [
|
||||
huggingface-hub
|
||||
self.lightrag-hku
|
||||
lightrag-hku
|
||||
mineru
|
||||
tqdm
|
||||
# mineru
|
||||
]
|
||||
++ self.mineru.optional-dependencies.core;
|
||||
# ++ mineru.optional-dependencies.core
|
||||
;
|
||||
pythonImportsCheck = [ "raganything" ];
|
||||
meta = {
|
||||
description = "RAGAnything: All-in-One RAG System";
|
||||
|
|
@ -90,11 +107,11 @@ self: {
|
|||
google-api-core
|
||||
google-genai
|
||||
json-repair
|
||||
self.nano-vectordb
|
||||
nano-vectordb
|
||||
networkx
|
||||
numpy
|
||||
pandas
|
||||
self.pipmaster
|
||||
pipmaster
|
||||
pydantic
|
||||
pypinyin
|
||||
python-dotenv
|
||||
|
|
@ -112,7 +129,7 @@ self: {
|
|||
hash = "sha256-IJ6VG4WvoEm584vbZDLTtRcEZekZh7HJto1hnABoodo=";
|
||||
force.version = "1.1.0";
|
||||
propagatedBuildDeps = [
|
||||
self.ascii-colors
|
||||
ascii-colors
|
||||
];
|
||||
meta = with lib; {
|
||||
description = "A versatile Python package manager utility for simplifying package installation, updates, checks, and environment management.";
|
||||
|
|
@ -159,35 +176,35 @@ self: {
|
|||
pypdfium2
|
||||
pypdf
|
||||
reportlab
|
||||
self.pdftext
|
||||
pdftext
|
||||
modelscope
|
||||
huggingface-hub
|
||||
json-repair
|
||||
opencv-python
|
||||
self.fast-langdetect
|
||||
fast-langdetect
|
||||
scikit-image
|
||||
openai
|
||||
beautifulsoup4
|
||||
magika
|
||||
self.mineru-vl-utils
|
||||
self.qwen-vl-utils
|
||||
mineru-vl-utils
|
||||
qwen-vl-utils
|
||||
];
|
||||
optional-dependencies = {
|
||||
core = [ self.doclayout-yolo ]
|
||||
# ++ (with mineru.optional-dependencies; [
|
||||
# vlm
|
||||
# pipeline
|
||||
# api
|
||||
# gradio
|
||||
# ])
|
||||
;
|
||||
vlm = [ ];
|
||||
pipeline = [
|
||||
self.doclayout-yolo
|
||||
];
|
||||
api = [ ];
|
||||
gradio = [ ];
|
||||
};
|
||||
# optional-dependencies = {
|
||||
# core = [ doclayout-yolo ]
|
||||
# # ++ (with mineru.optional-dependencies; [
|
||||
# # vlm
|
||||
# # pipeline
|
||||
# # api
|
||||
# # gradio
|
||||
# # ])
|
||||
# ;
|
||||
# vlm = [ ];
|
||||
# pipeline = [
|
||||
# doclayout-yolo
|
||||
# ];
|
||||
# api = [ ];
|
||||
# gradio = [ ];
|
||||
# };
|
||||
pythonRelaxDeps = [
|
||||
"fast-langdetect"
|
||||
];
|
||||
|
|
@ -223,12 +240,20 @@ self: {
|
|||
license = lib.licenses.agpl3Only;
|
||||
};
|
||||
};
|
||||
thop = {
|
||||
url = "https://pypi.org/project/thop/0.1.1.post2209072238/";
|
||||
hash = "";
|
||||
force.version = "0.1.1";
|
||||
meta = {
|
||||
description = "Count the MACs / FLOPs of your PyTorch model.";
|
||||
license = lib.licenses.mit;
|
||||
};
|
||||
};
|
||||
qwen-vl-utils = {
|
||||
url = "https://github.com/QwenLM/Qwen3-VL/tree/main/qwen-vl-utils";
|
||||
url = "https://github.com/QwenLM/Qwen3-VL/tree/fe12058/qwen-vl-utils";
|
||||
hash = "sha256-Vha/Tc4q2v5RCL31hB9U4ZrfIFxfZjgwM6PYlvDeoAQ=";
|
||||
force = {
|
||||
version = "0.0.14";
|
||||
rev = "fe12058";
|
||||
};
|
||||
propagatedBuildDeps = [
|
||||
hatchling
|
||||
|
|
@ -249,7 +274,7 @@ self: {
|
|||
hash = "sha256-hpTW/1nwXPxfld4nx0XHZBMerWj+UL1vzDhYpwjezRU=";
|
||||
propagatedBuildDeps = [
|
||||
httpx
|
||||
self.httpx-retries
|
||||
httpx-retries
|
||||
aiofiles
|
||||
pillow
|
||||
pydantic
|
||||
|
|
@ -266,7 +291,6 @@ self: {
|
|||
propagatedBuildDeps = [
|
||||
hatchling
|
||||
hatch-fancy-pypi-readme
|
||||
|
||||
httpx
|
||||
];
|
||||
meta = {
|
||||
|
|
@ -279,7 +303,7 @@ self: {
|
|||
hash = "sha256-pj46gHG9cjkSjnYc88bSctL/1LAUe0jkBuM/GZWMsUI=";
|
||||
propagatedBuildDeps = [
|
||||
pdm-backend
|
||||
self.robust-downloader
|
||||
robust-downloader
|
||||
requests
|
||||
fasttext-predict
|
||||
];
|
||||
|
|
@ -306,7 +330,6 @@ self: {
|
|||
hash = "sha256-EGVjzjDWtdcEPX//cOm5+xm9FvX0aP+h6fsD25hC8gA=";
|
||||
propagatedBuildDeps = [
|
||||
poetry-core
|
||||
|
||||
click
|
||||
pydantic
|
||||
pydantic-settings
|
||||
|
|
|
|||
|
|
@ -1,371 +1,371 @@
|
|||
{
|
||||
lib,
|
||||
fetchFromGitHub,
|
||||
buildPythonPackage,
|
||||
setuptools,
|
||||
wheel,
|
||||
hatchling,
|
||||
poetry-core,
|
||||
# {
|
||||
# lib,
|
||||
# fetchFromGitHub,
|
||||
# buildPythonPackage,
|
||||
# setuptools,
|
||||
# wheel,
|
||||
# hatchling,
|
||||
# poetry-core,
|
||||
|
||||
python,
|
||||
# python,
|
||||
|
||||
huggingface-hub,
|
||||
tqdm,
|
||||
# huggingface-hub,
|
||||
# tqdm,
|
||||
|
||||
aiohttp,
|
||||
configparser,
|
||||
google-api-core,
|
||||
google-genai,
|
||||
json-repair,
|
||||
networkx,
|
||||
numpy,
|
||||
pandas,
|
||||
pydantic,
|
||||
pypinyin,
|
||||
python-dotenv,
|
||||
tenacity,
|
||||
tiktoken,
|
||||
xlsxwriter,
|
||||
# aiohttp,
|
||||
# configparser,
|
||||
# google-api-core,
|
||||
# google-genai,
|
||||
# json-repair,
|
||||
# networkx,
|
||||
# numpy,
|
||||
# pandas,
|
||||
# pydantic,
|
||||
# pypinyin,
|
||||
# python-dotenv,
|
||||
# tenacity,
|
||||
# tiktoken,
|
||||
# xlsxwriter,
|
||||
|
||||
boto3,
|
||||
click,
|
||||
loguru,
|
||||
pdfminer-six,
|
||||
requests,
|
||||
httpx,
|
||||
pillow,
|
||||
pypdfium2,
|
||||
pypdf,
|
||||
reportlab,
|
||||
modelscope,
|
||||
opencv-python,
|
||||
scikit-image,
|
||||
openai,
|
||||
beautifulsoup4,
|
||||
magika,
|
||||
# boto3,
|
||||
# click,
|
||||
# loguru,
|
||||
# pdfminer-six,
|
||||
# requests,
|
||||
# httpx,
|
||||
# pillow,
|
||||
# pypdfium2,
|
||||
# pypdf,
|
||||
# reportlab,
|
||||
# modelscope,
|
||||
# opencv-python,
|
||||
# scikit-image,
|
||||
# openai,
|
||||
# beautifulsoup4,
|
||||
# magika,
|
||||
|
||||
av,
|
||||
# av,
|
||||
|
||||
pdm-backend,
|
||||
fasttext-predict,
|
||||
# pdm-backend,
|
||||
# fasttext-predict,
|
||||
|
||||
pydantic-settings,
|
||||
# pydantic-settings,
|
||||
|
||||
aiofiles,
|
||||
hatch-fancy-pypi-readme,
|
||||
# aiofiles,
|
||||
# hatch-fancy-pypi-readme,
|
||||
|
||||
colorlog,
|
||||
torch,
|
||||
torchvision,
|
||||
# colorlog,
|
||||
# torch,
|
||||
# torchvision,
|
||||
|
||||
wcwidth,
|
||||
}:
|
||||
# wcwidth,
|
||||
# }:
|
||||
|
||||
let
|
||||
prefab =
|
||||
{
|
||||
url,
|
||||
hash,
|
||||
force ? { },
|
||||
meta ? { },
|
||||
propagatedBuildDeps ? [ ],
|
||||
nativeBuildDeps ? [ ],
|
||||
pythonRelaxDeps ? [ ],
|
||||
pythonImportsCheck ? null,
|
||||
optional-dependencies ? null,
|
||||
}:
|
||||
let
|
||||
x = (import ../../lib/parse-github-url.nix { inherit lib; }) url;
|
||||
pname = x.pname;
|
||||
rev = force.rev or x.rev;
|
||||
version = force.version or x.version;
|
||||
# let
|
||||
# prefab =
|
||||
# {
|
||||
# url,
|
||||
# hash,
|
||||
# force ? { },
|
||||
# meta ? { },
|
||||
# propagatedBuildDeps ? [ ],
|
||||
# nativeBuildDeps ? [ ],
|
||||
# pythonRelaxDeps ? [ ],
|
||||
# pythonImportsCheck ? null,
|
||||
# optional-dependencies ? null,
|
||||
# }:
|
||||
# let
|
||||
# x = (import ../../lib/parse-github-url.nix { inherit lib; }) url;
|
||||
# pname = x.pname;
|
||||
# rev = force.rev or x.rev;
|
||||
# version = force.version or x.version;
|
||||
|
||||
defaultImports = [ (lib.toLower (lib.replaceStrings [ "-" ] [ "_" ] pname)) ];
|
||||
imports = if pythonImportsCheck == null then defaultImports else pythonImportsCheck;
|
||||
# defaultImports = [ (lib.toLower (lib.replaceStrings [ "-" ] [ "_" ] pname)) ];
|
||||
# imports = if pythonImportsCheck == null then defaultImports else pythonImportsCheck;
|
||||
|
||||
rawSubdir = force.subdir or x.subdir or null;
|
||||
subdir = if rawSubdir == null then null else lib.removeSuffix "/" (lib.removePrefix "/" rawSubdir);
|
||||
# rawSubdir = force.subdir or x.subdir or null;
|
||||
# subdir = if rawSubdir == null then null else lib.removeSuffix "/" (lib.removePrefix "/" rawSubdir);
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = x.owner;
|
||||
repo = x.repo;
|
||||
rev = rev;
|
||||
hash = hash;
|
||||
};
|
||||
in
|
||||
buildPythonPackage (
|
||||
{
|
||||
inherit pname version src;
|
||||
pyproject = true;
|
||||
# src = fetchFromGitHub {
|
||||
# owner = x.owner;
|
||||
# repo = x.repo;
|
||||
# rev = rev;
|
||||
# hash = hash;
|
||||
# };
|
||||
# in
|
||||
# buildPythonPackage (
|
||||
# {
|
||||
# inherit pname version src;
|
||||
# pyproject = true;
|
||||
|
||||
build-system = [
|
||||
setuptools
|
||||
wheel
|
||||
]
|
||||
++ nativeBuildDeps;
|
||||
dependencies = propagatedBuildDeps;
|
||||
optional-dependencies = optional-dependencies;
|
||||
pythonRelaxDeps = pythonRelaxDeps;
|
||||
# build-system = [
|
||||
# setuptools
|
||||
# wheel
|
||||
# ]
|
||||
# ++ nativeBuildDeps;
|
||||
# dependencies = propagatedBuildDeps;
|
||||
# optional-dependencies = optional-dependencies;
|
||||
# pythonRelaxDeps = pythonRelaxDeps;
|
||||
|
||||
pythonImportsCheck = imports;
|
||||
# pythonImportsCheck = imports;
|
||||
|
||||
installCheckPhase = ''
|
||||
runHook preInstallCheck
|
||||
${builtins.concatStringsSep "\n" (
|
||||
map (lib: ''
|
||||
${python.interpreter} -c "import ${lib}; print(getattr(${lib},'__version__','unknown'))"
|
||||
'') imports
|
||||
)}
|
||||
runHook postInstallCheck
|
||||
'';
|
||||
# installCheckPhase = ''
|
||||
# runHook preInstallCheck
|
||||
# ${builtins.concatStringsSep "\n" (
|
||||
# map (lib: ''
|
||||
# ${python.interpreter} -c "import ${lib}; print(getattr(${lib},'__version__','unknown'))"
|
||||
# '') imports
|
||||
# )}
|
||||
# runHook postInstallCheck
|
||||
# '';
|
||||
|
||||
meta = {
|
||||
homepage = x.homepage;
|
||||
}
|
||||
// meta;
|
||||
}
|
||||
// lib.optionalAttrs (subdir != null && subdir != "") {
|
||||
sourceRoot = "${src.name}/${subdir}";
|
||||
}
|
||||
);
|
||||
in
|
||||
rec {
|
||||
raganything = prefab {
|
||||
url = "https://github.com/HKUDS/RAG-Anything/tree/v1.2.9";
|
||||
hash = "sha256-yepiLYzPD6UcJRbAbovg/BwFE8nh903o/tHypiHGKSw=";
|
||||
propagatedBuildDeps = [
|
||||
huggingface-hub
|
||||
lightrag-hku
|
||||
tqdm
|
||||
# mineru
|
||||
]
|
||||
++ mineru.optional-dependencies.core;
|
||||
pythonImportsCheck = [ "raganything" ];
|
||||
meta = {
|
||||
description = "RAGAnything: All-in-One RAG System";
|
||||
license = lib.licenses.mit;
|
||||
};
|
||||
};
|
||||
lightrag-hku = prefab {
|
||||
url = "https://github.com/HKUDS/LightRAG/tree/v1.4.9.11";
|
||||
hash = "sha256-TX/HSF2ZqoDo2SRlFzM+bkUxQXiUpnrl6kwI+lljjdo=";
|
||||
propagatedBuildDeps = [
|
||||
aiohttp
|
||||
configparser
|
||||
google-api-core
|
||||
google-genai
|
||||
json-repair
|
||||
nano-vectordb
|
||||
networkx
|
||||
numpy
|
||||
pandas
|
||||
pipmaster
|
||||
pydantic
|
||||
pypinyin
|
||||
python-dotenv
|
||||
tenacity
|
||||
tiktoken
|
||||
xlsxwriter
|
||||
];
|
||||
meta = {
|
||||
description = "LightRAG: Simple and Fast Retrieval-Augmented Generation";
|
||||
license = lib.licenses.mit;
|
||||
};
|
||||
};
|
||||
pipmaster = prefab {
|
||||
url = "https://github.com/ParisNeo/pipmaster/tree/820acdbc9d541443942bd8afd6ba968036bd8452";
|
||||
hash = "sha256-IJ6VG4WvoEm584vbZDLTtRcEZekZh7HJto1hnABoodo=";
|
||||
force.version = "1.1.0";
|
||||
propagatedBuildDeps = [
|
||||
ascii-colors
|
||||
];
|
||||
meta = with lib; {
|
||||
description = "A versatile Python package manager utility for simplifying package installation, updates, checks, and environment management.";
|
||||
license = licenses.asl20;
|
||||
};
|
||||
};
|
||||
nano-vectordb = prefab {
|
||||
url = "https://github.com/gusye1234/nano-vectordb/tree/03f821348f04a93a9c36bb575faae05b61f4c02e";
|
||||
hash = "sha256-jLco+1TAncF8Ep+VGd3DhsoiZTW7G/H8fAEwnNsovTY=";
|
||||
force.version = "0.0.4.3";
|
||||
propagatedBuildDeps = [
|
||||
numpy
|
||||
];
|
||||
meta = {
|
||||
description = "A simple, easy-to-hack Vector Database implementation";
|
||||
license = lib.licenses.mit;
|
||||
};
|
||||
};
|
||||
ascii-colors = prefab {
|
||||
url = "https://github.com/ParisNeo/ascii_colors/tree/817a21485136635e3da89ff08444183254b25aab";
|
||||
hash = "sha256-jiolJzsBQMp9/DHdHTixJGvXoS19vAp/S7+8PyLARoo=";
|
||||
force.version = "0.11.12";
|
||||
propagatedBuildDeps = [
|
||||
wcwidth
|
||||
];
|
||||
meta = {
|
||||
description = "A python library for displaying stuff on the console in a pretty way";
|
||||
license = lib.licenses.asl20;
|
||||
};
|
||||
};
|
||||
mineru = prefab {
|
||||
url = "https://github.com/opendatalab/MinerU/tree/mineru-2.7.6-released";
|
||||
hash = "sha256-A/nFNQYGEfmWUdpX8N1lbex3mdiF8+eN8s0UKQFc42E=";
|
||||
propagatedBuildDeps = [
|
||||
boto3
|
||||
click
|
||||
loguru
|
||||
numpy
|
||||
pdfminer-six
|
||||
tqdm
|
||||
requests
|
||||
httpx
|
||||
pillow
|
||||
pypdfium2
|
||||
pypdf
|
||||
reportlab
|
||||
pdftext
|
||||
modelscope
|
||||
huggingface-hub
|
||||
json-repair
|
||||
opencv-python
|
||||
fast-langdetect
|
||||
scikit-image
|
||||
openai
|
||||
beautifulsoup4
|
||||
magika
|
||||
mineru-vl-utils
|
||||
qwen-vl-utils
|
||||
];
|
||||
optional-dependencies = {
|
||||
core = [ doclayout-yolo ]
|
||||
# ++ (with mineru.optional-dependencies; [
|
||||
# vlm
|
||||
# pipeline
|
||||
# api
|
||||
# gradio
|
||||
# ])
|
||||
;
|
||||
vlm = [ ];
|
||||
pipeline = [
|
||||
doclayout-yolo
|
||||
];
|
||||
api = [ ];
|
||||
gradio = [ ];
|
||||
};
|
||||
pythonRelaxDeps = [
|
||||
"fast-langdetect"
|
||||
];
|
||||
meta = {
|
||||
description = "Transforms complex documents like PDFs into LLM-ready markdown/JSON for your Agentic workflows.";
|
||||
license = lib.licenses.gpl3;
|
||||
};
|
||||
};
|
||||
doclayout-yolo = prefab {
|
||||
# reason for creating fetcher construct
|
||||
url = "https://github.com/opendatalab/DocLayout-YOLO/tree/main"; # master is 10 months old and last version was set in Oct 18, 2024
|
||||
hash = "sha256-wLCmJuWyw7Gqou7awsHtxqVyr4AN+yJSkF0q3GUSQh8=";
|
||||
force.version = "0.0.4"; # NOT 0.0.4 on github but fuck it we ball
|
||||
meta = {
|
||||
description = "DocLayout-YOLO: Enhancing Document Layout Analysis through Diverse Synthetic Data and Global-to-Local Adaptive Perception";
|
||||
license = lib.licenses.agpl3Only;
|
||||
};
|
||||
};
|
||||
qwen-vl-utils = prefab {
|
||||
url = "https://github.com/QwenLM/Qwen3-VL/tree/main/qwen-vl-utils";
|
||||
hash = "sha256-Vha/Tc4q2v5RCL31hB9U4ZrfIFxfZjgwM6PYlvDeoAQ=";
|
||||
force = {
|
||||
version = "0.0.14";
|
||||
rev = "fe12058";
|
||||
};
|
||||
propagatedBuildDeps = [
|
||||
hatchling
|
||||
av
|
||||
pillow
|
||||
requests
|
||||
torch
|
||||
torchvision
|
||||
];
|
||||
pythonImportsCheck = [ "qwen_vl_utils" ];
|
||||
meta = {
|
||||
description = "Qwen-VL Utils contains a set of helper functions for processing and integrating visual language information with Qwen-VL Series Model.";
|
||||
license = lib.licenses.asl20;
|
||||
};
|
||||
};
|
||||
mineru-vl-utils = prefab {
|
||||
url = "https://github.com/opendatalab/mineru-vl-utils/tree/mineru_vl_utils-0.1.22-released";
|
||||
hash = "sha256-hpTW/1nwXPxfld4nx0XHZBMerWj+UL1vzDhYpwjezRU=";
|
||||
propagatedBuildDeps = [
|
||||
httpx
|
||||
httpx-retries
|
||||
aiofiles
|
||||
pillow
|
||||
pydantic
|
||||
loguru
|
||||
];
|
||||
meta = {
|
||||
description = "A Python package for interacting with the MinerU Vision-Language Model.";
|
||||
license = lib.licenses.gpl3;
|
||||
};
|
||||
};
|
||||
httpx-retries = prefab {
|
||||
url = "https://github.com/will-ockmore/httpx-retries/tree/0.4.5";
|
||||
hash = "sha256-zJ3ExSEWxlHFluSdYA8/XZ3zb4KBelU+IOFyUu4ezvo=";
|
||||
propagatedBuildDeps = [
|
||||
hatchling
|
||||
hatch-fancy-pypi-readme
|
||||
# meta = {
|
||||
# homepage = x.homepage;
|
||||
# }
|
||||
# // meta;
|
||||
# }
|
||||
# // lib.optionalAttrs (subdir != null && subdir != "") {
|
||||
# sourceRoot = "${src.name}/${subdir}";
|
||||
# }
|
||||
# );
|
||||
# in
|
||||
# rec {
|
||||
# raganything = prefab {
|
||||
# url = "https://github.com/HKUDS/RAG-Anything/tree/v1.2.9";
|
||||
# hash = "sha256-yepiLYzPD6UcJRbAbovg/BwFE8nh903o/tHypiHGKSw=";
|
||||
# propagatedBuildDeps = [
|
||||
# huggingface-hub
|
||||
# lightrag-hku
|
||||
# tqdm
|
||||
# # mineru
|
||||
# ]
|
||||
# ++ mineru.optional-dependencies.core;
|
||||
# pythonImportsCheck = [ "raganything" ];
|
||||
# meta = {
|
||||
# description = "RAGAnything: All-in-One RAG System";
|
||||
# license = lib.licenses.mit;
|
||||
# };
|
||||
# };
|
||||
# lightrag-hku = prefab {
|
||||
# url = "https://github.com/HKUDS/LightRAG/tree/v1.4.9.11";
|
||||
# hash = "sha256-TX/HSF2ZqoDo2SRlFzM+bkUxQXiUpnrl6kwI+lljjdo=";
|
||||
# propagatedBuildDeps = [
|
||||
# aiohttp
|
||||
# configparser
|
||||
# google-api-core
|
||||
# google-genai
|
||||
# json-repair
|
||||
# nano-vectordb
|
||||
# networkx
|
||||
# numpy
|
||||
# pandas
|
||||
# pipmaster
|
||||
# pydantic
|
||||
# pypinyin
|
||||
# python-dotenv
|
||||
# tenacity
|
||||
# tiktoken
|
||||
# xlsxwriter
|
||||
# ];
|
||||
# meta = {
|
||||
# description = "LightRAG: Simple and Fast Retrieval-Augmented Generation";
|
||||
# license = lib.licenses.mit;
|
||||
# };
|
||||
# };
|
||||
# pipmaster = prefab {
|
||||
# url = "https://github.com/ParisNeo/pipmaster/tree/820acdbc9d541443942bd8afd6ba968036bd8452";
|
||||
# hash = "sha256-IJ6VG4WvoEm584vbZDLTtRcEZekZh7HJto1hnABoodo=";
|
||||
# force.version = "1.1.0";
|
||||
# propagatedBuildDeps = [
|
||||
# ascii-colors
|
||||
# ];
|
||||
# meta = with lib; {
|
||||
# description = "A versatile Python package manager utility for simplifying package installation, updates, checks, and environment management.";
|
||||
# license = licenses.asl20;
|
||||
# };
|
||||
# };
|
||||
# nano-vectordb = prefab {
|
||||
# url = "https://github.com/gusye1234/nano-vectordb/tree/03f821348f04a93a9c36bb575faae05b61f4c02e";
|
||||
# hash = "sha256-jLco+1TAncF8Ep+VGd3DhsoiZTW7G/H8fAEwnNsovTY=";
|
||||
# force.version = "0.0.4.3";
|
||||
# propagatedBuildDeps = [
|
||||
# numpy
|
||||
# ];
|
||||
# meta = {
|
||||
# description = "A simple, easy-to-hack Vector Database implementation";
|
||||
# license = lib.licenses.mit;
|
||||
# };
|
||||
# };
|
||||
# ascii-colors = prefab {
|
||||
# url = "https://github.com/ParisNeo/ascii_colors/tree/817a21485136635e3da89ff08444183254b25aab";
|
||||
# hash = "sha256-jiolJzsBQMp9/DHdHTixJGvXoS19vAp/S7+8PyLARoo=";
|
||||
# force.version = "0.11.12";
|
||||
# propagatedBuildDeps = [
|
||||
# wcwidth
|
||||
# ];
|
||||
# meta = {
|
||||
# description = "A python library for displaying stuff on the console in a pretty way";
|
||||
# license = lib.licenses.asl20;
|
||||
# };
|
||||
# };
|
||||
# mineru = prefab {
|
||||
# url = "https://github.com/opendatalab/MinerU/tree/mineru-2.7.6-released";
|
||||
# hash = "sha256-A/nFNQYGEfmWUdpX8N1lbex3mdiF8+eN8s0UKQFc42E=";
|
||||
# propagatedBuildDeps = [
|
||||
# boto3
|
||||
# click
|
||||
# loguru
|
||||
# numpy
|
||||
# pdfminer-six
|
||||
# tqdm
|
||||
# requests
|
||||
# httpx
|
||||
# pillow
|
||||
# pypdfium2
|
||||
# pypdf
|
||||
# reportlab
|
||||
# pdftext
|
||||
# modelscope
|
||||
# huggingface-hub
|
||||
# json-repair
|
||||
# opencv-python
|
||||
# fast-langdetect
|
||||
# scikit-image
|
||||
# openai
|
||||
# beautifulsoup4
|
||||
# magika
|
||||
# mineru-vl-utils
|
||||
# qwen-vl-utils
|
||||
# ];
|
||||
# optional-dependencies = {
|
||||
# core = [ doclayout-yolo ]
|
||||
# # ++ (with mineru.optional-dependencies; [
|
||||
# # vlm
|
||||
# # pipeline
|
||||
# # api
|
||||
# # gradio
|
||||
# # ])
|
||||
# ;
|
||||
# vlm = [ ];
|
||||
# pipeline = [
|
||||
# doclayout-yolo
|
||||
# ];
|
||||
# api = [ ];
|
||||
# gradio = [ ];
|
||||
# };
|
||||
# pythonRelaxDeps = [
|
||||
# "fast-langdetect"
|
||||
# ];
|
||||
# meta = {
|
||||
# description = "Transforms complex documents like PDFs into LLM-ready markdown/JSON for your Agentic workflows.";
|
||||
# license = lib.licenses.gpl3;
|
||||
# };
|
||||
# };
|
||||
# doclayout-yolo = prefab {
|
||||
# # reason for creating fetcher construct
|
||||
# url = "https://github.com/opendatalab/DocLayout-YOLO/tree/main"; # master is 10 months old and last version was set in Oct 18, 2024
|
||||
# hash = "sha256-wLCmJuWyw7Gqou7awsHtxqVyr4AN+yJSkF0q3GUSQh8=";
|
||||
# force.version = "0.0.4"; # NOT 0.0.4 on github but fuck it we ball
|
||||
# meta = {
|
||||
# description = "DocLayout-YOLO: Enhancing Document Layout Analysis through Diverse Synthetic Data and Global-to-Local Adaptive Perception";
|
||||
# license = lib.licenses.agpl3Only;
|
||||
# };
|
||||
# };
|
||||
# qwen-vl-utils = prefab {
|
||||
# url = "https://github.com/QwenLM/Qwen3-VL/tree/main/qwen-vl-utils";
|
||||
# hash = "sha256-Vha/Tc4q2v5RCL31hB9U4ZrfIFxfZjgwM6PYlvDeoAQ=";
|
||||
# force = {
|
||||
# version = "0.0.14";
|
||||
# rev = "fe12058";
|
||||
# };
|
||||
# propagatedBuildDeps = [
|
||||
# hatchling
|
||||
# av
|
||||
# pillow
|
||||
# requests
|
||||
# torch
|
||||
# torchvision
|
||||
# ];
|
||||
# pythonImportsCheck = [ "qwen_vl_utils" ];
|
||||
# meta = {
|
||||
# description = "Qwen-VL Utils contains a set of helper functions for processing and integrating visual language information with Qwen-VL Series Model.";
|
||||
# license = lib.licenses.asl20;
|
||||
# };
|
||||
# };
|
||||
# mineru-vl-utils = prefab {
|
||||
# url = "https://github.com/opendatalab/mineru-vl-utils/tree/mineru_vl_utils-0.1.22-released";
|
||||
# hash = "sha256-hpTW/1nwXPxfld4nx0XHZBMerWj+UL1vzDhYpwjezRU=";
|
||||
# propagatedBuildDeps = [
|
||||
# httpx
|
||||
# httpx-retries
|
||||
# aiofiles
|
||||
# pillow
|
||||
# pydantic
|
||||
# loguru
|
||||
# ];
|
||||
# meta = {
|
||||
# description = "A Python package for interacting with the MinerU Vision-Language Model.";
|
||||
# license = lib.licenses.gpl3;
|
||||
# };
|
||||
# };
|
||||
# httpx-retries = prefab {
|
||||
# url = "https://github.com/will-ockmore/httpx-retries/tree/0.4.5";
|
||||
# hash = "sha256-zJ3ExSEWxlHFluSdYA8/XZ3zb4KBelU+IOFyUu4ezvo=";
|
||||
# propagatedBuildDeps = [
|
||||
# hatchling
|
||||
# hatch-fancy-pypi-readme
|
||||
|
||||
httpx
|
||||
];
|
||||
meta = {
|
||||
description = "A retry layer for HTTPX.";
|
||||
license = lib.licenses.mit;
|
||||
};
|
||||
};
|
||||
fast-langdetect = prefab {
|
||||
url = "https://github.com/LlmKira/fast-langdetect/tree/pypi_1.0.0";
|
||||
hash = "sha256-pj46gHG9cjkSjnYc88bSctL/1LAUe0jkBuM/GZWMsUI=";
|
||||
propagatedBuildDeps = [
|
||||
pdm-backend
|
||||
robust-downloader
|
||||
requests
|
||||
fasttext-predict
|
||||
];
|
||||
meta = {
|
||||
description = "⚡️ 80x faster Fasttext language detection out of the box | Split text by language ";
|
||||
license = lib.licenses.mit;
|
||||
};
|
||||
};
|
||||
robust-downloader = prefab {
|
||||
url = "https://github.com/fedebotu/robust-downloader/tree/0.0.2";
|
||||
hash = "sha256-UmzfEIPiMtUkOG6sIMYgLxc8YwL5wRgMBRlywqKomv0=";
|
||||
propagatedBuildDeps = [
|
||||
tqdm
|
||||
colorlog
|
||||
requests
|
||||
];
|
||||
meta = {
|
||||
description = "Minimal Python downloader with robustness in mind - resumable downloads, retries, and more";
|
||||
license = lib.licenses.asl20;
|
||||
};
|
||||
};
|
||||
pdftext = prefab {
|
||||
url = "https://github.com/datalab-to/pdftext/tree/v0.6.3";
|
||||
hash = "sha256-EGVjzjDWtdcEPX//cOm5+xm9FvX0aP+h6fsD25hC8gA=";
|
||||
propagatedBuildDeps = [
|
||||
poetry-core
|
||||
# httpx
|
||||
# ];
|
||||
# meta = {
|
||||
# description = "A retry layer for HTTPX.";
|
||||
# license = lib.licenses.mit;
|
||||
# };
|
||||
# };
|
||||
# fast-langdetect = prefab {
|
||||
# url = "https://github.com/LlmKira/fast-langdetect/tree/pypi_1.0.0";
|
||||
# hash = "sha256-pj46gHG9cjkSjnYc88bSctL/1LAUe0jkBuM/GZWMsUI=";
|
||||
# propagatedBuildDeps = [
|
||||
# pdm-backend
|
||||
# robust-downloader
|
||||
# requests
|
||||
# fasttext-predict
|
||||
# ];
|
||||
# meta = {
|
||||
# description = "⚡️ 80x faster Fasttext language detection out of the box | Split text by language ";
|
||||
# license = lib.licenses.mit;
|
||||
# };
|
||||
# };
|
||||
# robust-downloader = prefab {
|
||||
# url = "https://github.com/fedebotu/robust-downloader/tree/0.0.2";
|
||||
# hash = "sha256-UmzfEIPiMtUkOG6sIMYgLxc8YwL5wRgMBRlywqKomv0=";
|
||||
# propagatedBuildDeps = [
|
||||
# tqdm
|
||||
# colorlog
|
||||
# requests
|
||||
# ];
|
||||
# meta = {
|
||||
# description = "Minimal Python downloader with robustness in mind - resumable downloads, retries, and more";
|
||||
# license = lib.licenses.asl20;
|
||||
# };
|
||||
# };
|
||||
# pdftext = prefab {
|
||||
# url = "https://github.com/datalab-to/pdftext/tree/v0.6.3";
|
||||
# hash = "sha256-EGVjzjDWtdcEPX//cOm5+xm9FvX0aP+h6fsD25hC8gA=";
|
||||
# propagatedBuildDeps = [
|
||||
# poetry-core
|
||||
|
||||
click
|
||||
pydantic
|
||||
pydantic-settings
|
||||
pypdfium2
|
||||
];
|
||||
pythonRelaxDeps = [
|
||||
"pypdfium2"
|
||||
];
|
||||
meta = {
|
||||
description = "Extract structured text from pdfs quickly";
|
||||
license = lib.licenses.asl20;
|
||||
};
|
||||
};
|
||||
}
|
||||
# click
|
||||
# pydantic
|
||||
# pydantic-settings
|
||||
# pypdfium2
|
||||
# ];
|
||||
# pythonRelaxDeps = [
|
||||
# "pypdfium2"
|
||||
# ];
|
||||
# meta = {
|
||||
# description = "Extract structured text from pdfs quickly";
|
||||
# license = lib.licenses.asl20;
|
||||
# };
|
||||
# };
|
||||
# }
|
||||
|
|
|
|||
|
|
@ -2,13 +2,17 @@
|
|||
lib,
|
||||
fetchFromGitHub,
|
||||
fetchPypi,
|
||||
buildPythonPackage,
|
||||
python,
|
||||
setuptools,
|
||||
wheel,
|
||||
pyPkgs,
|
||||
}:
|
||||
|
||||
let
|
||||
inherit (pyPkgs)
|
||||
buildPythonPackage
|
||||
setuptools
|
||||
wheel
|
||||
python
|
||||
;
|
||||
|
||||
parseRef = import ./parse-source-ref.nix { inherit lib; };
|
||||
|
||||
mkPrefab =
|
||||
|
|
@ -70,45 +74,43 @@ let
|
|||
passthru' =
|
||||
passthru
|
||||
// lib.optionalAttrs (optional-dependencies != { }) {
|
||||
optional-dependencies = optional-dependencies;
|
||||
passthru.optional-dependencies = optional-dependencies;
|
||||
};
|
||||
in
|
||||
buildPythonPackage (
|
||||
({
|
||||
inherit pname version src;
|
||||
pyproject = true;
|
||||
buildPythonPackage {
|
||||
inherit pname version src;
|
||||
pyproject = true;
|
||||
|
||||
build-system = [
|
||||
setuptools
|
||||
wheel
|
||||
]
|
||||
++ nativeBuildDeps;
|
||||
dependencies = propagatedBuildDeps;
|
||||
pythonRelaxDeps = pythonRelaxDeps;
|
||||
build-system = [
|
||||
setuptools
|
||||
wheel
|
||||
]
|
||||
++ nativeBuildDeps;
|
||||
dependencies = propagatedBuildDeps;
|
||||
pythonRelaxDeps = pythonRelaxDeps;
|
||||
|
||||
pythonImportsCheck = imports;
|
||||
pythonImportsCheck = imports;
|
||||
|
||||
installCheckPhase = ''
|
||||
runHook preInstallCheck
|
||||
${builtins.concatStringsSep "\n" (
|
||||
map (mod: ''
|
||||
${python.interpreter} -c "import ${mod}; print(getattr(${mod},'__version__','unknown'))"
|
||||
'') imports
|
||||
)}
|
||||
runHook postInstallCheck
|
||||
'';
|
||||
installCheckPhase = ''
|
||||
runHook preInstallCheck
|
||||
${builtins.concatStringsSep "\n" (
|
||||
map (mod: ''
|
||||
${python.interpreter} -c "import ${mod}; print(getattr(${mod},'__version__','unknown'))"
|
||||
'') imports
|
||||
)}
|
||||
runHook postInstallCheck
|
||||
'';
|
||||
|
||||
passthru = passthru';
|
||||
passthru = passthru';
|
||||
|
||||
meta = {
|
||||
inherit homepage;
|
||||
}
|
||||
// meta;
|
||||
})
|
||||
// lib.optionalAttrs (subdir != null && subdir != "") {
|
||||
sourceRoot = "${sourceRootBase}/${subdir}";
|
||||
meta = {
|
||||
inherit homepage;
|
||||
}
|
||||
);
|
||||
// meta;
|
||||
}
|
||||
// lib.optionalAttrs (subdir != null && subdir != "") {
|
||||
sourceRoot = "${sourceRootBase}/${subdir}";
|
||||
};
|
||||
mkPrefabs = specs: lib.mapAttrs (_: spec: mkPrefab spec) specs;
|
||||
|
||||
mkPrefabsRec = specsFn: lib.fix (self: mkPrefabs (specsFn self));
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue