moving to a cleaner python package registry, 1
This commit is contained in:
parent
585f17e085
commit
4223c18b84
5 changed files with 521 additions and 7 deletions
297
custom/pkgs/by-category/pythonPackages/prefabs-specs.nix
Normal file
297
custom/pkgs/by-category/pythonPackages/prefabs-specs.nix
Normal file
|
|
@ -0,0 +1,297 @@
|
|||
{
|
||||
lib,
|
||||
hatchling,
|
||||
poetry-core,
|
||||
|
||||
huggingface-hub,
|
||||
tqdm,
|
||||
|
||||
aiohttp,
|
||||
configparser,
|
||||
google-api-core,
|
||||
google-genai,
|
||||
json-repair,
|
||||
networkx,
|
||||
numpy,
|
||||
pandas,
|
||||
pydantic,
|
||||
pypinyin,
|
||||
python-dotenv,
|
||||
tenacity,
|
||||
tiktoken,
|
||||
xlsxwriter,
|
||||
|
||||
boto3,
|
||||
click,
|
||||
loguru,
|
||||
pdfminer-six,
|
||||
requests,
|
||||
httpx,
|
||||
pillow,
|
||||
pypdfium2,
|
||||
pypdf,
|
||||
reportlab,
|
||||
modelscope,
|
||||
opencv-python,
|
||||
scikit-image,
|
||||
openai,
|
||||
beautifulsoup4,
|
||||
magika,
|
||||
|
||||
av,
|
||||
|
||||
pdm-backend,
|
||||
fasttext-predict,
|
||||
|
||||
pydantic-settings,
|
||||
|
||||
aiofiles,
|
||||
hatch-fancy-pypi-readme,
|
||||
|
||||
colorlog,
|
||||
torch,
|
||||
torchvision,
|
||||
|
||||
wcwidth,
|
||||
}:
|
||||
|
||||
rec {
|
||||
raganything = {
|
||||
url = "https://github.com/HKUDS/RAG-Anything/tree/v1.2.9";
|
||||
hash = "sha256-yepiLYzPD6UcJRbAbovg/BwFE8nh903o/tHypiHGKSw=";
|
||||
propagatedBuildDeps = [
|
||||
huggingface-hub
|
||||
lightrag-hku
|
||||
tqdm
|
||||
# mineru
|
||||
]
|
||||
++ mineru.optional-dependencies.core;
|
||||
pythonImportsCheck = [ "raganything" ];
|
||||
meta = {
|
||||
description = "RAGAnything: All-in-One RAG System";
|
||||
license = lib.licenses.mit;
|
||||
};
|
||||
};
|
||||
lightrag-hku = {
|
||||
url = "https://github.com/HKUDS/LightRAG/tree/v1.4.9.11";
|
||||
hash = "sha256-TX/HSF2ZqoDo2SRlFzM+bkUxQXiUpnrl6kwI+lljjdo=";
|
||||
propagatedBuildDeps = [
|
||||
aiohttp
|
||||
configparser
|
||||
google-api-core
|
||||
google-genai
|
||||
json-repair
|
||||
nano-vectordb
|
||||
networkx
|
||||
numpy
|
||||
pandas
|
||||
pipmaster
|
||||
pydantic
|
||||
pypinyin
|
||||
python-dotenv
|
||||
tenacity
|
||||
tiktoken
|
||||
xlsxwriter
|
||||
];
|
||||
meta = {
|
||||
description = "LightRAG: Simple and Fast Retrieval-Augmented Generation";
|
||||
license = lib.licenses.mit;
|
||||
};
|
||||
};
|
||||
pipmaster = {
|
||||
url = "https://github.com/ParisNeo/pipmaster/tree/820acdbc9d541443942bd8afd6ba968036bd8452";
|
||||
hash = "sha256-IJ6VG4WvoEm584vbZDLTtRcEZekZh7HJto1hnABoodo=";
|
||||
force.version = "1.1.0";
|
||||
propagatedBuildDeps = [
|
||||
ascii-colors
|
||||
];
|
||||
meta = with lib; {
|
||||
description = "A versatile Python package manager utility for simplifying package installation, updates, checks, and environment management.";
|
||||
license = licenses.asl20;
|
||||
};
|
||||
};
|
||||
nano-vectordb = {
|
||||
url = "https://github.com/gusye1234/nano-vectordb/tree/03f821348f04a93a9c36bb575faae05b61f4c02e";
|
||||
hash = "sha256-jLco+1TAncF8Ep+VGd3DhsoiZTW7G/H8fAEwnNsovTY=";
|
||||
force.version = "0.0.4.3";
|
||||
propagatedBuildDeps = [
|
||||
numpy
|
||||
];
|
||||
meta = {
|
||||
description = "A simple, easy-to-hack Vector Database implementation";
|
||||
license = lib.licenses.mit;
|
||||
};
|
||||
};
|
||||
ascii-colors = {
|
||||
url = "https://github.com/ParisNeo/ascii_colors/tree/817a21485136635e3da89ff08444183254b25aab";
|
||||
hash = "sha256-jiolJzsBQMp9/DHdHTixJGvXoS19vAp/S7+8PyLARoo=";
|
||||
force.version = "0.11.12";
|
||||
propagatedBuildDeps = [
|
||||
wcwidth
|
||||
];
|
||||
meta = {
|
||||
description = "A python library for displaying stuff on the console in a pretty way";
|
||||
license = lib.licenses.asl20;
|
||||
};
|
||||
};
|
||||
mineru = {
|
||||
url = "https://github.com/opendatalab/MinerU/tree/mineru-2.7.6-released";
|
||||
hash = "sha256-A/nFNQYGEfmWUdpX8N1lbex3mdiF8+eN8s0UKQFc42E=";
|
||||
propagatedBuildDeps = [
|
||||
boto3
|
||||
click
|
||||
loguru
|
||||
numpy
|
||||
pdfminer-six
|
||||
tqdm
|
||||
requests
|
||||
httpx
|
||||
pillow
|
||||
pypdfium2
|
||||
pypdf
|
||||
reportlab
|
||||
pdftext
|
||||
modelscope
|
||||
huggingface-hub
|
||||
json-repair
|
||||
opencv-python
|
||||
fast-langdetect
|
||||
scikit-image
|
||||
openai
|
||||
beautifulsoup4
|
||||
magika
|
||||
mineru-vl-utils
|
||||
qwen-vl-utils
|
||||
];
|
||||
optional-dependencies = {
|
||||
core = [ doclayout-yolo ]
|
||||
# ++ (with mineru.optional-dependencies; [
|
||||
# vlm
|
||||
# pipeline
|
||||
# api
|
||||
# gradio
|
||||
# ])
|
||||
;
|
||||
vlm = [ ];
|
||||
pipeline = [
|
||||
doclayout-yolo
|
||||
];
|
||||
api = [ ];
|
||||
gradio = [ ];
|
||||
};
|
||||
pythonRelaxDeps = [
|
||||
"fast-langdetect"
|
||||
];
|
||||
meta = {
|
||||
description = "Transforms complex documents like PDFs into LLM-ready markdown/JSON for your Agentic workflows.";
|
||||
license = lib.licenses.gpl3;
|
||||
};
|
||||
};
|
||||
doclayout-yolo = {
|
||||
# reason for creating fetcher construct
|
||||
url = "https://github.com/opendatalab/DocLayout-YOLO/tree/main"; # master is 10 months old and last version was set in Oct 18, 2024
|
||||
hash = "sha256-wLCmJuWyw7Gqou7awsHtxqVyr4AN+yJSkF0q3GUSQh8=";
|
||||
force.version = "0.0.4"; # NOT 0.0.4 on github but fuck it we ball
|
||||
meta = {
|
||||
description = "DocLayout-YOLO: Enhancing Document Layout Analysis through Diverse Synthetic Data and Global-to-Local Adaptive Perception";
|
||||
license = lib.licenses.agpl3Only;
|
||||
};
|
||||
};
|
||||
qwen-vl-utils = {
|
||||
url = "https://github.com/QwenLM/Qwen3-VL/tree/main/qwen-vl-utils";
|
||||
hash = "sha256-Vha/Tc4q2v5RCL31hB9U4ZrfIFxfZjgwM6PYlvDeoAQ=";
|
||||
force = {
|
||||
version = "0.0.14";
|
||||
rev = "fe12058";
|
||||
};
|
||||
propagatedBuildDeps = [
|
||||
hatchling
|
||||
av
|
||||
pillow
|
||||
requests
|
||||
torch
|
||||
torchvision
|
||||
];
|
||||
pythonImportsCheck = [ "qwen_vl_utils" ];
|
||||
meta = {
|
||||
description = "Qwen-VL Utils contains a set of helper functions for processing and integrating visual language information with Qwen-VL Series Model.";
|
||||
license = lib.licenses.asl20;
|
||||
};
|
||||
};
|
||||
mineru-vl-utils = {
|
||||
url = "https://github.com/opendatalab/mineru-vl-utils/tree/mineru_vl_utils-0.1.22-released";
|
||||
hash = "sha256-hpTW/1nwXPxfld4nx0XHZBMerWj+UL1vzDhYpwjezRU=";
|
||||
propagatedBuildDeps = [
|
||||
httpx
|
||||
httpx-retries
|
||||
aiofiles
|
||||
pillow
|
||||
pydantic
|
||||
loguru
|
||||
];
|
||||
meta = {
|
||||
description = "A Python package for interacting with the MinerU Vision-Language Model.";
|
||||
license = lib.licenses.gpl3;
|
||||
};
|
||||
};
|
||||
httpx-retries = {
|
||||
url = "https://github.com/will-ockmore/httpx-retries/tree/0.4.5";
|
||||
hash = "sha256-zJ3ExSEWxlHFluSdYA8/XZ3zb4KBelU+IOFyUu4ezvo=";
|
||||
propagatedBuildDeps = [
|
||||
hatchling
|
||||
hatch-fancy-pypi-readme
|
||||
|
||||
httpx
|
||||
];
|
||||
meta = {
|
||||
description = "A retry layer for HTTPX.";
|
||||
license = lib.licenses.mit;
|
||||
};
|
||||
};
|
||||
fast-langdetect = {
|
||||
url = "https://github.com/LlmKira/fast-langdetect/tree/pypi_1.0.0";
|
||||
hash = "sha256-pj46gHG9cjkSjnYc88bSctL/1LAUe0jkBuM/GZWMsUI=";
|
||||
propagatedBuildDeps = [
|
||||
pdm-backend
|
||||
robust-downloader
|
||||
requests
|
||||
fasttext-predict
|
||||
];
|
||||
meta = {
|
||||
description = "⚡️ 80x faster Fasttext language detection out of the box | Split text by language ";
|
||||
license = lib.licenses.mit;
|
||||
};
|
||||
};
|
||||
robust-downloader = {
|
||||
url = "https://github.com/fedebotu/robust-downloader/tree/0.0.2";
|
||||
hash = "sha256-UmzfEIPiMtUkOG6sIMYgLxc8YwL5wRgMBRlywqKomv0=";
|
||||
propagatedBuildDeps = [
|
||||
tqdm
|
||||
colorlog
|
||||
requests
|
||||
];
|
||||
meta = {
|
||||
description = "Minimal Python downloader with robustness in mind - resumable downloads, retries, and more";
|
||||
license = lib.licenses.asl20;
|
||||
};
|
||||
};
|
||||
pdftext = {
|
||||
url = "https://github.com/datalab-to/pdftext/tree/v0.6.3";
|
||||
hash = "sha256-EGVjzjDWtdcEPX//cOm5+xm9FvX0aP+h6fsD25hC8gA=";
|
||||
propagatedBuildDeps = [
|
||||
poetry-core
|
||||
|
||||
click
|
||||
pydantic
|
||||
pydantic-settings
|
||||
pypdfium2
|
||||
];
|
||||
pythonRelaxDeps = [
|
||||
"pypdfium2"
|
||||
];
|
||||
meta = {
|
||||
description = "Extract structured text from pdfs quickly";
|
||||
license = lib.licenses.asl20;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -72,6 +72,7 @@ let
|
|||
nativeBuildDeps ? [ ],
|
||||
pythonRelaxDeps ? [ ],
|
||||
pythonImportsCheck ? null,
|
||||
optional-dependencies ? null,
|
||||
}:
|
||||
let
|
||||
x = (import ../../lib/parse-github-url.nix { inherit lib; }) url;
|
||||
|
|
@ -103,6 +104,7 @@ let
|
|||
]
|
||||
++ nativeBuildDeps;
|
||||
dependencies = propagatedBuildDeps;
|
||||
optional-dependencies = optional-dependencies;
|
||||
pythonRelaxDeps = pythonRelaxDeps;
|
||||
|
||||
pythonImportsCheck = imports;
|
||||
|
|
@ -134,9 +136,10 @@ rec {
|
|||
propagatedBuildDeps = [
|
||||
huggingface-hub
|
||||
lightrag-hku
|
||||
mineru
|
||||
tqdm
|
||||
];
|
||||
# mineru
|
||||
]
|
||||
++ mineru.optional-dependencies.core;
|
||||
pythonImportsCheck = [ "raganything" ];
|
||||
meta = {
|
||||
description = "RAGAnything: All-in-One RAG System";
|
||||
|
|
@ -234,6 +237,22 @@ rec {
|
|||
mineru-vl-utils
|
||||
qwen-vl-utils
|
||||
];
|
||||
optional-dependencies = {
|
||||
core = [ doclayout-yolo ]
|
||||
# ++ (with mineru.optional-dependencies; [
|
||||
# vlm
|
||||
# pipeline
|
||||
# api
|
||||
# gradio
|
||||
# ])
|
||||
;
|
||||
vlm = [ ];
|
||||
pipeline = [
|
||||
doclayout-yolo
|
||||
];
|
||||
api = [ ];
|
||||
gradio = [ ];
|
||||
};
|
||||
pythonRelaxDeps = [
|
||||
"fast-langdetect"
|
||||
];
|
||||
|
|
@ -242,6 +261,16 @@ rec {
|
|||
license = lib.licenses.gpl3;
|
||||
};
|
||||
};
|
||||
doclayout-yolo = prefab {
|
||||
# reason for creating fetcher construct
|
||||
url = "https://github.com/opendatalab/DocLayout-YOLO/tree/main"; # master is 10 months old and last version was set in Oct 18, 2024
|
||||
hash = "sha256-wLCmJuWyw7Gqou7awsHtxqVyr4AN+yJSkF0q3GUSQh8=";
|
||||
force.version = "0.0.4"; # NOT 0.0.4 on github but fuck it we ball
|
||||
meta = {
|
||||
description = "DocLayout-YOLO: Enhancing Document Layout Analysis through Diverse Synthetic Data and Global-to-Local Adaptive Perception";
|
||||
license = lib.licenses.agpl3Only;
|
||||
};
|
||||
};
|
||||
qwen-vl-utils = prefab {
|
||||
url = "https://github.com/QwenLM/Qwen3-VL/tree/main/qwen-vl-utils";
|
||||
hash = "sha256-Vha/Tc4q2v5RCL31hB9U4ZrfIFxfZjgwM6PYlvDeoAQ=";
|
||||
|
|
|
|||
|
|
@ -2,11 +2,9 @@
|
|||
|
||||
url:
|
||||
let
|
||||
# убрать ?query/#fragment и финальный /
|
||||
u0 = builtins.head (builtins.split "[?#]" url);
|
||||
u = lib.strings.removeSuffix "/" u0;
|
||||
|
||||
# owner / repo / (tree|tag|commit) / rev / (optional / subdir...)
|
||||
m = builtins.match "https://github.com/([^/]+)/([^/]+)(/(tree|tag|commit)/([^/]+)(/(.*))?)?" u;
|
||||
in
|
||||
assert m != null;
|
||||
|
|
@ -14,10 +12,10 @@ let
|
|||
owner = builtins.elemAt m 0;
|
||||
repo = builtins.elemAt m 1;
|
||||
|
||||
kind = builtins.elemAt m 3; # null если нет /tree|tag|commit
|
||||
rev = builtins.elemAt m 4; # null если нет
|
||||
kind = builtins.elemAt m 3;
|
||||
rev = builtins.elemAt m 4;
|
||||
|
||||
subdir = builtins.elemAt m 6; # null или "qwen-vl-utils" или "a/b/c"
|
||||
subdir = builtins.elemAt m 6;
|
||||
in
|
||||
{
|
||||
inherit
|
||||
|
|
|
|||
75
custom/pkgs/lib/parse-source-ref.nix
Normal file
75
custom/pkgs/lib/parse-source-ref.nix
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
{ lib }:
|
||||
|
||||
url:
|
||||
let
|
||||
# strip ?query/#fragment and trailing /
|
||||
u0 = builtins.head (builtins.split "[?#]" url);
|
||||
u = lib.removeSuffix "/" u0;
|
||||
|
||||
# GitHub: https://github.com/owner/repo/(tree|tag|commit)/rev/(optional/subdir)
|
||||
gh = builtins.match "https://github.com/([^/]+)/([^/]+)(/(tree|tag|commit)/([^/]+)(/(.*))?)?" u;
|
||||
|
||||
# PyPI project page: https://pypi.org/project/name/version
|
||||
pypiProject = builtins.match "https://pypi.org/project/([^/]+)(/([^/]+))?" u;
|
||||
|
||||
# PyPI short form:
|
||||
# pypi:requests@2.32.3
|
||||
# pypi:requests==2.32.3
|
||||
# pypi:requests/2.32.3
|
||||
pypiShort = builtins.match "^pypi:([^@=/]+)(==|@|/)?([^/]+)?$" u;
|
||||
in
|
||||
if gh != null then
|
||||
let
|
||||
owner = builtins.elemAt gh 0;
|
||||
repo = builtins.elemAt gh 1;
|
||||
|
||||
kind = builtins.elemAt gh 3; # null if absent
|
||||
rev = builtins.elemAt gh 4; # null if absent
|
||||
subdir = builtins.elemAt gh 6; # null or "a/b"
|
||||
|
||||
version =
|
||||
if rev == null then
|
||||
null
|
||||
else
|
||||
let
|
||||
vm = builtins.match "^.*([0-9]+(\\.[0-9]+)+).*$" rev;
|
||||
in
|
||||
if vm == null then null else builtins.elemAt vm 0;
|
||||
in
|
||||
{
|
||||
type = "github";
|
||||
pname = repo;
|
||||
homepage = "https://github.com/${owner}/${repo}";
|
||||
inherit
|
||||
owner
|
||||
repo
|
||||
kind
|
||||
rev
|
||||
subdir
|
||||
version
|
||||
;
|
||||
}
|
||||
else if pypiShort != null then
|
||||
let
|
||||
pname = builtins.elemAt pypiShort 0;
|
||||
version = builtins.elemAt pypiShort 2; # may be null
|
||||
in
|
||||
{
|
||||
type = "pypi";
|
||||
homepage = "https://pypi.org/project/${pname}/";
|
||||
inherit pname version;
|
||||
subdir = null;
|
||||
}
|
||||
else if pypiProject != null then
|
||||
let
|
||||
pname = builtins.elemAt pypiProject 0;
|
||||
version = builtins.elemAt pypiProject 2; # may be null
|
||||
in
|
||||
{
|
||||
type = "pypi";
|
||||
homepage = "https://pypi.org/project/${pname}/";
|
||||
inherit pname version;
|
||||
subdir = null;
|
||||
}
|
||||
else
|
||||
throw "parse-source-ref: unsupported url: ${url}"
|
||||
115
custom/pkgs/lib/prefab.nix
Normal file
115
custom/pkgs/lib/prefab.nix
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
{
|
||||
lib,
|
||||
fetchFromGitHub,
|
||||
fetchPypi,
|
||||
buildPythonPackage,
|
||||
python,
|
||||
setuptools,
|
||||
wheel,
|
||||
}:
|
||||
|
||||
let
|
||||
parseRef = import ./parse-source-ref.nix { inherit lib; };
|
||||
|
||||
mkPrefab =
|
||||
{
|
||||
url,
|
||||
hash,
|
||||
force ? { }, # overrides: pname/version/rev/subdir/sourceRootBase/homepage/src
|
||||
meta ? { },
|
||||
propagatedBuildDeps ? [ ],
|
||||
nativeBuildDeps ? [ ],
|
||||
pythonRelaxDeps ? [ ],
|
||||
pythonImportsCheck ? null,
|
||||
optional-dependencies ? null,
|
||||
}:
|
||||
let
|
||||
ref = parseRef url;
|
||||
|
||||
pname = force.pname or ref.pname;
|
||||
|
||||
version0 = force.version or ref.version or null;
|
||||
version = if version0 != null then version0 else "unstable";
|
||||
|
||||
rawSubdir = force.subdir or ref.subdir or null;
|
||||
subdir = if rawSubdir == null then null else lib.removeSuffix "/" (lib.removePrefix "/" rawSubdir);
|
||||
|
||||
rev = force.rev or ref.rev or null;
|
||||
|
||||
src =
|
||||
if (force.src or null) != null then
|
||||
force.src
|
||||
else if ref.type == "github" then
|
||||
(
|
||||
assert rev != null;
|
||||
fetchFromGitHub {
|
||||
owner = ref.owner;
|
||||
repo = ref.repo;
|
||||
inherit rev hash;
|
||||
}
|
||||
)
|
||||
else if ref.type == "pypi" then
|
||||
(
|
||||
assert (force.version or ref.version or null) != null;
|
||||
fetchPypi {
|
||||
inherit pname version;
|
||||
inherit hash;
|
||||
}
|
||||
)
|
||||
else
|
||||
throw "mkPrefab: unsupported ref.type=${ref.type}";
|
||||
|
||||
sourceRootBase =
|
||||
force.sourceRootBase or (if ref.type == "github" then src.name else "${pname}-${version}");
|
||||
|
||||
defaultImports = [ (lib.toLower (lib.replaceStrings [ "-" ] [ "_" ] pname)) ];
|
||||
imports = if pythonImportsCheck == null then defaultImports else pythonImportsCheck;
|
||||
|
||||
homepage = force.homepage or ref.homepage;
|
||||
in
|
||||
buildPythonPackage (
|
||||
(
|
||||
{
|
||||
inherit pname version src;
|
||||
pyproject = true;
|
||||
|
||||
build-system = [
|
||||
setuptools
|
||||
wheel
|
||||
]
|
||||
++ nativeBuildDeps;
|
||||
dependencies = propagatedBuildDeps;
|
||||
pythonRelaxDeps = pythonRelaxDeps;
|
||||
pythonImportsCheck = imports;
|
||||
|
||||
installCheckPhase = ''
|
||||
runHook preInstallCheck
|
||||
${builtins.concatStringsSep "\n" (
|
||||
map (mod: ''
|
||||
${python.interpreter} -c "import ${mod}; print(getattr(${mod},'__version__','unknown'))"
|
||||
'') imports
|
||||
)}
|
||||
runHook postInstallCheck
|
||||
'';
|
||||
|
||||
meta = {
|
||||
inherit homepage;
|
||||
}
|
||||
// meta;
|
||||
}
|
||||
// lib.optionalAttrs (optional-dependencies != null) {
|
||||
optional-dependencies = optional-dependencies;
|
||||
}
|
||||
)
|
||||
// lib.optionalAttrs (subdir != null && subdir != "") {
|
||||
sourceRoot = "${sourceRootBase}/${subdir}";
|
||||
}
|
||||
);
|
||||
|
||||
# 핵: specs = { pkgA = { url=...; hash=...; ...; }; ...; }
|
||||
# -> { pkgA = mkPrefab specs.pkgA; ...; }
|
||||
mkPrefabs = specs: lib.mapAttrs (_: spec: mkPrefab spec) specs;
|
||||
in
|
||||
{
|
||||
inherit mkPrefab mkPrefabs;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue