371 lines
9.8 KiB
Nix
371 lines
9.8 KiB
Nix
# {
|
|
# lib,
|
|
# fetchFromGitHub,
|
|
# buildPythonPackage,
|
|
# setuptools,
|
|
# wheel,
|
|
# hatchling,
|
|
# poetry-core,
|
|
|
|
# python,
|
|
|
|
# huggingface-hub,
|
|
# tqdm,
|
|
|
|
# aiohttp,
|
|
# configparser,
|
|
# google-api-core,
|
|
# google-genai,
|
|
# json-repair,
|
|
# networkx,
|
|
# numpy,
|
|
# pandas,
|
|
# pydantic,
|
|
# pypinyin,
|
|
# python-dotenv,
|
|
# tenacity,
|
|
# tiktoken,
|
|
# xlsxwriter,
|
|
|
|
# boto3,
|
|
# click,
|
|
# loguru,
|
|
# pdfminer-six,
|
|
# requests,
|
|
# httpx,
|
|
# pillow,
|
|
# pypdfium2,
|
|
# pypdf,
|
|
# reportlab,
|
|
# modelscope,
|
|
# opencv-python,
|
|
# scikit-image,
|
|
# openai,
|
|
# beautifulsoup4,
|
|
# magika,
|
|
|
|
# av,
|
|
|
|
# pdm-backend,
|
|
# fasttext-predict,
|
|
|
|
# pydantic-settings,
|
|
|
|
# aiofiles,
|
|
# hatch-fancy-pypi-readme,
|
|
|
|
# colorlog,
|
|
# torch,
|
|
# torchvision,
|
|
|
|
# wcwidth,
|
|
# }:
|
|
|
|
# let
|
|
# prefab =
|
|
# {
|
|
# url,
|
|
# hash,
|
|
# force ? { },
|
|
# meta ? { },
|
|
# propagatedBuildDeps ? [ ],
|
|
# nativeBuildDeps ? [ ],
|
|
# pythonRelaxDeps ? [ ],
|
|
# pythonImportsCheck ? null,
|
|
# optional-dependencies ? null,
|
|
# }:
|
|
# let
|
|
# x = (import ../../lib/parse-github-url.nix { inherit lib; }) url;
|
|
# pname = x.pname;
|
|
# rev = force.rev or x.rev;
|
|
# version = force.version or x.version;
|
|
|
|
# defaultImports = [ (lib.toLower (lib.replaceStrings [ "-" ] [ "_" ] pname)) ];
|
|
# imports = if pythonImportsCheck == null then defaultImports else pythonImportsCheck;
|
|
|
|
# rawSubdir = force.subdir or x.subdir or null;
|
|
# subdir = if rawSubdir == null then null else lib.removeSuffix "/" (lib.removePrefix "/" rawSubdir);
|
|
|
|
# src = fetchFromGitHub {
|
|
# owner = x.owner;
|
|
# repo = x.repo;
|
|
# rev = rev;
|
|
# hash = hash;
|
|
# };
|
|
# in
|
|
# buildPythonPackage (
|
|
# {
|
|
# inherit pname version src;
|
|
# pyproject = true;
|
|
|
|
# build-system = [
|
|
# setuptools
|
|
# wheel
|
|
# ]
|
|
# ++ nativeBuildDeps;
|
|
# dependencies = propagatedBuildDeps;
|
|
# optional-dependencies = optional-dependencies;
|
|
# pythonRelaxDeps = pythonRelaxDeps;
|
|
|
|
# pythonImportsCheck = imports;
|
|
|
|
# installCheckPhase = ''
|
|
# runHook preInstallCheck
|
|
# ${builtins.concatStringsSep "\n" (
|
|
# map (lib: ''
|
|
# ${python.interpreter} -c "import ${lib}; print(getattr(${lib},'__version__','unknown'))"
|
|
# '') imports
|
|
# )}
|
|
# runHook postInstallCheck
|
|
# '';
|
|
|
|
# meta = {
|
|
# homepage = x.homepage;
|
|
# }
|
|
# // meta;
|
|
# }
|
|
# // lib.optionalAttrs (subdir != null && subdir != "") {
|
|
# sourceRoot = "${src.name}/${subdir}";
|
|
# }
|
|
# );
|
|
# in
|
|
# rec {
|
|
# raganything = prefab {
|
|
# url = "https://github.com/HKUDS/RAG-Anything/tree/v1.2.9";
|
|
# hash = "sha256-yepiLYzPD6UcJRbAbovg/BwFE8nh903o/tHypiHGKSw=";
|
|
# propagatedBuildDeps = [
|
|
# huggingface-hub
|
|
# lightrag-hku
|
|
# tqdm
|
|
# # mineru
|
|
# ]
|
|
# ++ mineru.optional-dependencies.core;
|
|
# pythonImportsCheck = [ "raganything" ];
|
|
# meta = {
|
|
# description = "RAGAnything: All-in-One RAG System";
|
|
# license = lib.licenses.mit;
|
|
# };
|
|
# };
|
|
# lightrag-hku = prefab {
|
|
# url = "https://github.com/HKUDS/LightRAG/tree/v1.4.9.11";
|
|
# hash = "sha256-TX/HSF2ZqoDo2SRlFzM+bkUxQXiUpnrl6kwI+lljjdo=";
|
|
# propagatedBuildDeps = [
|
|
# aiohttp
|
|
# configparser
|
|
# google-api-core
|
|
# google-genai
|
|
# json-repair
|
|
# nano-vectordb
|
|
# networkx
|
|
# numpy
|
|
# pandas
|
|
# pipmaster
|
|
# pydantic
|
|
# pypinyin
|
|
# python-dotenv
|
|
# tenacity
|
|
# tiktoken
|
|
# xlsxwriter
|
|
# ];
|
|
# meta = {
|
|
# description = "LightRAG: Simple and Fast Retrieval-Augmented Generation";
|
|
# license = lib.licenses.mit;
|
|
# };
|
|
# };
|
|
# pipmaster = prefab {
|
|
# url = "https://github.com/ParisNeo/pipmaster/tree/820acdbc9d541443942bd8afd6ba968036bd8452";
|
|
# hash = "sha256-IJ6VG4WvoEm584vbZDLTtRcEZekZh7HJto1hnABoodo=";
|
|
# force.version = "1.1.0";
|
|
# propagatedBuildDeps = [
|
|
# ascii-colors
|
|
# ];
|
|
# meta = with lib; {
|
|
# description = "A versatile Python package manager utility for simplifying package installation, updates, checks, and environment management.";
|
|
# license = licenses.asl20;
|
|
# };
|
|
# };
|
|
# nano-vectordb = prefab {
|
|
# url = "https://github.com/gusye1234/nano-vectordb/tree/03f821348f04a93a9c36bb575faae05b61f4c02e";
|
|
# hash = "sha256-jLco+1TAncF8Ep+VGd3DhsoiZTW7G/H8fAEwnNsovTY=";
|
|
# force.version = "0.0.4.3";
|
|
# propagatedBuildDeps = [
|
|
# numpy
|
|
# ];
|
|
# meta = {
|
|
# description = "A simple, easy-to-hack Vector Database implementation";
|
|
# license = lib.licenses.mit;
|
|
# };
|
|
# };
|
|
# ascii-colors = prefab {
|
|
# url = "https://github.com/ParisNeo/ascii_colors/tree/817a21485136635e3da89ff08444183254b25aab";
|
|
# hash = "sha256-jiolJzsBQMp9/DHdHTixJGvXoS19vAp/S7+8PyLARoo=";
|
|
# force.version = "0.11.12";
|
|
# propagatedBuildDeps = [
|
|
# wcwidth
|
|
# ];
|
|
# meta = {
|
|
# description = "A python library for displaying stuff on the console in a pretty way";
|
|
# license = lib.licenses.asl20;
|
|
# };
|
|
# };
|
|
# mineru = prefab {
|
|
# url = "https://github.com/opendatalab/MinerU/tree/mineru-2.7.6-released";
|
|
# hash = "sha256-A/nFNQYGEfmWUdpX8N1lbex3mdiF8+eN8s0UKQFc42E=";
|
|
# propagatedBuildDeps = [
|
|
# boto3
|
|
# click
|
|
# loguru
|
|
# numpy
|
|
# pdfminer-six
|
|
# tqdm
|
|
# requests
|
|
# httpx
|
|
# pillow
|
|
# pypdfium2
|
|
# pypdf
|
|
# reportlab
|
|
# pdftext
|
|
# modelscope
|
|
# huggingface-hub
|
|
# json-repair
|
|
# opencv-python
|
|
# fast-langdetect
|
|
# scikit-image
|
|
# openai
|
|
# beautifulsoup4
|
|
# magika
|
|
# mineru-vl-utils
|
|
# qwen-vl-utils
|
|
# ];
|
|
# optional-dependencies = {
|
|
# core = [ doclayout-yolo ]
|
|
# # ++ (with mineru.optional-dependencies; [
|
|
# # vlm
|
|
# # pipeline
|
|
# # api
|
|
# # gradio
|
|
# # ])
|
|
# ;
|
|
# vlm = [ ];
|
|
# pipeline = [
|
|
# doclayout-yolo
|
|
# ];
|
|
# api = [ ];
|
|
# gradio = [ ];
|
|
# };
|
|
# pythonRelaxDeps = [
|
|
# "fast-langdetect"
|
|
# ];
|
|
# meta = {
|
|
# description = "Transforms complex documents like PDFs into LLM-ready markdown/JSON for your Agentic workflows.";
|
|
# license = lib.licenses.gpl3;
|
|
# };
|
|
# };
|
|
# doclayout-yolo = prefab {
|
|
# # reason for creating fetcher construct
|
|
# url = "https://github.com/opendatalab/DocLayout-YOLO/tree/main"; # master is 10 months old and last version was set in Oct 18, 2024
|
|
# hash = "sha256-wLCmJuWyw7Gqou7awsHtxqVyr4AN+yJSkF0q3GUSQh8=";
|
|
# force.version = "0.0.4"; # NOT 0.0.4 on github but fuck it we ball
|
|
# meta = {
|
|
# description = "DocLayout-YOLO: Enhancing Document Layout Analysis through Diverse Synthetic Data and Global-to-Local Adaptive Perception";
|
|
# license = lib.licenses.agpl3Only;
|
|
# };
|
|
# };
|
|
# qwen-vl-utils = prefab {
|
|
# url = "https://github.com/QwenLM/Qwen3-VL/tree/main/qwen-vl-utils";
|
|
# hash = "sha256-Vha/Tc4q2v5RCL31hB9U4ZrfIFxfZjgwM6PYlvDeoAQ=";
|
|
# force = {
|
|
# version = "0.0.14";
|
|
# rev = "fe12058";
|
|
# };
|
|
# propagatedBuildDeps = [
|
|
# hatchling
|
|
# av
|
|
# pillow
|
|
# requests
|
|
# torch
|
|
# torchvision
|
|
# ];
|
|
# pythonImportsCheck = [ "qwen_vl_utils" ];
|
|
# meta = {
|
|
# description = "Qwen-VL Utils contains a set of helper functions for processing and integrating visual language information with Qwen-VL Series Model.";
|
|
# license = lib.licenses.asl20;
|
|
# };
|
|
# };
|
|
# mineru-vl-utils = prefab {
|
|
# url = "https://github.com/opendatalab/mineru-vl-utils/tree/mineru_vl_utils-0.1.22-released";
|
|
# hash = "sha256-hpTW/1nwXPxfld4nx0XHZBMerWj+UL1vzDhYpwjezRU=";
|
|
# propagatedBuildDeps = [
|
|
# httpx
|
|
# httpx-retries
|
|
# aiofiles
|
|
# pillow
|
|
# pydantic
|
|
# loguru
|
|
# ];
|
|
# meta = {
|
|
# description = "A Python package for interacting with the MinerU Vision-Language Model.";
|
|
# license = lib.licenses.gpl3;
|
|
# };
|
|
# };
|
|
# httpx-retries = prefab {
|
|
# url = "https://github.com/will-ockmore/httpx-retries/tree/0.4.5";
|
|
# hash = "sha256-zJ3ExSEWxlHFluSdYA8/XZ3zb4KBelU+IOFyUu4ezvo=";
|
|
# propagatedBuildDeps = [
|
|
# hatchling
|
|
# hatch-fancy-pypi-readme
|
|
|
|
# httpx
|
|
# ];
|
|
# meta = {
|
|
# description = "A retry layer for HTTPX.";
|
|
# license = lib.licenses.mit;
|
|
# };
|
|
# };
|
|
# fast-langdetect = prefab {
|
|
# url = "https://github.com/LlmKira/fast-langdetect/tree/pypi_1.0.0";
|
|
# hash = "sha256-pj46gHG9cjkSjnYc88bSctL/1LAUe0jkBuM/GZWMsUI=";
|
|
# propagatedBuildDeps = [
|
|
# pdm-backend
|
|
# robust-downloader
|
|
# requests
|
|
# fasttext-predict
|
|
# ];
|
|
# meta = {
|
|
# description = "⚡️ 80x faster Fasttext language detection out of the box | Split text by language ";
|
|
# license = lib.licenses.mit;
|
|
# };
|
|
# };
|
|
# robust-downloader = prefab {
|
|
# url = "https://github.com/fedebotu/robust-downloader/tree/0.0.2";
|
|
# hash = "sha256-UmzfEIPiMtUkOG6sIMYgLxc8YwL5wRgMBRlywqKomv0=";
|
|
# propagatedBuildDeps = [
|
|
# tqdm
|
|
# colorlog
|
|
# requests
|
|
# ];
|
|
# meta = {
|
|
# description = "Minimal Python downloader with robustness in mind - resumable downloads, retries, and more";
|
|
# license = lib.licenses.asl20;
|
|
# };
|
|
# };
|
|
# pdftext = prefab {
|
|
# url = "https://github.com/datalab-to/pdftext/tree/v0.6.3";
|
|
# hash = "sha256-EGVjzjDWtdcEPX//cOm5+xm9FvX0aP+h6fsD25hC8gA=";
|
|
# propagatedBuildDeps = [
|
|
# poetry-core
|
|
|
|
# click
|
|
# pydantic
|
|
# pydantic-settings
|
|
# pypdfium2
|
|
# ];
|
|
# pythonRelaxDeps = [
|
|
# "pypdfium2"
|
|
# ];
|
|
# meta = {
|
|
# description = "Extract structured text from pdfs quickly";
|
|
# license = lib.licenses.asl20;
|
|
# };
|
|
# };
|
|
# }
|