From b07cb15af93210690d54fc9353c56bd974ce4729 Mon Sep 17 00:00:00 2001 From: TheK0tYaRa Date: Thu, 12 Feb 2026 21:41:42 +0200 Subject: [PATCH] python3Packages fix attempt for haystack-experimental --- configuration.nix | 12 +- custom/overlays/default.nix | 7 +- custom/overlays/override.nix | 55 ++++ .../ha/haystack-experimental/default.nix | 272 ++++++++++++++++++ custom/pkgs/default.nix | 50 +++- flake.lock | 12 +- flake.nix | 106 +++---- 7 files changed, 438 insertions(+), 76 deletions(-) create mode 100644 custom/overlays/override.nix create mode 100644 custom/pkgs/by-category/python3Packages/ha/haystack-experimental/default.nix diff --git a/configuration.nix b/configuration.nix index a00266f..c76521b 100644 --- a/configuration.nix +++ b/configuration.nix @@ -221,12 +221,12 @@ torsocks = { enable = true; }; - xrdp = { - # enable = true; - audio.enable = true; - sslKey = "/secrets/xrdp/key.pem"; # TODO move those - sslCert = "/secrets/xrdp/cert.pem"; # TODO move those - }; + }; + xrdp = { + # enable = true; + audio.enable = true; + sslKey = "/secrets/xrdp/key.pem"; # TODO move those + sslCert = "/secrets/xrdp/cert.pem"; # TODO move those }; }; diff --git a/custom/overlays/default.nix b/custom/overlays/default.nix index 27a89c2..65295f2 100644 --- a/custom/overlays/default.nix +++ b/custom/overlays/default.nix @@ -1 +1,6 @@ -final: prev: (import ../pkgs { pkgs = final; lib = prev.lib; }) +final: prev: +import ../pkgs { + pkgs = final; + prev = prev; + lib = prev.lib; +} diff --git a/custom/overlays/override.nix b/custom/overlays/override.nix new file mode 100644 index 0000000..eaa3ee8 --- /dev/null +++ b/custom/overlays/override.nix @@ -0,0 +1,55 @@ +final: prev: { + llama-cpp = prev.llama-cpp.overrideAttrs (old: { + src = prev.fetchFromGitHub { + owner = "ggml-org"; + repo = "llama.cpp"; + tag = "b7897"; + hash = "sha256-cc2tqEQEZCBEOQ+xJzmVK8ROjfsWOi6T2iX3KjpaAIU="; + }; + nativeBuildInputs = old.nativeBuildInputs ++ [ old.pkgs.curl ]; + }); + pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [ + (pyFinal: pyPrev: { + haystack-ai = pyPrev.haystack-ai.overridePythonAttrs (old: rec { + version = "2.23.0"; + + src = final.fetchFromGitHub { + owner = "deepset-ai"; + repo = "haystack"; + rev = "v${version}"; + hash = "sha256-KFgRKdhbHgjtpiuAeFq+0t0EUKKLOavbVplb0QnI/r8="; + }; + + propagatedBuildInputs = + (old.propagatedBuildInputs or [ ]) + ++ (with pyPrev; [ + docstring-parser + filetype + jinja2 + openai + ]) + ++ [ + pyFinal."haystack-experimental" + ]; + + meta = (old.meta or { }) // { + broken = false; + }; + }); + }) + ]; + winetricks = prev.winetricks.overrideAttrs ( + old: + let + version = "20260125"; + in + { + inherit version; + + src = final.fetchurl { + url = "https://github.com/Winetricks/winetricks/archive/refs/tags/${version}.tar.gz"; + hash = "sha256-KJC9n7ut5GOOWLSZmiNycxkt8DtYUWrnuHceCcItL1Y="; + }; + } + ); +} diff --git a/custom/pkgs/by-category/python3Packages/ha/haystack-experimental/default.nix b/custom/pkgs/by-category/python3Packages/ha/haystack-experimental/default.nix new file mode 100644 index 0000000..517bffc --- /dev/null +++ b/custom/pkgs/by-category/python3Packages/ha/haystack-experimental/default.nix @@ -0,0 +1,272 @@ +{ + lib, + buildPythonPackage, + fetchFromGitHub, + # hatchling, + # boilerpy3, + # events, + # httpx, + # jsonschema, + # lazy-imports, + # more-itertools, + # networkx, + # pandas, + # pillow, + # platformdirs, + # posthog, + # prompthub-py, + pydantic, + # quantulum3, + # rank-bm25, + # requests, + # requests-cache, + # scikit-learn, + # sseclient-py, + # tenacity, + # tiktoken, + # tqdm, + # transformers, + # openai-whisper, + # boto3, + # botocore, + # # , beir + # selenium, + # coverage, + # dulwich, + # # , jupytercontrib + # mkdocs, + # mypy, + # pre-commit, + # psutil, + # # , pydoc-markdown + # pylint, + # pytest, + # pytest-asyncio, + # pytest-cov, + # # , pytest-custom-exit-code + # python-multipart, + # reno, + # responses, + # toml, + # tox, + # watchdog, + # elastic-transport, + # elasticsearch, + # # , azure-ai-formrecognizer + # beautifulsoup4, + # markdown, + # python-docx, + # python-frontmatter, + # python-magic, + # tika, + # black, + # huggingface-hub, + # sentence-transformers, + # mlflow, + # rapidfuzz, + # scipy, + # seqeval, + # pdf2image, + # pytesseract, + # faiss, + # # , faiss-gpu + # pinecone-client, + # onnxruntime, + # onnxruntime-tools, + # # , onnxruntime-gpu + # opensearch-py, + # pymupdf, + # langdetect, + # nltk, + # canals, + # jinja2, + # openai, + # aiorwlock, + # ray, + # psycopg2, + # sqlalchemy, + # sqlalchemy-utils, + # weaviate-client, +}: + +buildPythonPackage rec { + pname = "haystack-experimental"; + version = "0.19.0"; + pyproject = true; + + src = fetchFromGitHub { + owner = "deepset-ai"; + repo = "haystack-experimental"; + tag = "v${version}"; + hash = ""; + }; + + # nativeBuildInputs = [ + # hatchling + # ]; + + # pythonRemoveDeps = [ + # # We call it faiss, not faiss-cpu. + # "faiss-cpu" + # ]; + + propagatedBuildInputs = [ + # boilerpy3 + # events + # httpx + # jsonschema + # lazy-imports + # more-itertools + # networkx + # pandas + # pillow + # platformdirs + # posthog + # prompthub-py + # pydantic + # quantulum3 + # rank-bm25 + # requests + # requests-cache + # scikit-learn + # sseclient-py + # tenacity + # tiktoken + # tqdm + # transformers + ]; + + env.HOME = "$(mktemp -d)"; + + # optional-dependencies = { + # # all = [ + # # farm-haystack + # # ]; + # # all-gpu = [ + # # farm-haystack + # # ]; + # audio = [ openai-whisper ]; + # aws = [ + # boto3 + # botocore + # ]; + # # beir = [ + # # beir + # # ]; + # colab = [ pillow ]; + # crawler = [ selenium ]; + # dev = [ + # coverage + # dulwich + # # jupytercontrib + # mkdocs + # mypy + # pre-commit + # psutil + # # pydoc-markdown + # pylint + # pytest + # pytest-asyncio + # pytest-cov + # # pytest-custom-exit-code + # python-multipart + # reno + # responses + # toml + # tox + # watchdog + # ]; + # elasticsearch7 = [ + # elastic-transport + # elasticsearch + # ]; + # elasticsearch8 = [ + # elastic-transport + # elasticsearch + # ]; + # file-conversion = [ + # # azure-ai-formrecognizer + # beautifulsoup4 + # markdown + # python-docx + # python-frontmatter + # python-magic + # # python-magic-bin + # tika + # ]; + # formatting = [ black ]; + # inference = [ + # huggingface-hub + # sentence-transformers + # transformers + # ]; + # metrics = [ + # mlflow + # rapidfuzz + # scipy + # seqeval + # ]; + # ocr = [ + # pdf2image + # pytesseract + # ]; + # only-faiss = [ faiss ]; + # # only-faiss-gpu = [ + # # faiss-gpu + # # ]; + # only-pinecone = [ pinecone-client ]; + # onnx = [ + # onnxruntime + # onnxruntime-tools + # ]; + # # onnx-gpu = [ + # # onnxruntime-gpu + # # onnxruntime-tools + # # ]; + # opensearch = [ opensearch-py ]; + # pdf = [ pymupdf ]; + # preprocessing = [ + # langdetect + # nltk + # ]; + # preview = [ + # canals + # jinja2 + # lazy-imports + # openai + # pandas + # rank-bm25 + # requests + # tenacity + # tqdm + # ]; + # ray = [ + # aiorwlock + # ray + # ]; + # sql = [ + # psycopg2 + # sqlalchemy + # sqlalchemy-utils + # ]; + # weaviate = [ weaviate-client ]; + # }; + + # the setup for test is intensive, hopefully can be done at some point + doCheck = false; + + # pythonImportsCheck = [ "haystack" ]; + + meta = { + description = "Experimental features for Haystack"; + longDescription = '' + Experimental features for Haystack + ''; + changelog = "https://github.com/deepset-ai/haystack-experimental/releases/tag/${src.tag}"; + homepage = "https://github.com/deepset-ai/haystack-experimental"; + license = lib.licenses.asl20; + maintainers = with lib.maintainers; [ happysalada ]; + # https://github.com/deepset-ai/haystack/issues/5304 + broken = lib.versionAtLeast pydantic.version "2"; + }; +} diff --git a/custom/pkgs/default.nix b/custom/pkgs/default.nix index bbed60d..f9ea691 100644 --- a/custom/pkgs/default.nix +++ b/custom/pkgs/default.nix @@ -1,25 +1,49 @@ -{ pkgs, lib, ... }: +{ + pkgs, + prev, + lib, + ... +}: let - byName = ./by-name; - dirs = path: - lib.attrNames (lib.filterAttrs (_: t: t == "directory") (builtins.readDir path)); + dirs = path: lib.attrNames (lib.filterAttrs (_: t: t == "directory") (builtins.readDir path)); - packagesFromByName = base: - lib.foldl' (acc: shorthand: + packagesFromTree = + base: callPackage: + lib.foldl' ( + acc: shorthand: let shorthandDir = base + "/${shorthand}"; names = dirs shorthandDir; - addOne = name: + + addOne = + name: let pkgDir = shorthandDir + "/${name}"; - def = pkgDir + "/default.nix"; in - if builtins.pathExists def - then { ${name} = pkgs.callPackage pkgDir { }; } - else { }; + if builtins.pathExists (pkgDir + "/default.nix") then + { ${name} = callPackage pkgDir { }; } + else + { }; in - acc // lib.foldl' (a: n: a // addOne n) { } names + acc // lib.foldl' (a: n: a // addOne n) { } names ) { } (dirs base); + + byName = ./by-name; + # topLevel = if builtins.pathExists byName then packagesFromTree byName pkgs.callPackage else { }; + + byPy3 = ./by-category/python3Packages; + # pyExts = + # if builtins.pathExists byPy3 then + # [ + # (pyFinal: _pyPrev: packagesFromTree byPy3 pyFinal.callPackage) + # ] + # else + # [ ]; in -packagesFromByName byName +(packagesFromTree byName pkgs.callPackage) +// { + pythonPackagesExtensions = (prev.pythonPackagesExtensions or [ ]) ++ [ + (pyFinal: _pyPrev: packagesFromTree byPy3 pyFinal.callPackage) + ]; +} diff --git a/flake.lock b/flake.lock index af56449..5764460 100644 --- a/flake.lock +++ b/flake.lock @@ -320,11 +320,11 @@ "zon2nix": "zon2nix" }, "locked": { - "lastModified": 1770888240, - "narHash": "sha256-FwF56DWjzQfWaQrQR61JkvpkYlu8lEWulXoK9sLVtTk=", + "lastModified": 1770909583, + "narHash": "sha256-c9VlVV7fmO1g764SF7Z91Gry3Y3/dV1AAus0CeyTnTM=", "owner": "ghostty-org", "repo": "ghostty", - "rev": "2868677af50af301ebe4ec1c7f79ad31dcd67cd3", + "rev": "d54f079517e30c407a1d86093f2255335a24b0e8", "type": "github" }, "original": { @@ -385,11 +385,11 @@ ] }, "locked": { - "lastModified": 1770818644, - "narHash": "sha256-DYS4jIRpRoKOzJjnR/QqEd/MlT4OZZpt8CrBLv+cjsE=", + "lastModified": 1770915843, + "narHash": "sha256-ZwU5wXKNqpOQvjNz6aBp1j5peiBZow1++6pLnk5VAhs=", "owner": "nix-community", "repo": "home-manager", - "rev": "0acbd1180697de56724821184ad2c3e6e7202cd7", + "rev": "6a1f7101d2c3ee87d485a87880d73b4665c6a4bd", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 6b369f3..6a045b8 100644 --- a/flake.nix +++ b/flake.nix @@ -54,60 +54,63 @@ }; nixpkgs.overlays = [ intel-hw.overlays.default - (import ./custom/overlays) - (final: prev: { - llama-cpp = prev.llama-cpp.overrideAttrs (old: { - src = prev.fetchFromGitHub { - owner = "ggml-org"; - repo = "llama.cpp"; - tag = "b7897"; - hash = "sha256-cc2tqEQEZCBEOQ+xJzmVK8ROjfsWOi6T2iX3KjpaAIU="; - }; - nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.curl ]; - }); - pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [ - (pyFinal: pyPrev: { - haystack-ai = pyPrev.haystack-ai.overridePythonAttrs (old: rec { - version = "2.23.0"; + (import ./custom/overlays/default.nix) + (import custom/overlays/override.nix) + # (final: prev: { + # llama-cpp = prev.llama-cpp.overrideAttrs (old: { + # src = prev.fetchFromGitHub { + # owner = "ggml-org"; + # repo = "llama.cpp"; + # tag = "b7897"; + # hash = "sha256-cc2tqEQEZCBEOQ+xJzmVK8ROjfsWOi6T2iX3KjpaAIU="; + # }; + # nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.curl ]; + # }); + # pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [ + # (pyFinal: pyPrev: { + # haystack-ai = pyPrev.haystack-ai.overridePythonAttrs (old: rec { + # version = "2.23.0"; - src = final.fetchFromGitHub { - owner = "deepset-ai"; - repo = "haystack"; - rev = "v${version}"; - hash = "sha256-KFgRKdhbHgjtpiuAeFq+0t0EUKKLOavbVplb0QnI/r8="; - }; + # src = final.fetchFromGitHub { + # owner = "deepset-ai"; + # repo = "haystack"; + # rev = "v${version}"; + # hash = "sha256-KFgRKdhbHgjtpiuAeFq+0t0EUKKLOavbVplb0QnI/r8="; + # }; - propagatedBuildInputs = - old.propagatedBuildInputs - ++ (with pyPrev; [ - docstring-parser - filetype - haystack-experimental # ./custom/pkgs/by-category/python3Packages/ha/haystack-experimental/default.nix - jinja2 - openai - ]); + # propagatedBuildInputs = + # (old.propagatedBuildInputs or [ ]) + # ++ (with pyPrev; [ + # docstring-parser + # filetype + # jinja2 + # openai + # ]) + # ++ [ + # pyFinal."haystack-experimental" + # ]; - meta = (old.meta or { }) // { - broken = false; - }; - }); - }) - ]; - winetricks = prev.winetricks.overrideAttrs ( - old: - let - version = "20260125"; - in - { - inherit version; + # meta = (old.meta or { }) // { + # broken = false; + # }; + # }); + # }) + # ]; + # winetricks = prev.winetricks.overrideAttrs ( + # old: + # let + # version = "20260125"; + # in + # { + # inherit version; - src = final.fetchurl { - url = "https://github.com/Winetricks/winetricks/archive/refs/tags/${version}.tar.gz"; - hash = "sha256-KJC9n7ut5GOOWLSZmiNycxkt8DtYUWrnuHceCcItL1Y="; - }; - } - ); - }) + # src = final.fetchurl { + # url = "https://github.com/Winetricks/winetricks/archive/refs/tags/${version}.tar.gz"; + # hash = "sha256-KJC9n7ut5GOOWLSZmiNycxkt8DtYUWrnuHceCcItL1Y="; + # }; + # } + # ); + # }) ]; hardware = { graphics = { @@ -209,6 +212,9 @@ extraCompatPackages = [ inputs.dw-proton.packages.${system}.dw-proton ]; + protontricks = { + enable = true; + }; }; traceroute.enable = true; ydotool = {