Nicolas Graves (3):
  packages: Add pylsp packages for eglot integration

Merging python patches in Guix is quite difficult, because of the lack
of a structured team. Until python-team branch is merged, we're better
off using our own versions.

  rde: python: Introduce lsp for python

This patch introduces the defaults for python lsp configuration. It
replaces our artisanal use of `black-format-buffer` by the more
generic `eglot-format-buffer`.

  rde: python: Add function rde-initialize-guix-python-environment

`run-python` was not aware of the local python environment. This patch
makes it trivial to use `run-python` in an existing project. It makes
the assumption that the user has a manifest.scm file in a parent
directory, and injects the python `site-packages` directory in the
python-shell-interpreter-args, as a local variable. (It makes some
assumptions but is a quality improvement assuming development using
manifest.scm).

 src/rde/features/python.scm     | 102 +++++++++++--
 src/rde/packages/python-xyz.scm | 256 ++++++++++++++++++++++++++++++++
 2 files changed, 343 insertions(+), 15 deletions(-)
 create mode 100644 src/rde/packages/python-xyz.scm

-- 
2.46.0

---
 src/rde/packages/python-xyz.scm | 256 ++++++++++++++++++++++++++++++++
 1 file changed, 256 insertions(+)
 create mode 100644 src/rde/packages/python-xyz.scm

diff --git a/src/rde/packages/python-xyz.scm b/src/rde/packages/python-xyz.scm
new file mode 100644
index 000000000..c204ae885
--- /dev/null
+++ b/src/rde/packages/python-xyz.scm
@@ -0,0 +1,256 @@
+;;; rde --- Reproducible development environment.
+;;;
+;;; Copyright © 2024 Nicolas Graves <ngraves@ngraves.fr>
+;;; Copyright © 2024 Jonathan Pieper <jpieper@mailbox.org>
+;;;
+;;; This file is part of rde.
+;;;
+;;; rde is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; rde is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with rde.  If not, see <http://www.gnu.org/licenses/>.
+
+(define-module (rde packages python-xyz)
+  #:use-module (gnu packages check)
+  #:use-module (gnu packages python-build)
+  #:use-module (gnu packages python-check)
+  #:use-module (gnu packages python-science)
+  #:use-module (gnu packages python-web)
+  #:use-module (gnu packages python-xyz)
+  #:use-module (gnu packages qt)
+  #:use-module (gnu packages sphinx)
+  #:use-module (guix git-download)
+  #:use-module (guix download)
+  #:use-module (guix packages)
+  #:use-module (guix build-system pyproject)
+  #:use-module (guix build-system python)
+  #:use-module ((guix licenses) #:prefix license:))
+
+(define-public python-docstring-to-markdown
+  (package
+    (name "python-docstring-to-markdown")
+    (version "0.15")
+    (source
+     (origin
+       (method url-fetch)
+       (uri (pypi-uri "docstring-to-markdown" version))
+       (sha256
+        (base32 "0gdpabnyl1kyy0cjrnph6xl4fyhgim50a1amsaqq3hahki6i2ip1"))))
+    (build-system python-build-system)
+    (home-page "https://github.com/python-lsp/docstring-to-markdown")
+    (synopsis "On the fly conversion of Python docstrings to markdown")
+    (description
+     "This module can convert Python docstrings to Markdown.
+It can recognise reStructuredText inside docstrings and convert multiple of its
+features to Markdown. It also includes initial support for Google-formatted docstrings.")
+    (license license:lgpl2.1+)))
+
+(define-public python-lsp-black
+  (package
+    (name "python-lsp-black")
+    (version "1.3.0")
+    (source
+     (origin
+       (method url-fetch)
+       (uri (pypi-uri "python-lsp-black" version))
+       (sha256
+        (base32 "0gqiwrlwvx8b3kij45mzjpvf10if4k6zpagjdqqs5rdpwzlmg8js"))))
+    (build-system pyproject-build-system)
+    (arguments
+     (list #:tests? #f))  ; No test in Pypi
+    (propagated-inputs
+     (list python-black python-lsp-server-1.7 python-tomli))
+    (native-inputs
+     (list python-pytest))
+    (home-page "https://github.com/python-lsp/python-lsp-black")
+    (synopsis "Black plugin for the Python LSP Server")
+    (description "This package provides a plugin with support for the
+@code{python-black} formatter for the Python LSP Server.")
+    (license license:expat)))
+
+(define-public python-lsp-jsonrpc-1.1
+  (package
+    (inherit python-lsp-jsonrpc)
+    (name "python-lsp-jsonrpc")
+    (version "1.1.2")
+    (source
+     (origin
+       (method url-fetch)
+       (uri (pypi-uri "python-lsp-jsonrpc" version))
+       (sha256
+        (base32 "04n95h0cqnsrdyh1gv0abh2i5ynyrq2wfqpppx9djp7mxr9y9226"))))
+    (arguments
+     `(#:phases (modify-phases %standard-phases
+                  (add-after 'unpack 'set-version
+                    (lambda _
+                      (substitute* "pyproject.toml"
+                        (("dynamic = \\[\"version\"\\]")
+                         (string-append "version = \""
+                                        ,version "\""))))))))
+    (build-system pyproject-build-system)
+    (native-inputs
+     (list python-mock python-pytest python-pytest-cov))
+    (propagated-inputs
+     (list python-ujson))))
+
+(define-public python-lsp-server-1.7
+  (package
+    (inherit python-lsp-server)
+    (name "python-lsp-server")
+    (version "1.7.0")
+    (source
+     (origin
+       (method url-fetch)
+       (uri (pypi-uri "python-lsp-server" version))
+       (sha256
+        (base32 "0dalri5g8zkdyfibjmc0pjm9r1wj5jrjx5ll5p8av379la7ff720"))))
+    (build-system pyproject-build-system)
+    (arguments
+     `(#:test-flags
+       (list "-k"
+             (string-append "not test_jedi_completion_item_resolve"
+                            " and not test_hover"
+                            ;; These tests are working but can cause a lot
+                            ;; of time-wasting rebuilds. Avoided instead.
+                            " and not test_pyqt_completion"
+                            " and not test_matplotlib_completions"
+                            " and not test_numpy_completions"
+                            " and not test_pandas_completions"))
+       #:phases (modify-phases %standard-phases
+                  (add-after 'unpack 'set-version
+                    (lambda _
+                      (substitute* "pyproject.toml"
+                        (("dynamic = \\[\"version\"\\]")
+                         (string-append "version = \""
+                                        ,version "\"")))))
+                  (add-before 'check 'set-HOME
+                    (lambda _
+                      (setenv "HOME" "/tmp"))))))
+    (propagated-inputs (list python-autopep8
+                             python-docstring-to-markdown
+                             python-pydocstyle
+                             python-flake8
+                             python-jedi
+                             python-lsp-jsonrpc-1.1
+                             python-pluggy
+                             python-pycodestyle
+                             python-pyflakes
+                             python-rope-1.13
+                             python-ujson
+                             python-yapf))
+    (native-inputs (list python-coverage
+                         python-flaky
+                         python-pylint
+                         python-pytest
+                         python-pytest-cov
+                         python-whatthepatch))))
+
+(define-public python-pylsp-mypy
+  (package
+    (name "python-pylsp-mypy")
+    (version "0.6.8")
+    (source
+     (origin
+       (method url-fetch)
+       (uri (pypi-uri "pylsp-mypy" version))
+       (sha256
+        (base32 "0pxx85nms0zs54s1rlrkpkharkkiqcqzxi9q1vjm7qnp0z50g0rz"))))
+    (build-system pyproject-build-system)
+    (propagated-inputs (list python-mypy python-lsp-server-1.7 python-tomli))
+    (native-inputs
+     (list python-coverage
+           python-pytest
+           python-pytest-cov
+           python-tox))
+    (home-page "https://github.com/python-lsp/pylsp-mypy")
+    (synopsis "Mypy linter for the Python LSP Server")
+    (description "This package is a plugin for the Python LSP Server.  It
+allows to add typing mypy lints to the LSP.")
+    (license license:expat)))
+
+(define-public python-pytoolconfig
+  (package
+    (name "python-pytoolconfig")
+    (version "1.2.2")
+    (source
+     (origin
+       (method url-fetch)
+       (uri (pypi-uri "pytoolconfig" version))
+       (sha256
+        (base32 "18isxi4ijarl949d0zmf0b4606r6hihpi3p5yb7763m4c7ra24i5"))))
+    (build-system pyproject-build-system)
+    (arguments
+     '(#:phases (modify-phases %standard-phases
+                  (add-after 'unpack 'update-license
+                    (lambda _
+                      (substitute* "pyproject.toml"
+                        (("license-expression = (\"[^\"]*\")" all license)
+                         (string-append "license = {text = " license "}")))))
+                  (add-after 'unpack 'remove-mypy
+                    (lambda _
+                      (substitute* "pyproject.toml"
+                        (("^.*mypy.*")
+                         "")
+                        (("strict = true")
+                         ""))))
+                  (add-after 'unpack 'use-pdm-backend-instead-of-pep517
+                    (lambda _
+                      (substitute* "pyproject.toml"
+                        (("pdm-pep517")
+                         "pdm-backend")
+                        (("pdm\\.pep517\\.api")
+                         "pdm.backend"))))
+                  (replace 'check
+                    (lambda* (#:key tests? #:allow-other-keys)
+                      (when tests?
+                        ;; Disable failing test.
+                        (invoke "python" "-m" "pytest" "-k"
+                                "not test_documentation")))))))
+    (native-inputs (list python-pdm-backend
+                         python-tomli
+                         python-pytest
+                         python-docutils
+                         python-sphinx
+                         python-tabulate))
+    (propagated-inputs (list python-appdirs))
+    (home-page "https://github.com/bagel897/pytoolconfig")
+    (synopsis "Python Tool Configuration")
+    (description
+     "This module manages configuration for python tools,
+such as rope and add support for a pyproject.toml configuration file.")
+    (license license:lgpl3+)))
+
+(define-public python-rope-1.13
+  (package
+    (inherit python-rope)
+    (name "python-rope")
+    (version "1.13.0")
+    (source
+     (origin
+       (method url-fetch)
+       (uri (pypi-uri "rope" version))
+       (sha256
+        (base32 "1078mkzivz45my8x2y5gxisr0vba630xj7yxx7anr068xhnpshsi"))))
+    (arguments
+     (list
+      #:phases `(modify-phases %standard-phases
+                  (add-after 'unpack 'disable-broken-test
+                    (lambda _
+                      (substitute* "ropetest/contrib/autoimporttest.py"
+                        (("def test_search_module")
+                         "def __notest_search_module")
+                        (("def test_search_submodule")
+                         "def __notest_search_submodule"))
+                      (substitute* "ropetest/type_hinting_test.py"
+                        (("def test_hint_or")
+                         "def __notest_hint_or")))))))
+    (native-inputs (list python-pytest-timeout python-pytest))
+    (propagated-inputs (list python-pytoolconfig))))
-- 
2.46.0

This commit also cleans out our previous approach with the black
tool. In particular, it uses the black plugin for lsp to provide the
same functionality.
---
 src/rde/features/python.scm | 54 ++++++++++++++++++++++++++-----------
 1 file changed, 39 insertions(+), 15 deletions(-)

diff --git a/src/rde/features/python.scm b/src/rde/features/python.scm
index cf180b19a..af3282574 100644
--- a/src/rde/features/python.scm
+++ b/src/rde/features/python.scm
@@ -21,13 +21,17 @@
   #:use-module (rde features)
   #:use-module (rde features emacs)
   #:use-module (rde features predicates)
+  #:use-module (rde packages python-xyz)
   #:use-module (gnu services)
   #:use-module (gnu home services)
+  #:use-module (gnu services configuration)
   #:use-module (gnu packages python)
   #:use-module (gnu packages emacs-xyz)
   #:use-module (guix gexp)
   #:export (feature-python))
 
+(define file-likes? (list-of file-like?))
+
 ;; This is the current recommended way to go by
 ;; https://github.com/b3nj5m1n/xdg-ninja, where it's specified that this won't
 ;; work if python is invoked with the -i flag. But it does work with `run-python'
@@ -71,14 +75,18 @@ if readline.get_current_history_length() == 0:
 (define* (feature-python
           #:key
           (python python-wrapper)
-          (emacs-python-black emacs-python-black)
-          (black? #f))
-  "Configure python for emacs. If black? is #t, configure the
-emacs-python-black package, which provides useful functions for formatting
-python files."
+          (python-lsp-server python-lsp-server-1.7)
+          (python-lsp-server-plugins (list python-lsp-black))
+          (lsp? #t))
+  "Configure python for emacs.  If lsp? is #t (default) and feature-eglot is
+present, provide advanced integration for lsp with python.  In particular,
+@itemize
+@item @code{python-lsp-black} provides @code{eglot-format-buffer}
+@end itemize"
   (ensure-pred file-like? python)
-  (ensure-pred file-like? emacs-python-black)
-  (ensure-pred boolean? black?)
+  (ensure-pred file-like? python-lsp-server)
+  (ensure-pred file-likes? python-lsp-server-plugins)
+  (ensure-pred boolean? lsp?)
 
   (define f-name 'python)
 
@@ -87,7 +95,10 @@ python files."
      (simple-service
       'add-python-home-package
       home-profile-service-type
-      (list python))
+      (cons* python
+             (if (and lsp? (get-value 'emacs-eglot config #f))
+                 (cons* python-lsp-server python-lsp-server-plugins)
+                 '())))
      (simple-service
       'python-xdg-base-dirs-specification
       home-environment-variables-service-type
@@ -97,10 +108,25 @@ python files."
        (rde-elisp-configuration-service
         f-name
         config
-        `(,@(if black?
-                '((eval-when-compile (require 'python-black))
-                  (add-hook 'python-mode
-                            'python-black-on-save-mode-enable-dwim))
+        `(,@(if lsp?
+                `((with-eval-after-load 'eglot
+                    (setq-default
+                     eglot-workspace-configuration
+                     (cons '(:pylsp
+                             . (:configurationSources ["flake8"]
+                                :plugins (:pycodestyle (:enabled :json-false)
+                                          :mccabe (:enabled :json-false)
+                                          :pyflakes (:enabled :json-false)
+                                          :flake8 (:enabled :json-false
+                                                   :maxLineLength 88)
+                                          :pydocstyle (:enabled t
+                                                       :convention "numpy")
+                                          :yapf (:enabled :json-false)
+                                          :autopep8 (:enabled :json-false)
+                                          :black (:enabled t
+                                                  :line_length 88
+                                                  :cache_config t))))
+                           eglot-workspace-configuration))))
                 '())
 
           ,@(if (get-value 'emacs-org config #f)
@@ -112,9 +138,7 @@ python files."
                   (with-eval-after-load 'ob-python
                     (setq org-babel-python-command
                           ,(file-append python "/bin/python"))))
-                '()))
-        #:elisp-packages
-        (if black? (list emacs-python-black) '())))))
+                '()))))))
 
   (feature
    (name f-name)
-- 
2.46.0

---
 src/rde/features/python.scm | 48 +++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/src/rde/features/python.scm b/src/rde/features/python.scm
index af3282574..5c07ba7f7 100644
--- a/src/rde/features/python.scm
+++ b/src/rde/features/python.scm
@@ -129,6 +129,54 @@ present, provide advanced integration for lsp with python.  In particular,
                            eglot-workspace-configuration))))
                 '())
 
+        `((eval-when-compile
+           (require 'subr-x))
+
+          (defun rde-find-manifest-file (directory)
+            "Recursively search for manifest.scm file in DIRECTORY parents."
+            (let ((manifest-file (expand-file-name "manifest.scm" directory))
+                  (parent-dir (file-name-directory
+                               (directory-file-name directory))))
+              (cond
+               ((file-exists-p manifest-file) manifest-file)
+               ((string= directory parent-dir) nil)
+               (t (rde-find-manifest-file parent-dir)))))
+
+          (defun rde-initialize-guix-python-environment ()
+            (interactive)
+            "Initialize the Python interpreter using Guix shell."
+            (let ((manifest-file (rde-find-manifest-file default-directory)))
+              (when manifest-file
+                (let* ((python-executable
+                        (string-trim (shell-command-to-string "\
+guix shell --container -m manifest.scm -- which python")))
+                       (python-found
+                        (and python-executable
+                             (not (string-empty-p python-executable))))
+                       (site-packages-paths
+                        (when python-found
+                          (directory-files-recursively
+                           (file-name-directory
+                            (directory-file-name
+                             (file-name-directory python-executable)))
+                           "^site-packages$"
+                           t))))
+
+                  (when python-found
+                    (setq-local python-shell-interpreter python-executable)
+                    (setq-local python-shell-interpreter-args
+                                (concat
+                                 "-i -c \"import sys; "
+                                 (mapconcat
+                                  (lambda (path)
+                                    (format "sys.path.append('%s')" path))
+                                  site-packages-paths
+                                  "; ")
+                                 "\"")))))))
+
+          (add-hook 'python-mode-hook 'rde-initialize-guix-python-environment)
+          (add-hook 'python-ts-mode-hook 'rde-initialize-guix-python-environment)
+
           ,@(if (get-value 'emacs-org config #f)
                 `((with-eval-after-load 'org
                     (add-to-list 'org-structure-template-alist
-- 
2.46.0

The third patch is extremely convenient but is probably not that safe,
as guix shell -m manifest.scm can execute arbitrary code and we might
have no way of marking the code as safe.

I've done such a trick building on direnv here : 
https://github.com/nicolas-graves/python-direnv

This works for https://git.sr.ht/~ngraves/nrepl-python
but is python-specific. We could however adapt this to inject
-c "import direnv; direnv.load_direnv()"

WDYT ?

-- 
Best regards,
Nicolas Graves