This patch series is a resend of 54944 v3, marked as SUPERSEDED.

v2 cover-letter:

This is a complete rewrite of the previous patch series.  The
rde-initialize-guix-python-environment has been completely rewritten
to parse the search paths instead of determining profile by hand. It
has been completed by a rde-project-get-manifest-search-paths in
feature-emacs-project that will probably be useful for other
languages.  The patch series is rebased to account for updates in Guix
upstream.  Lastly a fix is provided for the proper accounting of
max-line-length, with a dedicated-option.

v3 cover-letter:

Actually I'm not sure that the 5/6 patch fixes the issue, probably
requires more testing.
I'd also like to have Andrew's input before considering merging 3/6 ;)


Nicolas Graves (6):
  rde: python: Ensure home-services definition
  rde: python: Introduce lsp for python
  rde: project: Add function to extract search-paths
  rde: python: Add function rde-initialize-guix-python-environment
  rde: python: Fix and add option max-line-length
  rde: python: Add options dap? and python-debugpy

 src/rde/features/emacs-xyz.scm |  49 +++++++++++-
 src/rde/features/python.scm    | 137 +++++++++++++++++++++++++--------
 2 files changed, 153 insertions(+), 33 deletions(-)

-- 
2.47.1

---
 src/rde/features/python.scm | 46 +++++++++++++++++++------------------
 1 file changed, 24 insertions(+), 22 deletions(-)

diff --git a/src/rde/features/python.scm b/src/rde/features/python.scm
index cf180b19..212650b7 100644
--- a/src/rde/features/python.scm
+++ b/src/rde/features/python.scm
@@ -83,7 +83,7 @@ python files."
   (define f-name 'python)
 
   (define (get-home-services config)
-    (list
+    (cons*
      (simple-service
       'add-python-home-package
       home-profile-service-type
@@ -93,28 +93,30 @@ python files."
       home-environment-variables-service-type
       `(("IPYTHONDIR" . "$XDG_CONFIG_HOME/ipython")
         ("PYTHONSTARTUP" . ,python-startup-file)))
-     (when (get-value 'emacs config #f)
-       (rde-elisp-configuration-service
-        f-name
-        config
-        `(,@(if black?
-                '((eval-when-compile (require 'python-black))
-                  (add-hook 'python-mode
-                            'python-black-on-save-mode-enable-dwim))
-                '())
+     (if (get-value 'emacs config #f)
+         (list
+          (rde-elisp-configuration-service
+           f-name
+           config
+           `(,@(if black?
+                   '((eval-when-compile (require 'python-black))
+                     (add-hook 'python-mode
+                               'python-black-on-save-mode-enable-dwim))
+                   '())
 
-          ,@(if (get-value 'emacs-org config #f)
-                `((with-eval-after-load 'org
-                    (add-to-list 'org-structure-template-alist
-                                 '("py" . "src python")))
-                  (with-eval-after-load 'ob-core
-                    (require 'ob-python))
-                  (with-eval-after-load 'ob-python
-                    (setq org-babel-python-command
-                          ,(file-append python "/bin/python"))))
-                '()))
-        #:elisp-packages
-        (if black? (list emacs-python-black) '())))))
+             ,@(if (get-value 'emacs-org config #f)
+                   `((with-eval-after-load 'org
+                       (add-to-list 'org-structure-template-alist
+                                    '("py" . "src python")))
+                     (with-eval-after-load 'ob-core
+                       (require 'ob-python))
+                     (with-eval-after-load 'ob-python
+                       (setq org-babel-python-command
+                             ,(file-append python "/bin/python"))))
+                   '()))
+           #:elisp-packages
+           (if black? (list emacs-python-black) '())))
+         '())))
 
   (feature
    (name f-name)
-- 
2.47.1

This commit also cleans out our previous approach with the black
tool. In particular, it uses the black plugin for lsp to provide the
same functionality.
---
 src/rde/features/python.scm | 51 ++++++++++++++++++++++++++-----------
 1 file changed, 36 insertions(+), 15 deletions(-)

diff --git a/src/rde/features/python.scm b/src/rde/features/python.scm
index 212650b7..0821689d 100644
--- a/src/rde/features/python.scm
+++ b/src/rde/features/python.scm
@@ -23,6 +23,7 @@
   #:use-module (rde features predicates)
   #:use-module (gnu services)
   #:use-module (gnu home services)
+  #:use-module (gnu services configuration)
   #:use-module (gnu packages python)
   #:use-module (gnu packages emacs-xyz)
   #:use-module (guix gexp)
@@ -71,14 +72,18 @@ if readline.get_current_history_length() == 0:
 (define* (feature-python
           #:key
           (python python-wrapper)
-          (emacs-python-black emacs-python-black)
-          (black? #f))
-  "Configure python for emacs. If black? is #t, configure the
-emacs-python-black package, which provides useful functions for formatting
-python files."
+          (python-lsp-server python-lsp-server)
+          (python-lsp-server-plugins (list python-lsp-black))
+          (lsp? #t))
+  "Configure python for emacs.  If lsp? is #t (default) and feature-eglot is
+present, provide advanced integration for lsp with python.  In particular,
+@itemize
+@item @code{python-lsp-black} provides @code{eglot-format-buffer}
+@end itemize"
   (ensure-pred file-like? python)
-  (ensure-pred file-like? emacs-python-black)
-  (ensure-pred boolean? black?)
+  (ensure-pred file-like? python-lsp-server)
+  (ensure-pred list-of-file-likes? python-lsp-server-plugins)
+  (ensure-pred boolean? lsp?)
 
   (define f-name 'python)
 
@@ -87,7 +92,10 @@ python files."
      (simple-service
       'add-python-home-package
       home-profile-service-type
-      (list python))
+      (cons* python
+             (if (and lsp? (get-value 'emacs-eglot config #f))
+                 (cons* python-lsp-server python-lsp-server-plugins)
+                 '())))
      (simple-service
       'python-xdg-base-dirs-specification
       home-environment-variables-service-type
@@ -98,10 +106,25 @@ python files."
           (rde-elisp-configuration-service
            f-name
            config
-           `(,@(if black?
-                   '((eval-when-compile (require 'python-black))
-                     (add-hook 'python-mode
-                               'python-black-on-save-mode-enable-dwim))
+           `(,@(if lsp?
+                   `((with-eval-after-load 'eglot
+                       (setq-default
+                        eglot-workspace-configuration
+                        (cons '(:pylsp
+                                . (:configurationSources ["flake8"]
+                                   :plugins (:pycodestyle (:enabled :json-false)
+                                             :mccabe (:enabled :json-false)
+                                             :pyflakes (:enabled :json-false)
+                                             :flake8 (:enabled :json-false
+                                                      :maxLineLength 88)
+                                             :pydocstyle (:enabled t
+                                                          :convention "numpy")
+                                             :yapf (:enabled :json-false)
+                                             :autopep8 (:enabled :json-false)
+                                             :black (:enabled t
+                                                     :line_length 88
+                                                     :cache_config t))))
+                              eglot-workspace-configuration))))
                    '())
 
              ,@(if (get-value 'emacs-org config #f)
@@ -113,9 +136,7 @@ python files."
                      (with-eval-after-load 'ob-python
                        (setq org-babel-python-command
                              ,(file-append python "/bin/python"))))
-                   '()))
-           #:elisp-packages
-           (if black? (list emacs-python-black) '())))
+                   '()))))
          '())))
 
   (feature
-- 
2.47.1

They provide a really pleasant alist corresponding to the guix
search-paths of the current project.  It does not behave that well
when there is no guix shell cache (launches hidden builds), hence the
5 seconds timeout to stop any hidden build and warn the user.
---
 src/rde/features/emacs-xyz.scm | 49 ++++++++++++++++++++++++++++++++--
 1 file changed, 47 insertions(+), 2 deletions(-)

diff --git a/src/rde/features/emacs-xyz.scm b/src/rde/features/emacs-xyz.scm
index 45febfc8..5bad5e97 100644
--- a/src/rde/features/emacs-xyz.scm
+++ b/src/rde/features/emacs-xyz.scm
@@ -2797,8 +2797,9 @@ working environemnt."
 
 (define* (feature-emacs-project
           #:key
-          (project-extra-dominating-files
-           '(".project.el" ".dir-locals.el" ".gitignore")))
+          (project-extra-dominating-files  ;order matters
+           '(".project.el" "manifest.scm" "guix.scm"
+             ".dir-locals.el" ".gitignore")))
   "Configure project.el, a library to perform operations
 on the current project."
   (ensure-pred list? project-extra-dominating-files)
@@ -2858,6 +2859,50 @@ on the current project."
         (add-hook 'project-find-functions 'project-try-vc)
         (advice-add 'project-compile :override 'rde-project-compile)
 
+        (defun rde-project-parse-search-paths (search-paths)
+          "Parse the SEARCH-PATHS string into an alist of variables and their values.
+Each value is a list of strings, split by ':' in case of multiple paths."
+          (let ((lines (split-string search-paths "\n" t))
+                (alist '()))
+            (dolist (line lines)
+                    (when (string-prefix-p "export " line)
+                      (let ((parsed (split-string (substring line 7) "=" t "\"")))
+                        (push (cons (car parsed)
+                                    (split-string (cadr parsed) ":"))
+                              alist))))
+            (reverse alist)))
+
+        (defun rde-project-get-manifest-search-paths ()
+          "Get the content of guix shell --search-paths --pure as an alist."
+          (interactive)
+          (if (or (member "guix.scm" rde-project-dominating-files)
+                  (member "manifest.scm" rde-project-dominating-files))
+              (when-let* ((default-directory (project-root
+                                              (rde-project-custom-root
+                                               default-directory)))
+                          (manifest-file
+                           (seq-find
+                            'file-readable-p
+                            (list
+                             (file-truename "manifest.scm")
+                             (file-truename "guix.scm"))))
+                          (result
+                           (with-output-to-string
+                             (with-current-buffer
+                              standard-output
+                              (if (zerop (call-process
+                                          "guix" nil t nil "shell"
+                                          "-m" manifest-file
+                                          "--search-paths" "--pure"
+                                          "--timeout=5"))
+                                  (buffer-string)
+                                  (error "\
+guix shell times out. Ensure the profile is built."))))))
+                         (rde-project-parse-search-paths result))
+              (message "rde-projects-get-manifest-search-paths: \
+rde-project-dominating-files requires \"guix.scm\" or \"manifest.scm\" to be \
+able to load guix shell search-paths.")))
+
         (define-key global-map (kbd "s-p") project-prefix-map)
 
         (with-eval-after-load 'project
-- 
2.47.1

---
 src/rde/features/python.scm | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/src/rde/features/python.scm b/src/rde/features/python.scm
index 0821689d..995053ea 100644
--- a/src/rde/features/python.scm
+++ b/src/rde/features/python.scm
@@ -1,6 +1,6 @@
 ;;; rde --- Reproducible development environment.
 ;;;
-;;; Copyright © 2023 Nicolas Graves <ngraves@ngraves.fr>
+;;; Copyright © 2023, 2024 Nicolas Graves <ngraves@ngraves.fr>
 ;;;
 ;;; This file is part of rde.
 ;;;
@@ -25,6 +25,7 @@
   #:use-module (gnu home services)
   #:use-module (gnu services configuration)
   #:use-module (gnu packages python)
+  #:use-module (gnu packages python-xyz)
   #:use-module (gnu packages emacs-xyz)
   #:use-module (guix gexp)
   #:export (feature-python))
@@ -127,6 +128,28 @@ present, provide advanced integration for lsp with python.  In particular,
                               eglot-workspace-configuration))))
                    '())
 
+             (eval-when-compile (require 'subr-x))
+             (defun rde-initialize-guix-python-environment ()
+               "Initialize the Python interpreter using Guix shell."
+               (interactive)
+               (when-let*
+                ((search-paths (rde-project-get-manifest-search-paths))
+                 (exec-path (cdr (assoc "PATH" search-paths)))
+                 (python (executable-find "python3"))
+                 (pythonpath (cadr (assoc "GUIX_PYTHONPATH" search-paths))))
+                (if python
+                    (prog2
+                     (setq-local python-shell-interpreter python)
+                     (setq-local
+                      python-shell-interpreter-args
+                      (format
+                       "-i -c \"import sys; sys.path.append('%s');\""
+                       pythonpath)))
+                    (error "Unable to run %s" python))))
+
+             (add-hook 'python-mode-hook 'rde-initialize-guix-python-environment)
+             (add-hook 'python-ts-mode-hook 'rde-initialize-guix-python-environment)
+
              ,@(if (get-value 'emacs-org config #f)
                    `((with-eval-after-load 'org
                        (add-to-list 'org-structure-template-alist
-- 
2.47.1

---
 src/rde/features/python.scm | 44 +++++++++++++++++++++++++++----------
 1 file changed, 33 insertions(+), 11 deletions(-)

diff --git a/src/rde/features/python.scm b/src/rde/features/python.scm
index 995053ea..5048f4df 100644
--- a/src/rde/features/python.scm
+++ b/src/rde/features/python.scm
@@ -75,9 +75,12 @@ if readline.get_current_history_length() == 0:
           (python python-wrapper)
           (python-lsp-server python-lsp-server)
           (python-lsp-server-plugins (list python-lsp-black))
-          (lsp? #t))
-  "Configure python for emacs.  If lsp? is #t (default) and feature-eglot is
-present, provide advanced integration for lsp with python.  In particular,
+          (lsp? #t)
+          (max-line-length 88))
+  "Configure python for emacs.
+
+If lsp? is #t (default) and feature-eglot is present, provide advanced
+integration for lsp.  In particular:
 @itemize
 @item @code{python-lsp-black} provides @code{eglot-format-buffer}
 @end itemize"
@@ -85,6 +88,7 @@ present, provide advanced integration for lsp with python.  In particular,
   (ensure-pred file-like? python-lsp-server)
   (ensure-pred list-of-file-likes? python-lsp-server-plugins)
   (ensure-pred boolean? lsp?)
+  (ensure-pred integer? max-line-length)
 
   (define f-name 'python)
 
@@ -104,6 +108,22 @@ present, provide advanced integration for lsp with python.  In particular,
         ("PYTHONSTARTUP" . ,python-startup-file)))
      (if (get-value 'emacs config #f)
          (list
+          ;; In LSP config, :configurationSources ["flake8"] is necessary to
+          ;; enable picking project flake8 configurations when encountered.
+          ;; It however makes the eglot-workspace-configuration not complete
+          ;; enough to actually propagate the max-line-length option properly.
+          ;; That is because flake8 tries to read an absent config file
+          ;; and thus relies on system defaults.  Hence the need for this service:
+          (simple-service
+           'flake8-default-config
+           home-files-service-type
+           (list `(".config/flake8"
+                   ,(plain-file
+                     "rde-home-flake8-configuration"
+                     (format #f "\
+[flake8]
+max-line-length = ~a~%"
+                             max-line-length)))))
           (rde-elisp-configuration-service
            f-name
            config
@@ -113,17 +133,19 @@ present, provide advanced integration for lsp with python.  In particular,
                         eglot-workspace-configuration
                         (cons '(:pylsp
                                 . (:configurationSources ["flake8"]
-                                   :plugins (:pycodestyle (:enabled :json-false)
-                                             :mccabe (:enabled :json-false)
-                                             :pyflakes (:enabled :json-false)
-                                             :flake8 (:enabled :json-false
-                                                      :maxLineLength 88)
+                                   :plugins (:pycodestyle (:enabled nil)
+                                             :mccabe (:enabled nil)
+                                             :pyflakes (:enabled nil)
+                                             :yapf (:enabled nil)
+                                             :autopep8 (:enabled nil)
                                              :pydocstyle (:enabled t
                                                           :convention "numpy")
-                                             :yapf (:enabled :json-false)
-                                             :autopep8 (:enabled :json-false)
+                                             :flake8 (:enabled t
+                                                      :maxLineLength
+                                                      ,max-line-length)
                                              :black (:enabled t
-                                                     :line_length 88
+                                                     :line_length
+                                                     ,max-line-length
                                                      :cache_config t))))
                               eglot-workspace-configuration))))
                    '())
-- 
2.47.1

---
 src/rde/features/python.scm | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/rde/features/python.scm b/src/rde/features/python.scm
index 5048f4df..ed2e89ea 100644
--- a/src/rde/features/python.scm
+++ b/src/rde/features/python.scm
@@ -73,8 +73,10 @@ if readline.get_current_history_length() == 0:
 (define* (feature-python
           #:key
           (python python-wrapper)
+          (python-debugpy python-debugpy)
           (python-lsp-server python-lsp-server)
           (python-lsp-server-plugins (list python-lsp-black))
+          (dap? #t)
           (lsp? #t)
           (max-line-length 88))
   "Configure python for emacs.
@@ -85,8 +87,10 @@ integration for lsp.  In particular:
 @item @code{python-lsp-black} provides @code{eglot-format-buffer}
 @end itemize"
   (ensure-pred file-like? python)
+  (ensure-pred file-like? python-debugpy)
   (ensure-pred file-like? python-lsp-server)
   (ensure-pred list-of-file-likes? python-lsp-server-plugins)
+  (ensure-pred boolean? dap?)
   (ensure-pred boolean? lsp?)
   (ensure-pred integer? max-line-length)
 
@@ -97,10 +101,13 @@ integration for lsp.  In particular:
      (simple-service
       'add-python-home-package
       home-profile-service-type
-      (cons* python
-             (if (and lsp? (get-value 'emacs-eglot config #f))
-                 (cons* python-lsp-server python-lsp-server-plugins)
-                 '())))
+      (append (list python)
+              (if (and lsp? (get-value 'emacs-eglot config #f))
+                  (cons* python-lsp-server python-lsp-server-plugins)
+                  '())
+              (if (and dap? (get-value 'emacs-dape config #f))
+                  (list python-debugpy)
+                  '())))
      (simple-service
       'python-xdg-base-dirs-specification
       home-environment-variables-service-type
-- 
2.47.1