Windows support — sourcehut lists

[PATCH v2] Windows support Export this patch

Updates:

- Reinstate ForestScanner

- In Process.ml the code that obtains the address from a path is updated
  to work on windows. The code might be a bit ugly, but it works.

Jon Sterling <jon@jonmsterling.com>

11 months ago

Thanks for the updates! I’ve run the new version of the patch as-is on 
macOS, and got the following error:

     Fatal error: exception Eio.Io Fs Permission_denied Absolute_path,
       examining 
<cwd:/Users/jon/Documents/private-forest/trees/log/2024-07-29.tree>

Kento Okura <kentookura@gmail.com>

11 months ago

This is the source of the error:
https://github.com/ocaml-multicore/eio/blob/37840760b1b99fd9f59f0dbced315c72f08e639d/lib_eio/path.ml#L115
which gets called by the last_changed function. Does it work if you
just return None and comment the rest of that block out?

Best,
Jon

Notable changes:

- Rewrite Process.ml and remove ForestScanner
  I noticed no performance impedence on my personal forest of 400 trees,
  on both windows and linux.

- Replace Eio.Stdenv.fs with Eio.Stdenv.cwd. It caused a strange "file
  not found" error on windows.

- Move Reporter to Prelude because I wanted diagnostics for some
  functions in Eio_utils.

- Added an unbearably slow but portable version of a 'cp -R' function.
  The reason it is so slow is that I am reading everything to a string
  and saving it again. Need to revisit this to use Flows or something,
  but wanted to get this patch out there as I finally made a forest
  build on windows.
---
 bin/forester/Process.ml           | 13 +++++++--
 bin/forester/main.ml              | 14 +++++-----
 lib/core/Forester_core.ml         |  4 +--
 lib/frontend/Forest.ml            | 21 ++++++--------
 lib/frontend/Parse.ml             |  9 ++----
 lib/prelude/Eio_util.ml           | 46 +++++++++++++++++++++++++++++--
 lib/{core => prelude}/Reporter.ml |  4 +++
 lib/prelude/dune                  |  4 ++-
 lib/render/Build_latex.ml         |  1 +
 9 files changed, 81 insertions(+), 35 deletions(-)
 rename lib/{core => prelude}/Reporter.ml (92%)

diff --git a/bin/forester/Process.ml b/bin/forester/Process.ml
index 0077d35..ec20019 100644
--- a/bin/forester/Process.ml
@@ -3,10 +3,19 @@ open Forester_core
 let read_trees_in_dirs ~dev ?(ignore_malformed = false) dirs =
   ForestScanner.scan_directories dirs |> List.of_seq |> List.filter_map @@ fun fp ->
   Option.bind (Eio.Path.split fp) @@ fun (dir, basename) ->
-  let addr = Filename.chop_extension basename in
+  let addr = basename
+    |> String.split_on_char '/'
+    |> List.rev
+    |> List.hd
+    |> String.split_on_char '\\'
+    |> List.rev
+    |> List.hd
+    |> Filename.chop_extension
+  in
   let source_path = if dev then Option.map Unix.realpath @@ Eio.Path.native fp else None in
   match Forester_frontend.Parse.parse_file fp with
-  | Result.Ok code -> Some Code.{source_path; addr = Some addr; code}
+  | Result.Ok code ->
+    Some Code.{source_path; addr = Some addr; code}
   | Result.Error err -> None
   | exception exn ->
     if ignore_malformed then None else raise exn
diff --git a/bin/forester/main.ml b/bin/forester/main.ml
index ad72395..424789a 100644
--- a/bin/forester/main.ml
@@ -6,7 +6,7 @@ open Cmdliner
 module Tty = Asai.Tty.Make (Forester_core.Reporter.Message)
 
 let make_dir ~env dir =
-  Eio.Path.(Eio.Stdenv.fs env / dir)
+  Eio.Path.(Eio.Stdenv.cwd env / dir)
 
 let make_dirs ~env =
   List.map (make_dir ~env)
@@ -136,9 +136,9 @@ let init ~env () =
   in
   let theme_version = "4.0.0" in
   let (/) = Eio.Path.(/) in
-  let fs = Eio.Stdenv.fs env in
+  let cwd = Eio.Stdenv.cwd env in
   let try_create_dir name =
-    try Eio.Path.mkdir ~perm:0o700 (fs / name)
+    try Eio.Path.mkdir ~perm:0o700 (cwd / name)
     with _ ->
       Reporter.emitf Initialization_warning "Directory `%s` already exists" name
   in
@@ -167,13 +167,13 @@ theme = "theme"                      # The directory in which your theme is stor
 |} in
   let gitignore = {|output/|} in
   begin
-    if Eio.Path.is_file (Eio.Stdenv.fs env / "forest.toml") then
+    if Eio.Path.is_file (Eio.Stdenv.cwd env / "forest.toml") then
       Reporter.emitf Initialization_warning "forest.toml already exists"
     else
-      Eio.Path.(save ~create:(`Exclusive 0o600) (fs / "forest.toml") default_config_str)
+      Eio.Path.(save ~create:(`Exclusive 0o600) (cwd / "forest.toml") default_config_str)
   end;
 
-  Eio.Path.(save ~create:(`Exclusive 0o600) (fs / ".gitignore") gitignore);
+  Eio.Path.(save ~create:(`Exclusive 0o600) (cwd / ".gitignore") gitignore);
 
   let proc_mgr = Eio.Stdenv.process_mgr env in
   begin
@@ -203,7 +203,7 @@ theme = "theme"                      # The directory in which your theme is stor
 
   begin
     try
-      Eio.Path.(save ~create:(`Exclusive 0o600) (fs / "trees" / "index.tree") index_tree_str)
+      Eio.Path.(save ~create:(`Exclusive 0o600) (cwd / "trees" / "index.tree") index_tree_str)
     with _ ->
       Reporter.with_backtrace Emp @@ fun () ->
       Reporter.emitf Initialization_warning "`index.tree` already exists"
diff --git a/lib/core/Forester_core.ml b/lib/core/Forester_core.ml
index a541fd3..1cc0218 100644
--- a/lib/core/Forester_core.ml
+++ b/lib/core/Forester_core.ml
@@ -1,7 +1,6 @@
 module Base = Base
 
 include Base
-
 module Syn = Syn
 module Sem = Sem
 module Code = Code
@@ -11,10 +10,9 @@ module Expand = Expand
 module Eval = Eval
 module Query = Query
 module BaseN = BaseN
-
-module Reporter = Reporter
 module Range = Range
 module Prim = Prim
+module Reporter = Forester_prelude.Reporter
 
 module TeX_cs = TeX_cs
 module Symbol = Symbol
diff --git a/lib/frontend/Forest.ml b/lib/frontend/Forest.ml
index 5ea863c..a681118 100644
--- a/lib/frontend/Forest.ml
+++ b/lib/frontend/Forest.ml
@@ -159,22 +159,19 @@ let render_json ~cfg ~cwd docs =
 let is_hidden_file fname =
   String.starts_with ~prefix:"." fname
 
+
+let ( / ) = Eio.Path.( / )
+
 let copy_theme ~env ~theme_dir =
   let cwd = Eio.Stdenv.cwd env in
-  Eio.Path.read_dir theme_dir |> List.iter @@ fun fname ->
-  if not @@ is_hidden_file fname then
-    Eio.Path.native @@ Eio.Path.(theme_dir / fname) |> Option.iter @@ fun source ->
-    Eio_util.copy_to_dir ~env ~cwd ~source ~dest_dir:"output"
+  Eio_util.cp ~source:theme_dir ~dest:(cwd / "output") ()
 
 let copy_assets ~env ~assets_dirs =
   let cwd = Eio.Stdenv.cwd env in
   assets_dirs |> List.iter @@ fun assets_dir ->
   Eio.Path.read_dir assets_dir |> List.iter @@ fun fname ->
-  if not @@ is_hidden_file fname then
-    let path = Eio.Path.(assets_dir / fname) in
-    let source = Eio.Path.native_exn path in
-    Eio_util.copy_to_dir ~env ~cwd ~source ~dest_dir:"build";
-    Eio_util.copy_to_dir ~env ~cwd ~source ~dest_dir:"output"
+  Eio_util.cp ~source:assets_dir ~dest:(cwd / "build") ();
+  Eio_util.cp ~source:assets_dir ~dest:(cwd / "output") ()
 
 let copy_resources ~env =
   let cwd = Eio.Stdenv.cwd env in
@@ -189,13 +186,13 @@ let copy_resources ~env =
     in
     dest_opt |> Option.iter @@ fun dest_dir ->
     if not @@ Eio_util.file_exists Eio.Path.(cwd / dest_dir / fname) then
-      Eio_util.copy_to_dir ~cwd ~env ~source:fp ~dest_dir
+      Eio_util.cp ~source:(cwd / fp) ~dest:(cwd / dest_dir) ()
 
 let last_changed env forest scope =
   let (let*) = Option.bind in
   let* tree = M.find_opt scope forest.trees in
   let* source_path = tree.fm.source_path in
-  let path = Eio.Path.(Eio.Stdenv.fs env / source_path) in
+  let path = Eio.Path.(Eio.Stdenv.cwd env / source_path) in
   let stat  = Eio.Path.stat ~follow:true path in
   let* mtime = Some stat.mtime in
   let* ptime = Ptime.of_float_s mtime in
@@ -214,9 +211,7 @@ let render_trees ~cfg ~(forest : forest) ~render_only : unit =
     let root, trees, run_query, last_changed, enqueue_latex =
       cfg.root, forest.trees, forest.run_query, last_changed env forest, LaTeX_queue.enqueue
   end
-
   in
-
   let module C = Compile.Make (I) () in
   let module Sxml = Serialise_xml_tree.Make (I) () in
 
diff --git a/lib/frontend/Parse.ml b/lib/frontend/Parse.ml
index 20af484..a35bd15 100644
--- a/lib/frontend/Parse.ml
+++ b/lib/frontend/Parse.ml
@@ -140,13 +140,10 @@ let parse_channel filename ch =
   lexbuf.lex_curr_p <- { lexbuf.lex_curr_p with pos_fname = filename };
   maybe_with_errors (fun () -> try_parse lexbuf)
 
-let parse_file fp =
-  let filename = Eio.Path.native_exn fp in
-  let ch = open_in filename in
-  Fun.protect ~finally:(fun _ -> close_in ch) @@ fun _ ->
-  parse_channel filename ch
-
 let parse_string str =
   Reporter.tracef "when parsing string" @@ fun () ->
   let lexbuf = Lexing.from_string str in
   maybe_with_errors (fun () -> try_parse lexbuf)
+
+let parse_file fp =
+  parse_string @@ Eio.Path.load fp
diff --git a/lib/prelude/Eio_util.ml b/lib/prelude/Eio_util.ml
index 6b203b6..6f139a2 100644
--- a/lib/prelude/Eio_util.ml
+++ b/lib/prelude/Eio_util.ml
@@ -1,3 +1,4 @@
+open Reporter
 open Eio
 
 let formatter_of_writer w =
@@ -56,7 +57,46 @@ let file_exists path =
   try Eio.Path.with_open_in path @@ fun _ -> true with
   | Eio.Io (Eio.Fs.E (Eio.Fs.Not_found _), _) -> false
 
-(* TODO: make this portable *)
-let copy_to_dir ~env ~cwd ~source ~dest_dir =
-  run_process ~quiet:true ~env ~cwd ["cp"; "-R"; source; dest_dir ^ "/" ]
+let ( / ) = Path.( / )
 
+let cp ?(debug = false) ~source ~dest () =
+  let debug_info =
+    if debug then (fun p -> Eio.Std.traceln "%a" Eio.Path.pp p)
+    else Fun.const () 
+  in
+  let debug_error =
+    if debug then (fun p ex -> Eio.Std.traceln "%a: %a" Eio.Path.pp p Eio.Exn.pp ex)
+    else (fun _ _ -> ())
+  in
+  let rec aux ~source ~dest =
+    match Path.kind ~follow:false source with
+    | `Directory ->
+      begin
+        match Path.mkdir dest ~perm:0o700 with
+        | () -> debug_info dest
+        | exception ex -> debug_error dest ex
+      end;
+      Path.read_dir source |> List.iter (function
+        | item when String.starts_with ~prefix:"." item -> ()
+        | item -> aux ~source:(source / item) ~dest:(dest / item)
+      )
+    | `Regular_file ->
+      begin
+        (* Switch.run @@ fun sw -> *)
+        Path.with_open_in source @@ fun source ->
+        Path.with_open_out ~create:(`Or_truncate 0o700) dest @@ fun dest ->
+        Flow.copy source dest
+      end;
+      ()
+    | _ -> ()
+  in
+  begin
+  match Path.mkdir dest ~perm:0o700 with
+    | () -> debug_info dest
+    | exception (ex : exn) -> debug_error dest ex
+  end;
+  aux ~source ~dest
+
+  (* Path.with_open_in source @@ fun source ->  *)
+  (* Path.with_open_out ~create:(`Or_truncate 0o700) dest @@ fun dest -> *)
+  (* Flow.copy source dest *)
diff --git a/lib/core/Reporter.ml b/lib/prelude/Reporter.ml
similarity index 92%
rename from lib/core/Reporter.ml
rename to lib/prelude/Reporter.ml
index 8d5c04b..2303939 100644
--- a/lib/core/Reporter.ml
+++ b/lib/prelude/Reporter.ml
@@ -16,6 +16,8 @@ struct
     | Initialization_warning
     | Routing_error
     | Profiling
+    | Build_info
+    | Build_error
   [@@deriving show]
 
   let default_severity : t -> Asai.Diagnostic.severity =
@@ -35,6 +37,8 @@ struct
     | Initialization_warning -> Warning
     | Routing_error -> Error
     | Profiling -> Info
+    | Build_info -> Info
+    | Build_error -> Error
 
   let short_code : t -> string =
     show
diff --git a/lib/prelude/dune b/lib/prelude/dune
index c292fd2..1b573f6 100644
--- a/lib/prelude/dune
+++ b/lib/prelude/dune
@@ -1,6 +1,8 @@
 (library
  (name Forester_prelude)
- (libraries unix str ptime eio uucp bwd)
+ (preprocess
+  (pps ppx_deriving.show))
+ (libraries unix str ptime eio uucp bwd asai)
  (public_name forester.prelude))
 
 (env
diff --git a/lib/render/Build_latex.ml b/lib/render/Build_latex.ml
index 428a110..398e39b 100644
--- a/lib/render/Build_latex.ml
+++ b/lib/render/Build_latex.ml
@@ -1,4 +1,5 @@
 open Eio.Std
+open Forester_core
 open Forester_prelude
 
 type 'a env = 'a constraint 'a = <
-- 
2.45.1.windows.1

~jonsterling/forester-devel

Windows support v2 NEEDS REVISION

[PATCH v2] Windows support Export this patch