Windows support — sourcehut lists

[PATCH] Windows support Export this patch

Notable changes:

- Rewrite Process.ml and remove ForestScanner
  I noticed no performance impedence on my personal forest of 400 trees,
  on both windows and linux.

- Replace Eio.Stdenv.fs with Eio.Stdenv.cwd. It caused a strange "file
  not found" error on windows.

- Move Reporter to Prelude because I wanted diagnostics for some
  functions in Eio_utils.

- Added an unbearably slow but portable version of a 'cp -R' function.
  The reason it is so slow is that I am reading everything to a string
  and saving it again. Need to revisit this to use Flows or something,
  but wanted to get this patch out there as I finally made a forest
  build on windows.

I'd be interested to know if this patch works well on macOS. For
testing, you should comment out the bit in Forest.ml that copies the
resources. Then it should be as fast as upstream.

Jon Sterling <jon@jonmsterling.com>

8 months ago

After commenting out the body of copy_resources (which contains an 
unbound identifier), I ran this code on my first. Immediately in the 
“read and parse the forest” phase, I got the following error:

      ￫ error[Reporter.Message.Parse_error]
      ꭍ ￮ when parsing string
     Fatal error: exception Sys_error("could not open file ")

Best,
Jon

---
 bin/forester/ForestScanner.ml     | 20 ------------
 bin/forester/ForestScanner.mli    |  1 -
 bin/forester/Process.ml           | 52 +++++++++++++++++++++++++------
 bin/forester/main.ml              |  3 +-
 lib/core/Forester_core.ml         |  4 +--
 lib/frontend/Forest.ml            | 27 ++++++++--------
 lib/frontend/Parse.ml             |  9 ++----
 lib/prelude/Eio_util.ml           | 47 ++++++++++++++++++++++++++--
 lib/{core => prelude}/Reporter.ml |  4 +++
 lib/prelude/dune                  |  4 ++-
 lib/render/Build_latex.ml         |  1 +
 11 files changed, 113 insertions(+), 59 deletions(-)
 delete mode 100644 bin/forester/ForestScanner.ml
 delete mode 100644 bin/forester/ForestScanner.mli
 rename lib/{core => prelude}/Reporter.ml (92%)

diff --git a/bin/forester/ForestScanner.ml b/bin/forester/ForestScanner.ml
deleted file mode 100644
index 8dcb46d..0000000
--- a/bin/forester/ForestScanner.ml
@@ -1,20 +0,0 @@
-module S = Algaeff.Sequencer.Make (struct type t = Eio.Fs.dir_ty Eio.Path.t end)
-
-let rec process_file fp =
-  if Eio.Path.is_directory fp then
-    process_dir fp
-  else
-    Eio.Path.split fp |> Option.iter @@ fun (dir, basename) ->
-    if Filename.extension basename = ".tree" && not @@ String.starts_with ~prefix:"." basename then
-      S.yield fp
-
-and process_dir dir =
-  try
-    Eio.Path.read_dir dir |> List.iter @@ fun fp ->
-    process_file Eio.Path.(dir / fp)
-  with Eio.Io (Eio.Fs.E (Permission_denied _), _) -> ()
-
-let scan_directories dirs =
-  S.run @@ fun () ->
-  dirs |> List.iter @@ fun fp ->
-  process_dir fp
diff --git a/bin/forester/ForestScanner.mli b/bin/forester/ForestScanner.mli
deleted file mode 100644
index 50c0024..0000000
--- a/bin/forester/ForestScanner.mli
@@ -1 +0,0 @@
-val scan_directories : Eio.Fs.dir_ty Eio.Path.t list -> Eio.Fs.dir_ty Eio.Path.t Seq.t
diff --git a/bin/forester/Process.ml b/bin/forester/Process.ml
index 0077d35..6231ab5 100644
--- a/bin/forester/Process.ml
@@ -1,12 +1,46 @@
 open Forester_core
 
+open Eio.Std
+module Path = Eio.Path
+
+let ( / ) = Eio.Path.( / )
+
+let rec read_trees_in_dir ?(ignore_malformed = false) t =
+  let open Path in
+  let entries = read_dir t in
+  List.fold_left (fun acc entry ->
+    let path = t / entry in
+    if path |> is_directory then
+      acc @ (read_trees_in_dir path ~ignore_malformed)
+    else if path |> is_file then
+    match Forester_frontend.Parse.parse_file path with
+    | Ok code ->
+      begin
+        match Eio.Path.split path with
+        | Some t ->
+            let addr =
+              snd t
+              |> String.split_on_char '/'
+              |> List.rev
+              |> List.hd
+              |> String.split_on_char '\\'
+              |> List.rev
+              |> List.hd
+              |> Filename.chop_extension
+            in
+            Code.{ source_path = None; addr = Some addr; code } :: acc
+        | None -> acc
+      end
+    | Error _ ->
+        acc
+    | exception exn ->
+      begin
+        if ignore_malformed then acc else raise exn
+      end
+    else acc
+  ) [] entries
+
+
 let read_trees_in_dirs ~dev ?(ignore_malformed = false) dirs =
-  ForestScanner.scan_directories dirs |> List.of_seq |> List.filter_map @@ fun fp ->
-  Option.bind (Eio.Path.split fp) @@ fun (dir, basename) ->
-  let addr = Filename.chop_extension basename in
-  let source_path = if dev then Option.map Unix.realpath @@ Eio.Path.native fp else None in
-  match Forester_frontend.Parse.parse_file fp with
-  | Result.Ok code -> Some Code.{source_path; addr = Some addr; code}
-  | Result.Error err -> None
-  | exception exn ->
-    if ignore_malformed then None else raise exn
+  List.map (fun dir -> read_trees_in_dir dir ~ignore_malformed) dirs
+  |> List.concat
diff --git a/bin/forester/main.ml b/bin/forester/main.ml
index ad72395..47f5648 100644
--- a/bin/forester/main.ml
@@ -6,7 +6,7 @@ open Cmdliner
 module Tty = Asai.Tty.Make (Forester_core.Reporter.Message)
 
 let make_dir ~env dir =
-  Eio.Path.(Eio.Stdenv.fs env / dir)
+  Eio.Path.(Eio.Stdenv.cwd env / dir)
 
 let make_dirs ~env =
   List.map (make_dir ~env)
@@ -49,7 +49,6 @@ let build ~env config_filename dev render_only ignore_tex_cache no_assets no_the
     timed_log "expand, evaluate, and analyse forest" @@ fun () ->
     Forest.plant_forest parsed_trees
   in
-  timed_log "render forest" @@ fun () ->
   let render_only =
     render_only |>
     Option.map @@
diff --git a/lib/core/Forester_core.ml b/lib/core/Forester_core.ml
index a541fd3..1cc0218 100644
--- a/lib/core/Forester_core.ml
+++ b/lib/core/Forester_core.ml
@@ -1,7 +1,6 @@
 module Base = Base
 
 include Base
-
 module Syn = Syn
 module Sem = Sem
 module Code = Code
@@ -11,10 +10,9 @@ module Expand = Expand
 module Eval = Eval
 module Query = Query
 module BaseN = BaseN
-
-module Reporter = Reporter
 module Range = Range
 module Prim = Prim
+module Reporter = Forester_prelude.Reporter
 
 module TeX_cs = TeX_cs
 module Symbol = Symbol
diff --git a/lib/frontend/Forest.ml b/lib/frontend/Forest.ml
index 5ea863c..7770966 100644
--- a/lib/frontend/Forest.ml
+++ b/lib/frontend/Forest.ml
@@ -159,22 +159,19 @@ let render_json ~cfg ~cwd docs =
 let is_hidden_file fname =
   String.starts_with ~prefix:"." fname
 
+
+let ( / ) = Eio.Path.( / )
+
 let copy_theme ~env ~theme_dir =
   let cwd = Eio.Stdenv.cwd env in
-  Eio.Path.read_dir theme_dir |> List.iter @@ fun fname ->
-  if not @@ is_hidden_file fname then
-    Eio.Path.native @@ Eio.Path.(theme_dir / fname) |> Option.iter @@ fun source ->
-    Eio_util.copy_to_dir ~env ~cwd ~source ~dest_dir:"output"
+  Eio_util.cp ~source:theme_dir ~dest:(cwd / "output") ()
 
 let copy_assets ~env ~assets_dirs =
   let cwd = Eio.Stdenv.cwd env in
   assets_dirs |> List.iter @@ fun assets_dir ->
   Eio.Path.read_dir assets_dir |> List.iter @@ fun fname ->
-  if not @@ is_hidden_file fname then
-    let path = Eio.Path.(assets_dir / fname) in
-    let source = Eio.Path.native_exn path in
-    Eio_util.copy_to_dir ~env ~cwd ~source ~dest_dir:"build";
-    Eio_util.copy_to_dir ~env ~cwd ~source ~dest_dir:"output"
+  Eio_util.cp ~source:assets_dir ~dest:(cwd / "build") ();
+  Eio_util.cp ~source:assets_dir ~dest:(cwd / "output") ()
 
 let copy_resources ~env =
   let cwd = Eio.Stdenv.cwd env in
@@ -189,13 +186,13 @@ let copy_resources ~env =
     in
     dest_opt |> Option.iter @@ fun dest_dir ->
     if not @@ Eio_util.file_exists Eio.Path.(cwd / dest_dir / fname) then
-      Eio_util.copy_to_dir ~cwd ~env ~source:fp ~dest_dir
+      Eio_util.cp ~source:(fs / fp) ~dest:(fs / dest_dir) ()
 
 let last_changed env forest scope =
   let (let*) = Option.bind in
   let* tree = M.find_opt scope forest.trees in

Jon Sterling <jon@jonmsterling.com>

8 months ago
The ‘fs’ identifier here seems to be unbound, so I could not compile 
the code as-is.

   let* source_path = tree.fm.source_path in
-  let path = Eio.Path.(Eio.Stdenv.fs env / source_path) in
+  let path = Eio.Path.(Eio.Stdenv.cwd env / source_path) in
   let stat  = Eio.Path.stat ~follow:true path in
   let* mtime = Some stat.mtime in
   let* ptime = Ptime.of_float_s mtime in
@@ -214,7 +211,6 @@ let render_trees ~cfg ~(forest : forest) ~render_only : unit =
     let root, trees, run_query, last_changed, enqueue_latex =
       cfg.root, forest.trees, forest.run_query, last_changed env forest, LaTeX_queue.enqueue
   end
-
   in
 
   let module C = Compile.Make (I) () in
@@ -231,6 +227,7 @@ let render_trees ~cfg ~(forest : forest) ~render_only : unit =
     C.compile_tree tree
   in
 
+
   let trees =
     match render_only with
     | None -> forest.trees |> M.to_seq |> Seq.map snd |> List.of_seq
@@ -243,9 +240,11 @@ let render_trees ~cfg ~(forest : forest) ~render_only : unit =
   in
 
   trees
-  |> Sem.Util.sort
-  |> List.iter render_tree;
+    |> Sem.Util.sort
+    |> List.iter render_tree;
+
   render_json ~cfg ~cwd forest.trees;
+
   if not cfg.no_assets then
     copy_assets ~env ~assets_dirs:cfg.assets_dirs;
   if not cfg.no_theme then
diff --git a/lib/frontend/Parse.ml b/lib/frontend/Parse.ml
index 20af484..a35bd15 100644
--- a/lib/frontend/Parse.ml
+++ b/lib/frontend/Parse.ml
@@ -140,13 +140,10 @@ let parse_channel filename ch =
   lexbuf.lex_curr_p <- { lexbuf.lex_curr_p with pos_fname = filename };
   maybe_with_errors (fun () -> try_parse lexbuf)
 
-let parse_file fp =
-  let filename = Eio.Path.native_exn fp in
-  let ch = open_in filename in
-  Fun.protect ~finally:(fun _ -> close_in ch) @@ fun _ ->
-  parse_channel filename ch
-
 let parse_string str =
   Reporter.tracef "when parsing string" @@ fun () ->
   let lexbuf = Lexing.from_string str in
   maybe_with_errors (fun () -> try_parse lexbuf)
+
+let parse_file fp =
+  parse_string @@ Eio.Path.load fp
diff --git a/lib/prelude/Eio_util.ml b/lib/prelude/Eio_util.ml
index 6b203b6..ee453d0 100644
--- a/lib/prelude/Eio_util.ml
+++ b/lib/prelude/Eio_util.ml
@@ -1,3 +1,4 @@
+open Reporter
 open Eio
 
 let formatter_of_writer w =
@@ -56,7 +57,47 @@ let file_exists path =
   try Eio.Path.with_open_in path @@ fun _ -> true with
   | Eio.Io (Eio.Fs.E (Eio.Fs.Not_found _), _) -> false
 
-(* TODO: make this portable *)
-let copy_to_dir ~env ~cwd ~source ~dest_dir =
-  run_process ~quiet:true ~env ~cwd ["cp"; "-R"; source; dest_dir ^ "/" ]
+let ( / ) = Path.( / )
 
+(* This function is too slow *)
+let cp ?(debug = false) ~source ~dest () =
+  let debug_info = 
+    if debug then (fun p -> Reporter.emitf Build_info "%a" Eio.Path.pp p) 
+    else Fun.const () 
+  in
+  let debug_error = 
+    if debug then (fun p ex -> Reporter.emitf Build_info "%a: %a" Eio.Path.pp p Eio.Exn.pp ex)
+    else (fun _ _ -> ())
+  in
+  let rec aux ~source ~dest =
+    match Path.kind ~follow:false source with
+    | `Directory ->
+      begin
+        match Path.mkdir dest ~perm:0o700 with
+        | () -> debug_info dest
+        | exception ex -> debug_error dest ex
+      end;
+      Path.read_dir source |> List.iter (function
+        | item when String.starts_with ~prefix:"." item -> ()
+        | item -> aux ~source:(source / item) ~dest:(dest / item)
+      )
+    | `Regular_file ->
+      (* I am loading t oa string and the saving again. *)
+      begin
+        match Path.load source with
+        | s ->
+          begin
+            match Path.save ~create:(`Or_truncate 0o666) dest s with
+              | () -> debug_info dest
+              | exception ex -> debug_error dest ex
+            end
+        | exception ex -> traceln "@[<h>%a@]" Eio.Exn.pp ex
+      end
+    | _ -> ()
+  in
+  begin
+  match Path.mkdir dest ~perm:0o700 with
+    | () -> debug_info dest
+    | exception (ex : exn) -> debug_error dest ex
+  end;
+  aux ~source ~dest
diff --git a/lib/core/Reporter.ml b/lib/prelude/Reporter.ml
similarity index 92%
rename from lib/core/Reporter.ml
rename to lib/prelude/Reporter.ml
index 8d5c04b..2303939 100644
--- a/lib/core/Reporter.ml
+++ b/lib/prelude/Reporter.ml
@@ -16,6 +16,8 @@ struct
     | Initialization_warning
     | Routing_error
     | Profiling
+    | Build_info
+    | Build_error
   [@@deriving show]
 
   let default_severity : t -> Asai.Diagnostic.severity =
@@ -35,6 +37,8 @@ struct
     | Initialization_warning -> Warning
     | Routing_error -> Error
     | Profiling -> Info
+    | Build_info -> Info
+    | Build_error -> Error
 
   let short_code : t -> string =
     show
diff --git a/lib/prelude/dune b/lib/prelude/dune
index c292fd2..1b573f6 100644
--- a/lib/prelude/dune
+++ b/lib/prelude/dune
@@ -1,6 +1,8 @@
 (library
  (name Forester_prelude)
- (libraries unix str ptime eio uucp bwd)
+ (preprocess
+  (pps ppx_deriving.show))
+ (libraries unix str ptime eio uucp bwd asai)
  (public_name forester.prelude))
 
 (env
diff --git a/lib/render/Build_latex.ml b/lib/render/Build_latex.ml
index 428a110..398e39b 100644
--- a/lib/render/Build_latex.ml
+++ b/lib/render/Build_latex.ml
@@ -1,4 +1,5 @@
 open Eio.Std
+open Forester_core
 open Forester_prelude
 
 type 'a env = 'a constraint 'a = <
-- 
2.45.1.windows.1

~jonsterling/forester-devel

Windows support v1 SUPERSEDED

[PATCH] Windows support Export this patch