Kento Okura: 1 Initial attempt at supporting windows 9 files changed, 118 insertions(+), 55 deletions(-)
Copy & paste the following snippet into your terminal to import this patchset into git:
curl -s https://lists.sr.ht/~jonsterling/forester-devel/patches/53360/mbox | git am -3Learn more about email & git
don't merge this I am just sending this out so that you can test it on your forest. Strangely, my personal forest of ~400 trees takes 54 seconds to render with this patch applied, whereas it takes 0.8 seconds on upstream. I guess a possible explanation is that the Eio_util.cp function is unacceptably slow, but I don't have a lot of assets that get copied. Will investigate. What has changed: * Replace Eio_util.copy_to_dir with a portable function. * Remove calls to Eio.Path.native * Rewrite Process.ml and remove ForestScanner.ml This was done with the goal of tracking down the mysterious "file not found" error I was encounting only on windows when `Forester_frontend.Parse.parse_file` was called. This did not end up fixing it, but I do think that the changed code is easier to debug. We'll see. Here is the trace when running on windows with a single tree: ``` ￫ info[Reporter.Message.Profiling] ￮ [0.000000s] read configuration +Parsing <fs:trees> +hello.tree ￫ info[Reporter.Message.Profiling] ￮ [0.001002s] parse trees ￫ info[Reporter.Message.Profiling] ￮ [0.000000s] expand, evaluate, and analyse forest ￫ info[Reporter.Message.Profiling] ￮ [0.002993s] render forest ``` Here is the trace when running on Linux via WSL: ``` ￫ info[Reporter.Message.Profiling] ￮ [0.001087s] read configuration +Parsing <fs:trees> +hello.tree +Parsed <fs:trees/hello.tree> ￫ info[Reporter.Message.Profiling] ￮ [0.004601s] parse trees ￫ info[Reporter.Message.Profiling] ￮ [0.000018s] expand, evaluate, and analyse forest ￫ info[Reporter.Message.Profiling] ￮ [0.565760s] render forest ``` Here is the trace (with some lines removed) on my own forest with ~400 trees. As you can see, parsing is still fast, which is where the most substantial change in the code happens. ``` ￫ info[Reporter.Message.Profiling] ￮ [0.000036s] read configuration ￫ info[Reporter.Message.Profiling] ￮ [0.076402s] parse trees ￫ info[Reporter.Message.Profiling] ￮ [0.000098s] expand, evaluate, and analyse forest ￫ info[Reporter.Message.Profiling] ￮ [54.811927s] render forest ``` TODOS: - Actually load files on windows... - In order to remove calls to native, I put some None's in various places, we'll want to reverse that later, perhaps when https://github.com/ocaml-multicore/eio/pull/738 is done. --- bin/forester/ForestScanner.ml | 20 ------------ bin/forester/ForestScanner.mli | 1 - bin/forester/Process.ml | 58 ++++++++++++++++++++++++++++------ lib/core/Forester_core.ml | 2 -- lib/frontend/Forest.ml | 22 ++++++------- lib/frontend/Parse.ml | 10 +++--- lib/prelude/Eio_util.ml | 35 ++++++++++++++++++-- test/dune | 2 +- test/fs.ml | 23 ++++++++++++++ 9 files changed, 118 insertions(+), 55 deletions(-) delete mode 100644 bin/forester/ForestScanner.ml delete mode 100644 bin/forester/ForestScanner.mli create mode 100644 test/fs.ml diff --git a/bin/forester/ForestScanner.ml b/bin/forester/ForestScanner.ml deleted file mode 100644 index 8dcb46d..0000000 --- a/bin/forester/ForestScanner.ml @@ -1,20 +0,0 @@ -module S = Algaeff.Sequencer.Make (struct type t = Eio.Fs.dir_ty Eio.Path.t end) - -let rec process_file fp = - if Eio.Path.is_directory fp then - process_dir fp - else - Eio.Path.split fp |> Option.iter @@ fun (dir, basename) -> - if Filename.extension basename = ".tree" && not @@ String.starts_with ~prefix:"." basename then - S.yield fp - -and process_dir dir = - try - Eio.Path.read_dir dir |> List.iter @@ fun fp -> - process_file Eio.Path.(dir / fp) - with Eio.Io (Eio.Fs.E (Permission_denied _), _) -> () - -let scan_directories dirs = - S.run @@ fun () -> - dirs |> List.iter @@ fun fp -> - process_dir fp diff --git a/bin/forester/ForestScanner.mli b/bin/forester/ForestScanner.mli deleted file mode 100644 index 50c0024..0000000 --- a/bin/forester/ForestScanner.mli @@ -1 +0,0 @@ -val scan_directories : Eio.Fs.dir_ty Eio.Path.t list -> Eio.Fs.dir_ty Eio.Path.t Seq.t diff --git a/bin/forester/Process.ml b/bin/forester/Process.ml index 0077d35..e0acc69 100644 --- a/bin/forester/Process.ml @@ -1,12 +1,50 @@ open Forester_core -let read_trees_in_dirs ~dev ?(ignore_malformed = false) dirs = - ForestScanner.scan_directories dirs |> List.of_seq |> List.filter_map @@ fun fp -> - Option.bind (Eio.Path.split fp) @@ fun (dir, basename) -> - let addr = Filename.chop_extension basename in - let source_path = if dev then Option.map Unix.realpath @@ Eio.Path.native fp else None in - match Forester_frontend.Parse.parse_file fp with - | Result.Ok code -> Some Code.{source_path; addr = Some addr; code} - | Result.Error err -> None - | exception exn -> - if ignore_malformed then None else raise exn +open Eio.Std +module Path = Eio.Path + +let ( / ) = Eio.Path.( / ) + +let rec read_trees_in_dir ?(ignore_malformed = false) t = + let open Path in + Eio.Std.traceln "Parsing %a" Path.pp t; + let entries = read_dir t in + entries |> List.iter (Eio.Std.traceln "%s"); + List.fold_left (fun acc entry -> + let path = t / entry in + if path |> is_directory then + acc @ (read_trees_in_dir path ~ignore_malformed) + else if path |> is_file then + match Forester_frontend.Parse.parse_file path with + | Ok code -> + Eio.Std.traceln "Parsed %a" Path.pp path; + Code.{source_path = None; addr = None; code} :: acc + | Error _ -> + Eio.Std.traceln "Failed to parse %a" Path.pp path; + acc + | exception exn -> + Eio.Std.traceln "exn"; + begin + if ignore_malformed then acc else raise exn + end + else acc + ) [] entries + + +let read_trees_in_dirs ~dev ?(ignore_malformed = false) dirs = + List.map (fun dir -> read_trees_in_dir dir ~ignore_malformed) dirs + |> List.concat + + +(* let read_trees_in_dirs ~dev ?(ignore_malformed = false) dirs = *) +(* ForestScanner.scan_directories dirs |> List.of_seq |> List.filter_map @@ fun fp -> *) +(* Option.bind (Eio.Path.split fp) @@ fun (dir, basename) -> *) +(* let addr = Filename.chop_extension basename in *) +(* let source_path = None in *) +(* Eio.Std.traceln "Parsing %a" Eio.Path.pp fp; *) +(* (* let source_path = if dev then Option.map Unix.realpath @@ Eio.Path.native fp else None in *) *) +(* match Forester_frontend.Parse.parse_file fp with *) +(* | Result.Ok code -> Some Code.{source_path; addr = Some addr; code} *) +(* | Result.Error err -> None *) +(* | exception exn -> *) +(* if ignore_malformed then None else raise exn *) diff --git a/lib/core/Forester_core.ml b/lib/core/Forester_core.ml index f424d05..b424a85 100644 --- a/lib/core/Forester_core.ml +++ b/lib/core/Forester_core.ml @@ -1,5 +1,4 @@ include Base - module Syn = Syn module Sem = Sem module Code = Code @@ -9,7 +8,6 @@ module Expand = Expand module Eval = Eval module Query = Query module BaseN = BaseN - module Reporter = Reporter module Range = Range module Prim = Prim diff --git a/lib/frontend/Forest.ml b/lib/frontend/Forest.ml index da815ef..5fbf4a9 100644 --- a/lib/frontend/Forest.ml +++ b/lib/frontend/Forest.ml @@ -280,25 +280,23 @@ let render_json ~cwd docs = let is_hidden_file fname = String.starts_with ~prefix:"." fname + +let ( / ) = Eio.Path.( / ) + let copy_theme ~env ~theme_dir = - let cwd = Eio.Stdenv.cwd env in - Eio.Path.read_dir theme_dir |> List.iter @@ fun fname -> - if not @@ is_hidden_file fname then - Eio.Path.native @@ Eio.Path.(theme_dir / fname) |> Option.iter @@ fun source -> - Eio_util.copy_to_dir ~env ~cwd ~source ~dest_dir:"output" + let fs = Eio.Stdenv.fs env in + Eio_util.cp ~source:theme_dir ~dest:(fs / "output") let copy_assets ~env ~assets_dirs = - let cwd = Eio.Stdenv.cwd env in + let fs = Eio.Stdenv.fs env in assets_dirs |> List.iter @@ fun assets_dir -> Eio.Path.read_dir assets_dir |> List.iter @@ fun fname -> - if not @@ is_hidden_file fname then - let path = Eio.Path.(assets_dir / fname) in - let source = Eio.Path.native_exn path in - Eio_util.copy_to_dir ~env ~cwd ~source ~dest_dir:"build"; - Eio_util.copy_to_dir ~env ~cwd ~source ~dest_dir:"output" + Eio_util.cp ~source:assets_dir ~dest:(fs / "build"); + Eio_util.cp ~source:assets_dir ~dest:(fs / "output") let copy_resources ~env = let cwd = Eio.Stdenv.cwd env in + let fs = Eio.Stdenv.fs env in Eio.Path.read_dir Eio.Path.(cwd / "build") |> List.iter @@ fun fname -> if not @@ is_hidden_file fname then let ext = Filename.extension fname in @@ -310,7 +308,7 @@ let copy_resources ~env = in dest_opt |> Option.iter @@ fun dest_dir -> if not @@ Eio_util.file_exists Eio.Path.(cwd / dest_dir / fname) then - Eio_util.copy_to_dir ~cwd ~env ~source:fp ~dest_dir + Eio_util.cp ~source:(fs / fp) ~dest:(fs / dest_dir) let render_trees ~cfg ~forest ~render_only : unit = let env = cfg.env in diff --git a/lib/frontend/Parse.ml b/lib/frontend/Parse.ml index 20af484..23ab247 100644 --- a/lib/frontend/Parse.ml +++ b/lib/frontend/Parse.ml @@ -140,13 +140,11 @@ let parse_channel filename ch = lexbuf.lex_curr_p <- { lexbuf.lex_curr_p with pos_fname = filename }; maybe_with_errors (fun () -> try_parse lexbuf) -let parse_file fp = - let filename = Eio.Path.native_exn fp in - let ch = open_in filename in - Fun.protect ~finally:(fun _ -> close_in ch) @@ fun _ -> - parse_channel filename ch - let parse_string str = Reporter.tracef "when parsing string" @@ fun () -> let lexbuf = Lexing.from_string str in maybe_with_errors (fun () -> try_parse lexbuf) + +let parse_file fp = + let content = Eio.Path.load fp in + parse_string content diff --git a/lib/prelude/Eio_util.ml b/lib/prelude/Eio_util.ml index 6b203b6..3018b3b 100644 --- a/lib/prelude/Eio_util.ml +++ b/lib/prelude/Eio_util.ml @@ -56,7 +56,36 @@ let file_exists path = try Eio.Path.with_open_in path @@ fun _ -> true with | Eio.Io (Eio.Fs.E (Eio.Fs.Not_found _), _) -> false -(* TODO: make this portable *) -let copy_to_dir ~env ~cwd ~source ~dest_dir = - run_process ~quiet:true ~env ~cwd ["cp"; "-R"; source; dest_dir ^ "/" ] +let ( / ) = Path.( / ) +let cp ~source ~dest = + let rec aux ~source ~dest = + match Path.kind ~follow:false source with + | `Directory -> + begin + match Path.mkdir dest ~perm:0o700 with + | () -> () (* traceln "created directory %a" Path.pp dest *) + | exception ex -> () (* traceln "@[<h>%a@]" Eio.Exn.pp ex *) + end; + Path.read_dir source |> List.iter (function + | "_build" -> () + | item when String.starts_with ~prefix:"." item -> () + | item -> aux ~source:(source / item) ~dest:(dest / item) + ) + | `Regular_file -> + begin + match Path.load source with + | s -> + begin + match Path.save ~create:(`Or_truncate 0o666) dest s with + | () -> () (* traceln "write %a -> ok" Path.pp dest *) + | exception ex -> () (* traceln "@[<h>%a@]" Eio.Exn.pp ex *) + end + | exception ex -> traceln "@[<h>%a@]" Eio.Exn.pp ex + end + | _ -> () + in + match Path.mkdir dest ~perm:0o700 with + | () -> () (* traceln "mkdir %a -> ok" Path.pp dest *) + | exception ex -> (); (* traceln "@[<h>%a@]" Eio.Exn.pp ex; *) + aux ~source ~dest diff --git a/test/dune b/test/dune index 7981f54..7ee16fd 100644 --- a/test/dune +++ b/test/dune @@ -1,3 +1,3 @@ (tests - (names parse) + (names parse fs) (libraries forester.frontend)) diff --git a/test/fs.ml b/test/fs.ml new file mode 100644 index 0000000..fe9e10f --- /dev/null +++ b/test/fs.ml @@ -0,0 +1,23 @@ +module Path = Eio.Path + +let () = Eio.Exn.Backend.show := false + +open Eio.Std + +let ( / ) = Path.( / ) + +let run ?clear:(paths = []) fn = + Eio_main.run @@ fun env -> + let cwd = Eio.Stdenv.cwd env in + List.iter (fun p -> Eio.Path.rmtree ~missing_ok:true (cwd / p)) paths; + fn env + +let try_read_file path = + match Path.load path with + | s -> traceln "read %a -> %S" Path.pp path s + | exception ex -> traceln "@[<h>%a@]" Eio.Exn.pp ex + +let () = + run @@ fun env -> + let cwd = Eio.Stdenv.cwd env in + try_read_file (cwd / "fs.ml") -- 2.45.1.windows.1
Seems like development of this will have to be put on hold until opam-repository for windows gets an update containing pure-html