| [2] | 1 | open Logarion
|
|---|
| 2 |
|
|---|
| [20] | 3 | (*TODO: move to converters (style, feed checks)*)
|
|---|
| 4 | let is_older s d = try Unix.((stat d).st_mtime < (stat s).st_mtime) with _-> true
|
|---|
| [2] | 5 |
|
|---|
| [3] | 6 | let convert cs r (text, files) = match Text.str "Content-Type" text with
|
|---|
| 7 | | "" | "text/plain" ->
|
|---|
| 8 | let source = List.hd files in
|
|---|
| 9 | let dest = Filename.concat r.Conversion.dir (Text.short_id text) in
|
|---|
| [43] | 10 | List.fold_left (fun a f ->
|
|---|
| 11 | match f.Conversion.page with None -> false || a
|
|---|
| 12 | | Some page ->
|
|---|
| 13 | let dest = dest ^ f.Conversion.ext in
|
|---|
| 14 | (if is_older source dest || Conversion.Rel.Id_map.mem text.Text.id r.relations
|
|---|
| 15 | then (File_store.file dest (page r text); true) else false)
|
|---|
| 16 | || a)
|
|---|
| [3] | 17 | false cs
|
|---|
| 18 | | x -> Printf.eprintf "Can't convert Content-Type: %s file: %s" x text.Text.title; false
|
|---|
| [2] | 19 |
|
|---|
| [3] | 20 | let converters types kv =
|
|---|
| [19] | 21 | let n = String.split_on_char ',' types in
|
|---|
| [3] | 22 | let t = [] in
|
|---|
| [19] | 23 | let t = if List.(mem "all" n || mem "htm" n) then (Html.converter kv)::t else t in
|
|---|
| 24 | let t = if List.(mem "all" n || mem "atom" n) then (Atom.converter "text/html")::t else t in
|
|---|
| 25 | let t = if List.(mem "all" n || mem "gmi" n) then (Gemini.converter)::t else t in
|
|---|
| 26 | let t = if List.(mem "all" n || mem "gmi-atom" n) then (Atom.converter "text/gemini")::t else t in
|
|---|
| [3] | 27 | t
|
|---|
| [2] | 28 |
|
|---|
| [20] | 29 | let directory converters noindex repo =
|
|---|
| [42] | 30 | let order = File_store.oldest in
|
|---|
| 31 | let repo =
|
|---|
| [43] | 32 | let open Conversion in
|
|---|
| 33 | let rels = File_store.fold ~dir:repo.dir ~order Rel.acc_txt Rel.empty_map in
|
|---|
| 34 | let relations = Peers.fold Rel.acc_pck rels in
|
|---|
| 35 | { repo with relations } in
|
|---|
| [42] | 36 | let acc (ts,ls,acc) ((elt,_) as r) = Topic_set.to_map ts (Text.set "topics" elt), elt::ls,
|
|---|
| 37 | if convert converters repo r then acc+1 else acc in
|
|---|
| 38 | let topics, texts, count =
|
|---|
| 39 | File_store.fold ~dir:repo.Conversion.dir ~order acc (Topic_set.Map.empty, [], 0) in
|
|---|
| [20] | 40 | let topic_roots = try List.rev @@ String_set.list_of_csv (Store.KV.find "Topics" repo.kv)
|
|---|
| [3] | 41 | with Not_found -> Topic_set.roots topics in
|
|---|
| [42] | 42 | let repo = Conversion.{ repo with topic_roots; topics; texts = List.rev texts } in
|
|---|
| [41] | 43 | if not noindex then
|
|---|
| 44 | List.iter (fun c -> match c.Conversion.indices with None -> () | Some f -> f repo) converters;
|
|---|
| [3] | 45 | Printf.printf "Converted: %d Indexed: %d\n" count (List.length texts)
|
|---|
| [2] | 46 |
|
|---|
| [20] | 47 | let load_kv dir =
|
|---|
| [21] | 48 | let kv = File_store.of_kv_file () in
|
|---|
| [20] | 49 | let idx = Filename.concat dir "index.pck" in
|
|---|
| 50 | if not (Sys.file_exists idx) then kv else
|
|---|
| 51 | match Header_pack.of_string @@ File_store.to_string (idx) with
|
|---|
| 52 | | Error s -> prerr_endline s; kv
|
|---|
| [15] | 53 | | Ok { info; peers; _ } ->
|
|---|
| [20] | 54 | let kv = if Store.KV.mem "Id" kv then kv else Store.KV.add "Id" info.Header_pack.id kv in
|
|---|
| [19] | 55 | let kv = if Store.KV.mem "Title" kv then kv else Store.KV.add "Title" info.Header_pack.title kv in
|
|---|
| [21] | 56 | let kv = if Store.KV.mem "Locations" kv then kv else Store.KV.add "Locations" (String.concat ";\n" info.Header_pack.locations) kv in
|
|---|
| [15] | 57 | let kv = Store.KV.add "Peers" (String.concat ";\n" Header_pack.(to_str_list peers)) kv in
|
|---|
| [20] | 58 | kv
|
|---|
| 59 |
|
|---|
| 60 | let at_path types noindex path = match path with
|
|---|
| 61 | | "" -> prerr_endline "unspecified text file or directory"
|
|---|
| [13] | 62 | | path when Sys.file_exists path ->
|
|---|
| [20] | 63 | if Sys.is_directory path then (
|
|---|
| 64 | let kv = load_kv path in
|
|---|
| 65 | let repo = { (Conversion.empty ()) with dir = path; kv } in
|
|---|
| 66 | directory (converters types kv) noindex repo
|
|---|
| 67 | ) else (
|
|---|
| 68 | match File_store.to_text path with
|
|---|
| 69 | | Error s -> prerr_endline s
|
|---|
| 70 | | Ok text ->
|
|---|
| [39] | 71 | let dir = "." in
|
|---|
| [43] | 72 | let open Conversion in
|
|---|
| 73 | let relations = File_store.(fold ~dir ~order:newest Rel.acc_txt Rel.empty_map) in
|
|---|
| 74 | let repo = { (Conversion.empty ()) with dir; kv = load_kv ""; relations } in
|
|---|
| [20] | 75 | ignore @@ convert (converters types repo.kv) repo (text, [path])
|
|---|
| 76 | )
|
|---|
| [13] | 77 | | path -> Printf.eprintf "Path doesn't exist: %s" path
|
|---|
| [2] | 78 |
|
|---|
| [66] | 79 | open Cmdliner
|
|---|
| 80 |
|
|---|
| 81 | let path = Arg.(value & pos 0 string "" & info [] ~docv:"path" ~doc:"Text file or directory to convert. If directory is provided, it must contain an index.pck (see: txt index)")
|
|---|
| 82 | let types = Arg.(value & opt string "all" & info ["t"; "type"] ~docv:"output type" ~doc:"Convert to file type")
|
|---|
| 83 | let noindex = Arg.(value & flag & info ["noindex"] ~doc:"Don't create indices in target format")
|
|---|
| 84 |
|
|---|
| 85 | let convert_t = Term.(const at_path $ types $ noindex $ path)
|
|---|
| 86 |
|
|---|
| 87 | let cmd =
|
|---|
| 88 | let doc = "Convert texts" in
|
|---|
| 89 | let man = [
|
|---|
| 90 | `S Manpage.s_description;
|
|---|
| 91 | `P "Convert text or indexed texts within a directory to another format.";
|
|---|
| 92 | `P "If path is a directory must contain an index.pck.";
|
|---|
| 93 | `P "Run `txt index` first." ]
|
|---|
| 94 | in
|
|---|
| [69] | 95 | let info = Cmd.info "convert" ~doc ~man in
|
|---|
| [66] | 96 | Cmd.v info convert_t
|
|---|
