| 1 | open Logarion
|
|---|
| 2 |
|
|---|
| 3 | let is_older source dest = try
|
|---|
| 4 | Unix.((stat dest).st_mtime < (stat source).st_mtime) with _-> true
|
|---|
| 5 |
|
|---|
| 6 | let convert cs r (text, files) = match Text.str "Content-Type" text with
|
|---|
| 7 | | "" | "text/plain" ->
|
|---|
| 8 | let source = List.hd files in
|
|---|
| 9 | let dest = Filename.concat r.Conversion.dir (Text.short_id text) in
|
|---|
| 10 | List.fold_left
|
|---|
| 11 | (fun a f ->
|
|---|
| 12 | let dest = dest ^ f.Conversion.ext in
|
|---|
| 13 | if is_older source dest then (File_store.file dest (f.Conversion.page r text); true) else false
|
|---|
| 14 | || a)
|
|---|
| 15 | false cs
|
|---|
| 16 | | x -> Printf.eprintf "Can't convert Content-Type: %s file: %s" x text.Text.title; false
|
|---|
| 17 |
|
|---|
| 18 | let converters types kv =
|
|---|
| 19 | let t = [] in
|
|---|
| 20 | let t = if ("htm" = types || "all" = types) then
|
|---|
| 21 | (let htm = Html.init kv in
|
|---|
| 22 | Conversion.{ ext = Html.ext; page = Html.page htm; indices = Html.indices htm })::t
|
|---|
| 23 | else t in
|
|---|
| 24 | let t = if ("gmi" = types || "all" = types) then
|
|---|
| 25 | Conversion.{ ext = Gemini.ext; page = Gemini.page; indices = Gemini.indices}::t else t in
|
|---|
| 26 | t
|
|---|
| 27 |
|
|---|
| 28 | let directory converters noindex dir id kv =
|
|---|
| 29 | let empty = Topic_set.Map.empty in
|
|---|
| 30 | let repo = Conversion.{ id; dir; kv; topic_roots = []; topics = empty; texts = [] } in
|
|---|
| 31 | let fn (ts,ls,acc) ((elt,_) as r) =
|
|---|
| 32 | (Topic_set.to_map ts (Text.set "topics" elt)), elt::ls,
|
|---|
| 33 | if convert converters repo r then acc+1 else acc in
|
|---|
| 34 | let topics, texts, count = File_store.(fold ~dir ~order:newest fn (empty,[],0)) in
|
|---|
| 35 | let topic_roots = try List.rev @@ String_set.list_of_csv (Store.KV.find "Topics" kv)
|
|---|
| 36 | with Not_found -> Topic_set.roots topics in
|
|---|
| 37 | let repo = Conversion.{ repo with topic_roots; topics; texts } in
|
|---|
| 38 | if not noindex then List.iter (fun c -> c.Conversion.indices repo) converters;
|
|---|
| 39 | Printf.printf "Converted: %d Indexed: %d\n" count (List.length texts)
|
|---|
| 40 |
|
|---|
| 41 | let at_path types noindex path =
|
|---|
| 42 | match path with "" -> prerr_endline "unspecified text file or directory"
|
|---|
| 43 | | dir when Sys.file_exists dir && Sys.is_directory dir ->
|
|---|
| 44 | let fname = Filename.concat dir "index.pck" in
|
|---|
| 45 | (match Header_pack.of_string @@ File_store.to_string fname with
|
|---|
| 46 | | Error s -> prerr_endline s
|
|---|
| 47 | | Ok { info; peers; _ } ->
|
|---|
| 48 | let kv = let f = Filename.concat dir ".convert.conf" in (* TODO: better place to store convert conf? *)
|
|---|
| 49 | if Sys.file_exists f then File_store.of_kv_file f else Store.KV.empty in
|
|---|
| 50 | let kv = if Store.KV.mem "Title" kv then kv
|
|---|
| 51 | else Store.KV.add "Title" info.Header_pack.title kv in
|
|---|
| 52 | let kv = Store.KV.add "Locations" (String.concat ";\n" info.Header_pack.locations) kv in
|
|---|
| 53 | let kv = Store.KV.add "Peers" (String.concat ";\n" Header_pack.(to_str_list peers)) kv in
|
|---|
| 54 | let cs = converters types kv in
|
|---|
| 55 | directory cs noindex dir info.Header_pack.id kv)
|
|---|
| 56 | | path when Sys.file_exists path ->
|
|---|
| 57 | let repo = Conversion.{
|
|---|
| 58 | id = ""; dir = ""; kv = Store.KV.empty; topic_roots = [];
|
|---|
| 59 | topics = Topic_set.Map.empty; texts = [] } in
|
|---|
| 60 | let cs = converters types repo.kv in
|
|---|
| 61 | (match File_store.to_text path with
|
|---|
| 62 | | Ok text -> ignore @@ convert cs repo (text, [path])
|
|---|
| 63 | | Error s -> prerr_endline s)
|
|---|
| 64 | | path -> Printf.eprintf "Path doesn't exist: %s" path
|
|---|
| 65 |
|
|---|
| 66 | open Cmdliner
|
|---|
| 67 | let term =
|
|---|
| 68 | let path = Arg.(value & pos 0 string "" & info [] ~docv:"path"
|
|---|
| 69 | ~doc:"Text file or directory to convert. Ff directory is provided, it must contain an index.pck (see: txt index)") in
|
|---|
| 70 | let types = Arg.(value & opt string "all" & info ["t"; "type"] ~docv:"output type"
|
|---|
| 71 | ~doc:"Convert to file type") in
|
|---|
| 72 | let noindex = Arg.(value & flag & info ["noindex"]
|
|---|
| 73 | ~doc:"Don't create indices in target format") in
|
|---|
| 74 | Term.(const at_path $ types $ noindex $ path),
|
|---|
| 75 | Term.info "convert" ~doc:"convert texts"
|
|---|
| 76 | ~man:[ `S "DESCRIPTION"; `P "Convert text or indexed texts within a directory to another format.
|
|---|
| 77 | If path is a directory must contain an index.pck. Run `txt index` first." ]
|
|---|
