source: code/trunk/cli/convert.ml@ 40

Last change on this file since 40 was 39, checked in by fox, 2 years ago

Read References field; referred by listing; test & tidy documentation

File size: 4.4 KB
Line 
1open Logarion
2
3(*TODO: move to converters (style, feed checks)*)
4let is_older s d = try Unix.((stat d).st_mtime < (stat s).st_mtime) with _-> true
5
6let convert cs r (text, files) = match Text.str "Content-Type" text with
7 | "" | "text/plain" ->
8 let source = List.hd files in
9 let dest = Filename.concat r.Conversion.dir (Text.short_id text) in
10 List.fold_left
11 (fun a f ->
12 match f.Conversion.page with None -> false || a
13 | Some page ->
14 let dest = dest ^ f.Conversion.ext in
15 (if is_older source dest then (File_store.file dest (page r text); true) else false)
16 || a)
17 false cs
18 | x -> Printf.eprintf "Can't convert Content-Type: %s file: %s" x text.Text.title; false
19
20let converters types kv =
21 let n = String.split_on_char ',' types in
22 let t = [] in
23 let t = if List.(mem "all" n || mem "htm" n) then (Html.converter kv)::t else t in
24 let t = if List.(mem "all" n || mem "atom" n) then (Atom.converter "text/html")::t else t in
25 let t = if List.(mem "all" n || mem "gmi" n) then (Gemini.converter)::t else t in
26 let t = if List.(mem "all" n || mem "gmi-atom" n) then (Atom.converter "text/gemini")::t else t in
27 t
28
29let acc_ref id t a =
30 Conversion.Id_map.update t (function
31 | Some s -> Some (Conversion.Ref_set.add id s)
32 | None -> Some (Conversion.Ref_set.singleton id)
33 ) a
34
35let fold_refs text refs = String_set.fold (acc_ref text.Text.id) (Text.set "references" text) refs
36
37let directory converters noindex repo =
38 let fn (ts,refs,ls,acc) ((elt,_) as r) =
39 Topic_set.to_map ts (Text.set "topics" elt),
40 fold_refs elt refs,
41 elt::ls,
42 if convert converters repo r then acc+1 else acc in
43 let topics, references, texts, count =
44 File_store.(fold ~dir:repo.Conversion.dir ~order:newest fn (Topic_set.Map.empty, Conversion.Id_map.empty, [], 0)) in
45 let topic_roots = try List.rev @@ String_set.list_of_csv (Store.KV.find "Topics" repo.kv)
46 with Not_found -> Topic_set.roots topics in
47 let repo = Conversion.{ repo with topic_roots; topics; references; texts } in
48 if not noindex then List.iter (fun c -> match c.Conversion.indices with None -> () | Some f -> f repo) converters;
49 Printf.printf "Converted: %d Indexed: %d\n" count (List.length texts)
50
51let load_kv dir =
52 let kv = File_store.of_kv_file () in
53 let idx = Filename.concat dir "index.pck" in
54 if not (Sys.file_exists idx) then kv else
55 match Header_pack.of_string @@ File_store.to_string (idx) with
56 | Error s -> prerr_endline s; kv
57 | Ok { info; peers; _ } ->
58 let kv = if Store.KV.mem "Id" kv then kv else Store.KV.add "Id" info.Header_pack.id kv in
59 let kv = if Store.KV.mem "Title" kv then kv else Store.KV.add "Title" info.Header_pack.title kv in
60 let kv = if Store.KV.mem "Locations" kv then kv else Store.KV.add "Locations" (String.concat ";\n" info.Header_pack.locations) kv in
61 let kv = Store.KV.add "Peers" (String.concat ";\n" Header_pack.(to_str_list peers)) kv in
62 kv
63
64let at_path types noindex path = match path with
65 | "" -> prerr_endline "unspecified text file or directory"
66 | path when Sys.file_exists path ->
67 if Sys.is_directory path then (
68 let kv = load_kv path in
69 let repo = { (Conversion.empty ()) with dir = path; kv } in
70 directory (converters types kv) noindex repo
71 ) else (
72 match File_store.to_text path with
73 | Error s -> prerr_endline s
74 | Ok text ->
75 let dir = "." in
76 let references = File_store.(fold ~dir ~order:newest
77 (fun refs (elt, _) -> fold_refs elt refs) Conversion.Id_map.empty) in
78 Conversion.Id_map.iter
79 (fun k v -> Conversion.Ref_set.iter (fun e -> Printf.eprintf "%s %s\n" k e) v)
80 references;
81 let repo = { (Conversion.empty ()) with dir; kv = load_kv ""; references } in
82 ignore @@ convert (converters types repo.kv) repo (text, [path])
83 )
84 | path -> Printf.eprintf "Path doesn't exist: %s" path
85
86open Cmdliner
87let term =
88 let path = Arg.(value & pos 0 string "" & info [] ~docv:"path"
89 ~doc:"Text file or directory to convert. If directory is provided, it must contain an index.pck (see: txt index)") in
90 let types = Arg.(value & opt string "all" & info ["t"; "type"] ~docv:"output type"
91 ~doc:"Convert to file type") in
92 let noindex = Arg.(value & flag & info ["noindex"]
93 ~doc:"Don't create indices in target format") in
94 Term.(const at_path $ types $ noindex $ path),
95 Term.info "convert" ~doc:"convert texts"
96 ~man:[ `S "DESCRIPTION"; `P "Convert text or indexed texts within a directory to another format.
97 If path is a directory must contain an index.pck. Run `txt index` first." ]
Note: See TracBrowser for help on using the repository browser.