source: code/trunk/cli/convert.ml@ 42

open Logarion

(*TODO: move to converters (style, feed checks)*)
let is_older s d = try Unix.((stat d).st_mtime < (stat s).st_mtime) with _-> true

let convert cs r (text, files) = match Text.str "Content-Type" text with
        | "" | "text/plain" ->
                let source = List.hd files in
                let dest = Filename.concat r.Conversion.dir (Text.short_id text) in
                List.fold_left
                        (fun a f ->
                                match f.Conversion.page with None -> false || a
                                | Some page ->
                                        let dest = dest ^ f.Conversion.ext in
                                        (if is_older source dest then (File_store.file dest (page r text); true) else false)
                                        || a)
                        false cs
        | x -> Printf.eprintf "Can't convert Content-Type: %s file: %s" x text.Text.title; false

let converters types kv =
        let n = String.split_on_char ',' types in
        let t = [] in
        let t = if List.(mem "all" n || mem "htm" n) then (Html.converter kv)::t else t in
        let t = if List.(mem "all" n || mem "atom" n) then (Atom.converter "text/html")::t else t in
        let t = if List.(mem "all" n || mem "gmi" n) then (Gemini.converter)::t else t in
        let t = if List.(mem "all" n || mem "gmi-atom" n) then (Atom.converter "text/gemini")::t else t in
        t

let acc_rel source target a =
        prerr_endline source;
        Conversion.Id_map.update target
        (function Some set -> Some (Conversion.Ref_set.add source set)
                | None -> Some (Conversion.Ref_set.singleton source))
        a

let empty_rels () = Conversion.Id_map.empty, Conversion.Id_map.empty

let acc_txt_refs text refs = String_set.fold (acc_rel text.Text.id) (Text.set "references" text) refs
let acc_txt_reps text reps = String_set.fold (acc_rel text.Text.id) (Text.set "in-reply-to" text) reps
let acc_txt_rels (refs, reps) (elt, _paths) =
        acc_txt_refs elt refs, acc_txt_reps elt reps

let acc_pck_refs id refs_ls refs = String_set.fold (acc_rel id) (String_set.of_list refs_ls) refs
let acc_pck_reps id reps_ls reps = String_set.fold (acc_rel id) (String_set.of_list reps_ls) reps
let acc_pck_rels refs_reps peer =
        let path = try List.hd peer.Peers.pack.Header_pack.info.locations with Failure _ -> "" in
        try Header_pack.fold
                (fun (refs, reps) id _t _title _authors _topics refs_ls reps_ls ->
                        let id = Filename.concat path id in
                        acc_pck_refs id refs_ls refs, acc_pck_reps id reps_ls reps)
                refs_reps peer.Peers.pack
        with e -> prerr_endline "acc_pck_rels"; raise e

let directory converters noindex repo =
        let order = File_store.oldest in
        let repo =
                let references, replies =
                        File_store.fold ~dir:repo.Conversion.dir ~order acc_txt_rels (empty_rels ()) in
                let references, replies = Peers.fold acc_pck_rels (references, replies) in
                Printf.eprintf "%s %d\n" repo.Conversion.dir (Conversion.Id_map.cardinal replies);
                { repo with references; replies } in
        let acc (ts,ls,acc) ((elt,_) as r) = Topic_set.to_map ts (Text.set "topics" elt), elt::ls,
                if convert converters repo r then acc+1 else acc in
        let topics, texts, count =
                File_store.fold ~dir:repo.Conversion.dir ~order acc (Topic_set.Map.empty, [], 0) in
        let topic_roots = try List.rev @@ String_set.list_of_csv (Store.KV.find "Topics" repo.kv)
                with Not_found -> Topic_set.roots topics in
        let repo = Conversion.{ repo with topic_roots; topics; texts = List.rev texts } in
        if not noindex then
                List.iter (fun c -> match c.Conversion.indices with None -> () | Some f -> f repo) converters;
        Printf.printf "Converted: %d Indexed: %d\n" count (List.length texts)

let load_kv dir =
        let kv = File_store.of_kv_file () in
        let idx = Filename.concat dir "index.pck" in
        if not (Sys.file_exists idx) then kv else
                match Header_pack.of_string @@ File_store.to_string (idx) with
                | Error s -> prerr_endline s; kv
                | Ok { info; peers; _ } ->
                        let kv = if Store.KV.mem "Id" kv then kv else Store.KV.add "Id" info.Header_pack.id kv in
                        let kv = if Store.KV.mem "Title" kv then kv else Store.KV.add "Title" info.Header_pack.title kv in
                        let kv = if Store.KV.mem "Locations" kv then kv else Store.KV.add "Locations" (String.concat ";\n" info.Header_pack.locations) kv in
                        let kv = Store.KV.add "Peers" (String.concat ";\n" Header_pack.(to_str_list peers)) kv in
                        kv

let at_path types noindex path = match path with
        | "" -> prerr_endline "unspecified text file or directory"
        | path when Sys.file_exists path ->
                if Sys.is_directory path then (
                        let kv = load_kv path in
                        let repo = { (Conversion.empty ()) with dir = path; kv } in
                        directory (converters types kv) noindex repo
                ) else (
                        match File_store.to_text path with
                        | Error s -> prerr_endline s
                        | Ok text ->
                                let dir = "." in
                                let references, replies = File_store.(fold ~dir ~order:newest acc_txt_rels (empty_rels ())) in
                                let repo = { (Conversion.empty ()) with dir; kv = load_kv ""; references; replies } in
                                ignore @@ convert (converters types repo.kv) repo (text, [path])
                )
        | path -> Printf.eprintf "Path doesn't exist: %s" path

open Cmdliner
let term =
        let path = Arg.(value & pos 0 string "" & info [] ~docv:"path"
                ~doc:"Text file or directory to convert. If directory is provided, it must contain an index.pck (see: txt index)") in
        let types = Arg.(value & opt string "all" & info ["t"; "type"] ~docv:"output type"
                ~doc:"Convert to file type") in
        let noindex = Arg.(value & flag & info ["noindex"]
                ~doc:"Don't create indices in target format") in
        Term.(const at_path $ types $ noindex $ path),
        Term.info "convert" ~doc:"convert texts"
                ~man:[ `S "DESCRIPTION"; `P "Convert text or indexed texts within a directory to another format.
                If path is a directory must contain an index.pck. Run `txt index` first." ]