Changeset 7 in code

Timestamp:

Jun 18, 2022, 8:45:45 PM (3 years ago)

Author:

fox

Message:

Implement author & topic filter for txt pull

Location:

trunk

Files:

• cli/pull.ml (modified) (3 diffs)
• lib/header_pack.ml (modified) (1 diff)

trunk/cli/pull.ml

@@ -50 +50 @@
                 List.fold_left (fun a x -> Printf.sprintf "%s %s" a (Msgpck.to_string x)) "peers: " ps
 
-let parse_index _is_selected fn url dir p =
-        let open Logarion.Header_pack in
-        match Msgpck.to_list p.texts with
-        | [] -> Printf.printf "%s => %s, has empty index\n" p.info.title dir; false
-        | texts ->
-                let numof_texts = string_of_int @@ List.length texts in
-                let text_num_len = String.length numof_texts in
-                Printf.printf "%*d/%s %s => %s\r" text_num_len 0 numof_texts p.info.title dir;
-                let of_pck i x =
-                        Printf.printf "\r%*d/%s %!" text_num_len (i+1) numof_texts;
-                        match x with
-                        | Msgpck.List (id::time::title::_authors::_topics) ->
-                                (match Logarion.Header_pack.to_id id with
-                                | "" -> Printf.eprintf "Invalid id for%s " (Msgpck.to_string title)
-                                | id ->
-                                        let t = match time with Msgpck.Int i -> Int32.of_int i | Msgpck.Uint32 i -> i | x -> Msgpck.to_uint32 x in
-                                        if newer t id dir then fn url dir id)
-                        | _ -> prerr_endline ("Invalid record structure") in
-                List.iteri of_pck texts;
-                print_newline ();
-                true
+type filter_t = { authors: Logarion.Person.Set.t; topics: Logarion.String_set.t }
+
+let print_pull_start width total title dir =
+        Printf.printf "%*d/%s %s => %s %!" width 0 total title dir
+
+let print_pull width total i =
+        Printf.printf "\r%*d/%s %!" width (i+1) total
+
+let printers total title dir =
+        let width = String.length total in
+        print_pull_start width total title dir;
+        print_pull width total
 
 let fname dir text = Filename.concat dir (Logarion.Text.short_id text ^ ".txt")
+
 let pull_text url dir id =
         let u = Filename.concat url ((Logarion.Id.short id) ^ ".txt") in
         match curl_pull u with
         | Error msg -> Printf.eprintf "Failed getting %s: %s" u msg
-        | Ok txt ->
-                let txt = Buffer.contents txt in
+        | Ok txt -> let txt = Buffer.contents txt in
                 match Logarion.Text.of_string txt with
                 | Error s -> prerr_endline s
@@ -85 +76 @@
                         output_string file txt; close_out file
 
-let pull_index url _authors _topics =
+let per_text url dir filter print i id time title authors topics = match id with
+        | "" -> Printf.eprintf "\nInvalid id for %s\n" title
+        | id -> let open Logarion in
+                print i;
+                if newer time id dir
+                && (String_set.empty = filter.topics
+                        || String_set.exists (fun t -> List.mem t topics) filter.topics)
+                && (Person.Set.empty = filter.authors
+                        || Person.Set.exists (fun t -> List.mem (Person.to_string t) authors) filter.authors)
+                then pull_text url dir id
+
+let pull_index url authors_opt topics_opt =
         let index_url = url ^ "/index.pck" in
         match curl_pull index_url with
@@ -95 +97 @@
                         let dir = Filename.concat Logarion.Peers.text_dir pk.info.id in
                         Logarion.File_store.with_dir dir;
-                        let file = open_out_gen [Open_creat; Open_trunc; Open_wronly] 0o640 (Filename.concat dir "index.pck") in
+                        let file = open_out_gen [Open_creat; Open_trunc; Open_wronly] 0o640
+                                (Filename.concat dir "index.pck") in
                         output_string file ( Logarion.Header_pack.string {
                                         pk with info = { pk.info with locations = url::pk.info.locations }});
                         close_out file;
-(*                      let predicates = A.predicate A.authored authors_opt*)
-(*                              @ A.predicate A.topics topics_opt in*)
-                        let is_selected text = List.fold_left (fun a e -> a && e text) true [](*predicates*) in
-                        try parse_index is_selected pull_text url dir pk with
-                        Invalid_argument msg -> Printf.eprintf "Failed to parse: %s\n%!" msg; false
+                        let filter = let open Logarion in {
+                                authors = (match authors_opt with Some s -> Person.Set.of_string s | None -> Person.Set.empty);
+                                topics =( match topics_opt with Some s -> String_set.of_string s | None -> String_set.empty);
+                        } in
+                        let print = printers (string_of_int @@ Logarion.Header_pack.numof_texts pk) pk.info.title dir in
+                        try Logarion.Header_pack.iteri (per_text url dir filter print) pk; print_newline (); true
+                        with Invalid_argument msg -> Printf.eprintf "\nFailed to parse %s: %s\n%!" url msg; false
 
 let pull_list auths topics =

trunk/lib/header_pack.ml

@@ -69 +69 @@
         | _ -> prerr_endline ("Invalid record pattern"); false
 
+let numof_texts pack = List.length (Msgpck.to_list pack.texts)
+
+let iteri fn pack =
+        let of_pck i = function Msgpck.List (id::time::title::authors::topics::[]) ->
+                let t = match time with Msgpck.Int i -> Int32.of_int i | Msgpck.Uint32 i -> i
+                        | x -> Msgpck.to_uint32 x in
+                let id = to_id id in
+                let title = Msgpck.to_string title in
+                let topics = to_str_list topics in
+                let authors = to_str_list authors in
+                fn i id t title authors topics
+        | _  -> prerr_endline ("\n\nInvalid record structure\n\n")
+        in List.iteri of_pck (Msgpck.to_list pack.texts);
 
 (*let pack_filename ?(filename="index.pck") archive =*)