Changeset 3 in code for trunk/lib/header_pack.ml

Timestamp:

Apr 15, 2022, 1:17:01 PM (3 years ago)

Author:

fox

Message:

• Removed 'txt init'

Format

• New B32 ID

Index

• New option: txt index --print
• Move scheme to peers
• Replace peer.*.conf files with index packed locations Instead of adding a URL to peers.*.conf, run txt pull <url>

Conversion

• Rewritten converters
• txt-convert looks for a .convert.conf containing key: value lines.
• Specifiable topic-roots from .convert.conf.
• Added Topics: key, with comma seperated topics.

If set only those topics will appear in the main index and used as topic roots.
Other topics will have sub-indices generated, but won't be listed in the main index.

• HTML converter header & footer options
• HTML-index renamed to HTM-index

Internal

• Change types: uuid:Uuid -> id:string
• File_store merges identical texts
• Use peer ID for store path, store peers' texts in .local/share/texts
• Simple URN resolution for converter

Continue to next feed if parsing one fails

• Phasing-out Archive, replaced by improved packs
• Eliminate Bos, Cohttp, lwt, uri, tls, Re, Ptime, dependencies
• Lock version for Cmdliner, fix dune-project
• Optional resursive store
• Improve header_pack
• Fix recursive mkdir

File:

• trunk/lib/header_pack.ml (modified) (2 diffs)

trunk/lib/header_pack.ml

@@ -1 @@
-type info_t = { version: int; name: string; archivists: string list }
-type text_t = { id: Msgpck.t; time: Msgpck.t; title: Msgpck.t; authors: Msgpck.t }
-type t = { info: info_t; fields: string list; texts: Msgpck.t; peers: Msgpck.t }
+let version = 0
+type info_t = { version: int; id: string; title: string; people: string list; locations: string list }
+type t = { info: info_t; fields: Msgpck.t; texts: Msgpck.t; peers: Msgpck.t }
 
-let of_id id = Msgpck.Bytes (Id.to_bytes id)
-let to_id pck_id = Id.of_bytes Msgpck.(to_bytes pck_id)
+let of_id id = Msgpck.of_string id
+let to_id = Msgpck.to_string
 
 let person p = Msgpck.String (Person.to_string p)
-let persons ps = List.rev @@ Person.Set.fold (fun p a -> person p :: a) ps []
+let persons ps = Msgpck.of_list @@ List.rev @@ Person.Set.fold (fun p a -> person p :: a) ps []
+
+let str = Msgpck.of_string
+let str_list ls = Msgpck.of_list @@ List.map str ls
+let to_str_list x = List.map Msgpck.to_string (Msgpck.to_list x)
 
 let of_set field t =
         List.rev @@ String_set.fold (fun s a -> Msgpck.String s :: a) (Text.set field t) []
 
-let date = function
-        | None -> Int32.zero
-        | Some date ->
-                let days, ps = Ptime.Span.to_d_ps (Ptime.to_span date) in
-                Int32.add Int32.(mul (of_int days) 86400l) Int64.(to_int32 (div ps 1000000000000L))
+let date = function "" -> Int32.zero | date -> Int32.of_int (Date.to_secs date)
 
-let to_sec = function
-        Msgpck.Int i -> Int32.of_int i | Msgpck.Uint32 i -> i | x -> Msgpck.to_uint32 x
-
-let public_peers () =
-        Peers.fold_file (fun x a -> Msgpck.String x :: a) [] Peers.public_fname
+let to_sec = function Msgpck.Int i -> Int32.of_int i | Msgpck.Uint32 i -> i | x -> Msgpck.to_uint32 x
 
 let fields = Msgpck.(List [String "id"; String "time";  String "title"; String "authors"; String "topics"])
 let to_fields fieldpack = List.map Msgpck.to_string (Msgpck.to_list fieldpack)
 
-let to_pack a t =
+let to_info = function
+        | Msgpck.List (v::id::n::a::ls::[]) ->
+                let people = to_str_list a in
+                let locations = to_str_list ls in
+                Msgpck.({version = to_int v; id = to_string id; title = to_string n; people; locations})
+        | _ -> invalid_arg "Pack header"
+
+let of_info i = let open Msgpck in
+        List [Int i.version; String i.id; String i.title; str_list i.people; str_list i.locations]
+
+let of_text a t =
         let open Text in
         Msgpck.(List [
-                Bytes (Id.to_bytes t.uuid); of_uint32 (date (Date.listing t.date));
-                String t.title; List (persons t.authors); List (of_set "topics" t)
+                of_id t.id; of_uint32 (date (Date.listing t.date));
+                String t.title; persons t.authors; List (of_set "topics" t)
         ]) :: a
 
-let pack_filename ?(filename="index.pck") archive =
-        let dir = Store.KV.find "Export-Dir" archive.File_store.kv in (*raises Not_found*)
-        dir ^ "/" ^ filename
+let of_text_list l = Msgpck.List l
 
-let to_info = function
-        | Msgpck.List (v::n::a::[]) ->
-                let archivists = List.map Msgpck.to_string (Msgpck.to_list a) in
-                Msgpck.({version = to_int v; name = to_string n; archivists})
-        | _ -> invalid_arg "Pack header"
+let pack p = Msgpck.List [of_info p.info; p.fields; p.texts; p.peers]
+let string p = Bytes.to_string @@ Msgpck.Bytes.to_string @@ pack p
 
 let unpack = function
-        | Msgpck.List (i::f::texts::[]) ->
-                Some { info = to_info i; fields = to_fields f; texts; peers = Msgpck.List [] }
-        | Msgpck.List (i::f::texts::peers::[]) ->
-                Some { info = to_info i; fields = to_fields f; texts; peers }
-        | _ -> None
+        | Msgpck.List (i::fields::texts::[]) ->
+                Ok { info = to_info i; fields; texts; peers = Msgpck.List [] }
+        | Msgpck.List (i::fields::texts::peers::[]) ->
+                Ok { info = to_info i; fields; texts; peers }
+        | _ -> Error "format mismatch"
+
+let of_string s = unpack @@ snd @@ Msgpck.StringBuf.read s
 
 let list filename = try
@@ -61 +64 @@
 let contains text = function
         | Msgpck.List (id::_time::title::_authors::_topics::[]) ->
-                (match Id.of_bytes (Msgpck.to_bytes id) with
-                 | None -> prerr_endline ("Invalid id for " ^ Msgpck.to_string title); false
-                 | Some id -> text.Text.uuid = id)
+                (match to_id id with
+                 | "" -> prerr_endline ("Invalid id for " ^ Msgpck.to_string title); false
+                 | id -> text.Text.id = id)
         | _ -> prerr_endline ("Invalid record pattern"); false
 
-let pack archive records =
-        let header_pack = List.fold_left to_pack [] records in
-        let info = Msgpck.(List [Int 0; String archive.File_store.name; List (persons archive.archivists)]) in
-        Bytes.to_string @@ Msgpck.Bytes.to_string
-                (List [info; fields; Msgpck.List header_pack; Msgpck.List (public_peers ())])
 
-let add archive records =
-        let fname = pack_filename archive in
-        let append published (t, _f) = if List.exists (contains t) published then published else to_pack published t in
-        match list fname with Error e -> prerr_endline e | Ok published_list ->
-                let header_pack = List.fold_left append published_list records in
-                let archive = Msgpck.(List [Int 0; String archive.File_store.name;
-                        List (persons archive.archivists)]) in
-                File_store.file fname @@ Bytes.to_string
-                        @@ Msgpck.Bytes.to_string (List [archive; fields; Msgpck.List header_pack])
+(*let pack_filename ?(filename="index.pck") archive =*)
+(*      let dir = Store.KV.find "Export-Dir" archive.File_store.kv in (*raises Not_found*)*)
+(*      dir ^ "/" ^ filename*)
 
-let unpublish _archive _records = ()
+(*let add archive records =*)
+(*      let fname = pack_filename archive in*)
+(*      let append published (t, _f) = if List.exists (contains t) published then published else to_pack published t in*)
+(*      match list fname with Error e -> prerr_endline e | Ok published_list ->*)
+(*              let header_pack = List.fold_left append published_list records in*)
+(*              let archive = Msgpck.(List [*)
+(*                      Int 0; String archive.File_store.name; persons archive.people]) in*)
+(*              File_store.file fname @@ Bytes.to_string*)
+(*                      @@ Msgpck.Bytes.to_string (List [archive; fields; Msgpck.List header_pack])*)