Changeset 3 in code for trunk/lib


Ignore:
Timestamp:
Apr 15, 2022, 1:17:01 PM (3 years ago)
Author:
fox
Message:
  • Removed 'txt init'

Format

  • New B32 ID

Index

  • New option: txt index --print
  • Move scheme to peers
  • Replace peer.*.conf files with index packed locations Instead of adding a URL to peers.*.conf, run txt pull <url>

Conversion

  • Rewritten converters
  • txt-convert looks for a .convert.conf containing key: value lines.
  • Specifiable topic-roots from .convert.conf.
  • Added Topics: key, with comma seperated topics.

If set only those topics will appear in the main index and used as topic roots.
Other topics will have sub-indices generated, but won't be listed in the main index.

  • HTML converter header & footer options
  • HTML-index renamed to HTM-index

Internal

  • Change types: uuid:Uuid -> id:string
  • File_store merges identical texts
  • Use peer ID for store path, store peers' texts in .local/share/texts
  • Simple URN resolution for converter

Continue to next feed if parsing one fails

  • Phasing-out Archive, replaced by improved packs
  • Eliminate Bos, Cohttp, lwt, uri, tls, Re, Ptime, dependencies
  • Lock version for Cmdliner, fix dune-project
  • Optional resursive store
  • Improve header_pack
  • Fix recursive mkdir
Location:
trunk/lib
Files:
12 edited

Legend:

Unmodified
Added
Removed

  • trunk/lib/archive.ml

    r2 r3  
    1 (*let module S = Set.Make (Text) in*)
    2 (*let module M = Map.Make (String) in*)
    3 (*let module I = Map.Make (Id) in*)
    4 (*let aggr = I.empty, M.empty, M.empty, M.empty in*)
    5 (*let fn (id, a, t, k) (n,_) =*)
    6 (* let id = I.add n.Text.uuid n id in*)
    7 (* let a =*)
    8 (*   let f e a = M.update (e.Person.name) (function Some v -> Some (S.add n v) | None -> Some (S.singleton n)) a in*)
    9 (*   Person.Set.fold f n.Text.authors a in*)
    10 (* let t =*)
    11 (*   let f e a = M.update e (function Some v -> Some (S.add n v) | None -> Some (S.singleton n)) a in*)
    12 (*   String_set.fold f (Text.set "Topics" n) t in*)
    13 (* let k =*)
    14 (*   let f e a = M.update e (function Some v -> Some (S.add n v) | None -> Some (S.singleton n)) a in*)
    15 (*   String_set.fold f (Text.set "Keywords" n) k in*)
    16 (* (id, a, t, k)*)
     1let predicate fn opt = Option.(to_list @@ map fn opt)
    172
    18 module Make (Store : Store.T) = struct
    19   include Store
    20   let predicate fn opt = Option.(to_list @@ map fn opt)
     3let authored query_string =
     4  let q = Person.Set.of_query @@ String_set.query query_string in
     5  fun n -> Person.Set.predicate q n.Text.authors
    216
    22   let authored query_string =
    23     let q = Person.Set.of_query @@ String_set.query query_string in
    24     fun n -> Person.Set.predicate q n.Text.authors
     7let keyworded query_string =
     8  let q = String_set.query query_string in
     9  fun n -> String_set.(predicate q (Text.set "Keywords" n))
    2510
    26   let keyworded query_string =
    27     let q = String_set.query query_string in
    28     fun n -> String_set.(predicate q (Text.set "Keywords" n))
    29 
    30   let topics query_string =
    31     let q = String_set.query query_string in
    32     fun n -> String_set.(predicate q (Text.set "Topics" n))
    33 end
     11let topics query_string =
     12  let q = String_set.query query_string in
     13  fun n -> String_set.(predicate q (Text.set "Topics" n))

  • trunk/lib/date.ml

    r2 r3  
    1 type t = { created: Ptime.t option; edited: Ptime.t option }
     1type t = { created: string; edited: string }
    22let compare = compare
    3 let rfc_string date = match date with Some t -> Ptime.to_rfc3339 t | None -> ""
    4 let of_string (rfc : string) = match Ptime.of_rfc3339 rfc with Ok (t,_,_) -> Some t | Error _ -> None
    5 let listing date = if Option.is_some date.edited then date.edited else date.created
    6 let pretty_date = function
    7   | Some t -> Ptime.to_date t |> fun (y, m, d) -> Printf.sprintf "%04d-%02d-%02d" y m d
    8   | None -> ""
     3let rfc_string date = date
     4let of_string (rfc : string) = rfc
     5let listing date = if date.edited <> "" then date.edited else date.created
     6let pretty_date date =
     7        try Scanf.sscanf date "%4s-%2s-%2s" (fun y m d -> Printf.sprintf "%s %s %s" y m d)
     8        with Scanf.Scan_failure s as e -> Printf.fprintf stderr "%s for %s\n" s date; raise e
     9let now () = Unix.time () |> Unix.gmtime |>
     10        (fun t -> Printf.sprintf "%4d-%02d-%02dT%02d:%02d:%02dZ"
     11         (t.tm_year+1900) (t.tm_mon+1) t.tm_mday t.tm_hour t.tm_min t.tm_sec)
     12let to_secs date =
     13        Scanf.sscanf date "%4d-%02d-%02dT%02d:%02d:%02d"
     14                (fun y mo d h mi s -> (y-1970)*31557600 + mo*2629800 + d*86400 + h*3600 + mi*60 + s)

  • trunk/lib/dune

    r2 r3  
    22 (name        logarion)
    33 (public_name logarion)
    4  (libraries ptime uuidm uri re.str bos text_parse text_parse.parsers msgpck))
     4 (libraries text_parse text_parse.parsers unix str msgpck))

  • trunk/lib/file_store.ml

    r2 r3  
    11type t = string
    2 type item_t = string
    3 type archive_t = {
    4         name: string; archivists: Person.Set.t; id: Id.t;
    5         kv: string Store.KV.t; store: t }
     2type item_t = t list
    63type record_t = Text.t * item_t
    74
    85let extension = ".txt"
     6let def_dir = try Sys.getenv "LOGARION_DIR" with Not_found -> "."
    97
    108let to_string f =
    119        let ic = open_in f in
    12         let n = in_channel_length ic in
    13         let s = Bytes.create n in
    14         really_input ic s 0 n;
     10        let s = really_input_string ic (in_channel_length ic) in
    1511        close_in ic;
    16         Bytes.to_string s
     12        s
    1713
    18 let file path content = let out = open_out path in
    19         output_string out content; close_out out
     14let fold_file_line fn init file = match open_in file with
     15        | exception (Sys_error msg) -> prerr_endline msg; init
     16        | file ->
     17                let rec read acc = match input_line file with
     18                        | "" as s | s when String.get s 0 = '#' -> read acc
     19                        | s -> read (fn s acc)
     20                        | exception End_of_file -> close_in file; acc
     21                in read init
    2022
    21 let (//) a b = a ^ "/" ^ b
     23let file path str = let o = open_out path in output_string o str; close_out o
    2224
    2325let to_text path =
    2426        if Filename.extension path = extension then
    2527                (to_string path |> Text.of_string |> Result.map_error (fun m -> path ^": "^ m))
    26         else Error "Not txt"
     28        else Error (Printf.sprintf "Not txt: %s" path)
    2729
    2830let newest (a,_pa) (b,_pb) = Text.newest a b
    2931let oldest (a,_pa) (b,_pb) = Text.oldest a b
    3032
    31 let list_iter fn {store;_} paths =
    32         let link f = match to_text (Filename.concat store f)
    33                 with Ok t -> fn store t f | Error s -> prerr_endline s in
     33let list_iter fn dir paths =
     34        let link f = match to_text (Filename.concat dir f) with
     35                | Ok t -> fn dir t f | Error s -> prerr_endline s in
    3436        List.iter link paths
    3537
    36 let iter_valid_text pred fn p =
    37         match to_text p with Error _ -> () | Ok t -> if pred t then fn (t, p)
     38module TextMap = Map.Make(Text)
    3839
    39 let fold_valid_text pred fn acc p =
    40         match to_text p with Error _ -> acc | Ok t -> if pred t then fn acc (t, p) else acc
     40type iteration_t = item_t TextMap.t
     41let new_iteration = TextMap.empty
    4142
    42 let list_fs dir =
     43(*let iter_valid_text pred fn path =*)
     44(*      match to_text path with Error _ -> () | Ok t -> if pred t then fn (t, p)*)
     45
     46let fold_valid_text pred it path =
     47        match to_text path with Error _ -> it
     48        | Ok t -> if pred t then (TextMap.update t
     49                        (function None -> Some [path] | Some ps -> Some (path::ps)) it
     50                ) else it
     51
     52(* Compare file system nodes to skip reparsing? *)
     53let list_fs ?(r=false) dir =
     54        let valid_dir f = r && String.get f 0 <> '.' && Sys.is_directory f in
     55        let expand_dir d = Array.(to_list @@ map (Filename.concat d) (Sys.readdir d)) in
    4356        let rec loop result = function
    44                 | [] -> result
    45                 | f::fs when Sys.is_directory f ->
    46                         Array.map (Filename.concat f) (Sys.readdir f)
    47                         |> Array.to_list |> List.append fs |> loop result
     57                | f::fs when valid_dir f -> expand_dir f |> List.append fs |> loop result
    4858                | f::fs -> loop (f::result) fs
    49         in loop [] [dir]
     59                | [] -> result in
     60        let dirs = if dir = "." then Array.to_list (Sys.readdir dir) else
     61                if not r then expand_dir dir else [dir] in
     62        loop [] dirs
    5063
    5164let list_take n =
    … …  
    5568        in take [] n
    5669
    57 let iter ?(predicate=fun _ -> true) ?order ?number fn {store;_} =
    58         match order with
    59         | None -> List.iter (iter_valid_text predicate fn) @@ list_fs store
    60         | Some comp ->
    61                 List.iter fn
    62                 @@ (match number with None -> (fun x -> x) | Some n -> list_take n)
    63                 @@ List.fast_sort comp
    64                 @@ List.fold_left (fold_valid_text predicate (fun a e -> List.cons e a)) []
    65                 @@ list_fs store
     70let fold_sort_take ?(predicate=fun _ -> true) ?(number=None) comp flist =
     71        (match number with None -> (fun x -> x) | Some n -> list_take n)
     72        @@ List.fast_sort comp @@ TextMap.bindings
     73        @@ List.fold_left (fold_valid_text predicate) new_iteration flist
    6674
    67 let fold ?(predicate=fun _ -> true) ?order ?number fn acc {store;_} =
    68         match order with
    69         | None -> List.fold_left (fold_valid_text predicate fn) acc @@ list_fs store
    70         | Some comp ->
    71                 List.fold_left fn acc
    72                 @@ (match number with None -> (fun x -> x) | Some n -> list_take n)
    73                 @@ List.fast_sort comp
    74                 @@ List.fold_left (fold_valid_text predicate (fun a e -> List.cons e a)) []
    75                 @@ list_fs store
     75let iter ?(r=false) ?(dir=def_dir) ?(predicate=fun _ -> true) ?order ?number fn =
     76        let flist = list_fs ~r dir in match order with
     77        | Some comp -> List.iter fn @@ fold_sort_take ~predicate ~number comp flist
     78        | None -> List.iter fn @@ TextMap.bindings @@
     79                List.fold_left (fold_valid_text predicate) new_iteration flist
    7680
    77 let with_id { store; _ } id =
     81let fold ?(r=false) ?(dir=def_dir) ?(predicate=fun _ -> true) ?order ?number fn acc =
     82        let flist = list_fs ~r dir in match order with
     83        | Some comp -> List.fold_left fn acc @@ fold_sort_take ~predicate ~number comp flist
     84        | None -> List.fold_left fn acc @@ TextMap.bindings @@
     85                List.fold_left (fold_valid_text predicate) new_iteration flist
     86
     87let with_id ?(r=false) ?(dir=def_dir) id =
    7888        let matched acc path =
    7989                match to_text path with
    8090                | Error x -> prerr_endline x; acc
    81                 | Ok text when text.Text.uuid <> id -> acc
     91                | Ok text when text.Text.id <> id -> acc
    8292                | Ok text ->
    8393                        match acc with
    … …  
    8595                        | Ok (Some prev) -> if prev = text then acc else Error [text; prev]
    8696                        | Error x -> Error (text :: x)
    87         in List.fold_left matched (Ok None) (list_fs store)
     97        in List.fold_left matched (Ok None) (list_fs ~r dir)
    8898
    89 module Directory = struct
    90         let print ?(descr="") dir result =
    91                 let () = match result with
    92                         | Ok true -> print_endline ("Created " ^ descr ^ " directory " ^ dir)
    93                         | Ok false -> print_endline ("Using existing " ^ descr ^ " directory " ^ dir)
    94                         | Error (`Msg msg) -> prerr_endline @@ "Failed making " ^ descr ^ ". " ^ msg
    95                 in
    96                 result
     99let with_dir ?(descr="") ?(perm=0o740) dir =
     100        let mkdir dir = match Unix.mkdir dir perm with
     101        | exception Unix.Unix_error (EEXIST, _, _) -> ()
     102        | exception Unix.Unix_error (code, _fn, arg) ->
     103                failwith @@ Printf.sprintf "Error %s making %s dir: %s"
     104                        (Unix.error_message code) descr arg
     105        | _ -> () in
     106        let rec mkeach path = function [] | [""] -> () | ""::t -> mkeach path t
     107                | hd::t -> let d = Filename.concat path hd in mkdir d; mkeach d t in
     108        mkeach
     109                (if Filename.is_relative dir then "" else "/")
     110                (String.split_on_char '/' dir)
    97111
    98         let directory dir = Result.bind (Fpath.of_string dir) Bos.OS.Dir.create
    99 
    100         let rec directories = function
    101                 | [] -> Ok ()
    102                 | (d, descr)::tl ->
    103                         match directory d |> print ~descr d with
    104                         | Ok _ -> directories tl
    105                         | Error _ -> Error (d, descr)
    106 end
    107 
    108 let copy ?(recursive = false) src dst =
    109         Bos.OS.Cmd.run (Bos.Cmd.(v "cp" %% (on recursive @@ v "-r") % src % dst))
     112let rec with_dirs = function [] -> () | (d, descr)::tl -> with_dir ~descr d; with_dirs tl
    110113
    111114let versioned_basename_of_title ?(version=0) repo extension (title : string) =
    112115        let basename = Text.string_alias title in
    113116        let rec next version =
    114                 let candidate = repo // basename ^ "." ^ string_of_int version ^ extension in
     117                let candidate = Filename.concat repo
     118                        (basename ^ "." ^ string_of_int version ^ extension) in
    115119                if Sys.file_exists candidate then next (succ version) else candidate
    116120        in
    117121        next version
    118122
    119 let uuid_filename repo extension text =
     123let id_filename repo extension text =
    120124        let basename = Text.alias text in
    121         let candidate = repo // String.sub (Id.to_string text.uuid) 0 6 ^ "." ^ basename ^ extension in
     125        let candidate = Filename.concat repo (text.id ^ "." ^ basename ^ extension) in
    122126        if Sys.file_exists candidate then Error "Name clash, try again" else Ok candidate
    123127
    124 let with_text {store;_} new_text =
    125         Result.bind (uuid_filename store extension new_text) @@
    126         fun path ->
    127                 try file path (Text.to_string new_text); Ok (path, new_text) with Sys_error s -> Error s
    128 
    129 let basic_config () =
    130         "Archive-Name: "
    131         ^ "\nArchive-ID: " ^ Id.(generate () |> to_string)
    132         ^ "\nArchivists: " ^ Bos.OS.Env.opt_var "USER" ~absent:""
    133         |> Bytes.of_string
    134 
    135 let init ?(dotdir=".logarion/") () =
    136         match Directory.directories [dotdir, "dotdir"] with
    137         | Error (_dir, _desc) -> ()
    138         | Ok () ->
    139                 let config_file =
    140                         open_out_gen [Open_creat; Open_excl; Open_wronly]
    141                                 0o700 (dotdir // "config") in
    142                 output_bytes config_file (basic_config ());
    143                 close_out config_file
     128let with_text ?(dir=def_dir) new_text =
     129        match id_filename dir extension new_text with
     130        | Error _ as e -> e
     131        | Ok path ->
     132                try file path (Text.to_string new_text); Ok (path, new_text)
     133                with Sys_error s -> Error s
    144134
    145135module Config = struct
    146         type t = archive_t
    147         let key_value k v a = match k with
    148                 | "Archive-Name" -> { a with name = String.trim v }
    149                 | "Archive-ID" -> { a with id = Option.get (Id.of_string (String.trim v)) }
    150                 | "Archivists" -> { a with archivists = Person.Set.of_string v }
    151                 | _ -> { a with kv = Store.KV.add k (String.trim v) a.kv }
     136        type t = string Store.KV.t
     137        let key_value k v a = Store.KV.add k (String.trim v) a
    152138end
    153139
    154 let of_path store =
     140let of_kv_file path =
    155141        let open Text_parse in
    156         let subsyntaxes = [| (module Parsers.Key_value.Make (Config) : Parser.S with type t = Config.t); (module Parsers.Key_value.Make (Config)); |] in
    157         let of_string text acc = Parser.parse subsyntaxes { text; pos = 0; right_boundary = String.length text - 1 } acc in
    158         Ok (
    159                 of_string (to_string @@ store ^ "/.logarion/config") {
    160                         name = "";
    161                         archivists = Person.Set.empty;
    162                         id = Id.nil;
    163                         kv = Store.KV.empty;
    164                         store = try Sys.getenv "LOGARION_DIR" with Not_found -> "."
    165                 }
    166         )
     142        let subsyntaxes = Parsers.Key_value.[|
     143                (module Make (Config) : Parser.S with type t = Config.t); (module Make (Config)); |] in
     144        let of_string text acc =
     145                Parser.parse subsyntaxes { text; pos = 0; right_boundary = String.length text - 1 } acc in
     146        of_string (to_string @@ path) Store.KV.empty

  • trunk/lib/header_pack.ml

    r2 r3  
    1 type info_t = { version: int; name: string; archivists: string list }
    2 type text_t = { id: Msgpck.t; time: Msgpck.t; title: Msgpck.t; authors: Msgpck.t }
    3 type t = { info: info_t; fields: string list; texts: Msgpck.t; peers: Msgpck.t }
     1let version = 0
     2type info_t = { version: int; id: string; title: string; people: string list; locations: string list }
     3type t = { info: info_t; fields: Msgpck.t; texts: Msgpck.t; peers: Msgpck.t }
    44
    5 let of_id id = Msgpck.Bytes (Id.to_bytes id)
    6 let to_id pck_id = Id.of_bytes Msgpck.(to_bytes pck_id)
     5let of_id id = Msgpck.of_string id
     6let to_id = Msgpck.to_string
    77
    88let person p = Msgpck.String (Person.to_string p)
    9 let persons ps = List.rev @@ Person.Set.fold (fun p a -> person p :: a) ps []
     9let persons ps = Msgpck.of_list @@ List.rev @@ Person.Set.fold (fun p a -> person p :: a) ps []
     10
     11let str = Msgpck.of_string
     12let str_list ls = Msgpck.of_list @@ List.map str ls
     13let to_str_list x = List.map Msgpck.to_string (Msgpck.to_list x)
    1014
    1115let of_set field t =
    1216        List.rev @@ String_set.fold (fun s a -> Msgpck.String s :: a) (Text.set field t) []
    1317
    14 let date = function
    15         | None -> Int32.zero
    16         | Some date ->
    17                 let days, ps = Ptime.Span.to_d_ps (Ptime.to_span date) in
    18                 Int32.add Int32.(mul (of_int days) 86400l) Int64.(to_int32 (div ps 1000000000000L))
     18let date = function "" -> Int32.zero | date -> Int32.of_int (Date.to_secs date)
    1919
    20 let to_sec = function
    21         Msgpck.Int i -> Int32.of_int i | Msgpck.Uint32 i -> i | x -> Msgpck.to_uint32 x
    22 
    23 let public_peers () =
    24         Peers.fold_file (fun x a -> Msgpck.String x :: a) [] Peers.public_fname
     20let to_sec = function Msgpck.Int i -> Int32.of_int i | Msgpck.Uint32 i -> i | x -> Msgpck.to_uint32 x
    2521
    2622let fields = Msgpck.(List [String "id"; String "time";  String "title"; String "authors"; String "topics"])
    2723let to_fields fieldpack = List.map Msgpck.to_string (Msgpck.to_list fieldpack)
    2824
    29 let to_pack a t =
     25let to_info = function
     26        | Msgpck.List (v::id::n::a::ls::[]) ->
     27                let people = to_str_list a in
     28                let locations = to_str_list ls in
     29                Msgpck.({version = to_int v; id = to_string id; title = to_string n; people; locations})
     30        | _ -> invalid_arg "Pack header"
     31
     32let of_info i = let open Msgpck in
     33        List [Int i.version; String i.id; String i.title; str_list i.people; str_list i.locations]
     34
     35let of_text a t =
    3036        let open Text in
    3137        Msgpck.(List [
    32                 Bytes (Id.to_bytes t.uuid); of_uint32 (date (Date.listing t.date));
    33                 String t.title; List (persons t.authors); List (of_set "topics" t)
     38                of_id t.id; of_uint32 (date (Date.listing t.date));
     39                String t.title; persons t.authors; List (of_set "topics" t)
    3440        ]) :: a
    3541
    36 let pack_filename ?(filename="index.pck") archive =
    37         let dir = Store.KV.find "Export-Dir" archive.File_store.kv in (*raises Not_found*)
    38         dir ^ "/" ^ filename
     42let of_text_list l = Msgpck.List l
    3943
    40 let to_info = function
    41         | Msgpck.List (v::n::a::[]) ->
    42                 let archivists = List.map Msgpck.to_string (Msgpck.to_list a) in
    43                 Msgpck.({version = to_int v; name = to_string n; archivists})
    44         | _ -> invalid_arg "Pack header"
     44let pack p = Msgpck.List [of_info p.info; p.fields; p.texts; p.peers]
     45let string p = Bytes.to_string @@ Msgpck.Bytes.to_string @@ pack p
    4546
    4647let unpack = function
    47         | Msgpck.List (i::f::texts::[]) ->
    48                 Some { info = to_info i; fields = to_fields f; texts; peers = Msgpck.List [] }
    49         | Msgpck.List (i::f::texts::peers::[]) ->
    50                 Some { info = to_info i; fields = to_fields f; texts; peers }
    51         | _ -> None
     48        | Msgpck.List (i::fields::texts::[]) ->
     49                Ok { info = to_info i; fields; texts; peers = Msgpck.List [] }
     50        | Msgpck.List (i::fields::texts::peers::[]) ->
     51                Ok { info = to_info i; fields; texts; peers }
     52        | _ -> Error "format mismatch"
     53
     54let of_string s = unpack @@ snd @@ Msgpck.StringBuf.read s
    5255
    5356let list filename = try
    … …  
    6164let contains text = function
    6265        | Msgpck.List (id::_time::title::_authors::_topics::[]) ->
    63                 (match Id.of_bytes (Msgpck.to_bytes id) with
    64                  | None -> prerr_endline ("Invalid id for " ^ Msgpck.to_string title); false
    65                  | Some id -> text.Text.uuid = id)
     66                (match to_id id with
     67                 | "" -> prerr_endline ("Invalid id for " ^ Msgpck.to_string title); false
     68                 | id -> text.Text.id = id)
    6669        | _ -> prerr_endline ("Invalid record pattern"); false
    6770
    68 let pack archive records =
    69         let header_pack = List.fold_left to_pack [] records in
    70         let info = Msgpck.(List [Int 0; String archive.File_store.name; List (persons archive.archivists)]) in
    71         Bytes.to_string @@ Msgpck.Bytes.to_string
    72                 (List [info; fields; Msgpck.List header_pack; Msgpck.List (public_peers ())])
    7371
    74 let add archive records =
    75         let fname = pack_filename archive in
    76         let append published (t, _f) = if List.exists (contains t) published then published else to_pack published t in
    77         match list fname with Error e -> prerr_endline e | Ok published_list ->
    78                 let header_pack = List.fold_left append published_list records in
    79                 let archive = Msgpck.(List [Int 0; String archive.File_store.name;
    80                         List (persons archive.archivists)]) in
    81                 File_store.file fname @@ Bytes.to_string
    82                         @@ Msgpck.Bytes.to_string (List [archive; fields; Msgpck.List header_pack])
     72(*let pack_filename ?(filename="index.pck") archive =*)
     73(*      let dir = Store.KV.find "Export-Dir" archive.File_store.kv in (*raises Not_found*)*)
     74(*      dir ^ "/" ^ filename*)
    8375
    84 let unpublish _archive _records = ()
     76(*let add archive records =*)
     77(*      let fname = pack_filename archive in*)
     78(*      let append published (t, _f) = if List.exists (contains t) published then published else to_pack published t in*)
     79(*      match list fname with Error e -> prerr_endline e | Ok published_list ->*)
     80(*              let header_pack = List.fold_left append published_list records in*)
     81(*              let archive = Msgpck.(List [*)
     82(*                      Int 0; String archive.File_store.name; persons archive.people]) in*)
     83(*              File_store.file fname @@ Bytes.to_string*)
     84(*                      @@ Msgpck.Bytes.to_string (List [archive; fields; Msgpck.List header_pack])*)

  • trunk/lib/id.ml

    r2 r3  
    1 let random_state = Random.State.make_self_init ()
    2 type t = Uuidm.t
    3 let compare = Uuidm.compare
    4 let to_string = Uuidm.to_string
    5 let of_string = Uuidm.of_string
    6 let to_bytes = Uuidm.to_bytes
    7 let of_bytes = Uuidm.of_bytes
    8 let generate ?(random_state=random_state) = Uuidm.v4_gen random_state
    9 let nil = Uuidm.nil
     1let random_state = Random.State.make_self_init
     2
     3(*module UUID = struct*)
     4(*type t = Uuidm.t*)
     5(*let compare = Uuidm.compare*)
     6(*let to_string = Uuidm.to_string*)
     7(*let of_string = Uuidm.of_string*)
     8(*let to_bytes = Uuidm.to_bytes*)
     9(*let of_bytes = Uuidm.of_bytes*)
     10(*let generate ?(random_state=random_state ()) = Uuidm.v4_gen random_state*)
     11(*let nil = Uuidm.nil*)
     12(*end*)
     13
     14type t = string
     15let compare = String.compare   
     16let nil = ""
     17
     18let short ?(len) id =
     19        let id_len = String.length id in
     20        let l = match len with Some l -> l | None -> if id_len = 36 then 8 else 6 in
     21        String.sub id 0 (min l id_len)
     22
     23let generate ?(len=6) ?(seed=random_state ()) () =
     24        let b32 i = char_of_int @@
     25                if i < 10 then i+48 else
     26                if i < 18 then i+87 else
     27                if i < 20 then i+88 else
     28                if i < 22 then i+89 else
     29                if i < 27 then i+90 else
     30                if i < 32 then i+91 else
     31                (invalid_arg ("id.char" ^ string_of_int i)) in
     32        let c _ = b32 (Random.State.int seed 31) in
     33        String.init len c

  • trunk/lib/peers.ml

    r2 r3  
    1 let public_fname = "peers.pub.conf"
    2 let private_fname = "peers.priv.conf"
     1let text_dir = Filename.concat (Sys.getenv "HOME") ".local/share/texts"
    32
    4 let fold_file fn init file = match open_in file with
    5         | exception (Sys_error msg) -> prerr_endline msg; init
    6         | file ->
    7                 let rec read acc = try read (fn (input_line file) acc)
    8                         with End_of_file -> close_in file; acc in
    9                 read init
     3let fold fn init = match Sys.readdir text_dir with
     4        | exception (Sys_error msg) -> prerr_endline msg
     5        | dirs ->
     6                let read_pack path =
     7                        let pack_path = Filename.(concat text_dir @@ concat path "index.pck") in
     8                        match Sys.file_exists pack_path with false -> () | true ->
     9                                match Header_pack.of_string (File_store.to_string pack_path) with
     10                                | Error s -> Printf.eprintf "%s %s\n" s pack_path
     11                                | Ok p -> ignore @@ List.fold_left fn init Header_pack.(p.info.locations)
     12                in
     13                Array.iter read_pack dirs
     14
     15let scheme url =
     16        let colon_idx = String.index_from url 0 ':' in
     17        let scheme = String.sub url 0 colon_idx in
     18(*      let remain = String.(sub url (colon_idx+1) (length url - length scheme - 1)) in*)
     19        scheme

  • trunk/lib/person.ml

    r2 r3  
    11module Person = struct
    22  type name_t = string
    3   type address_t = Uri.t
     3  type address_t = string
    44  type t = { name: name_t; addresses: address_t list }
    55  let empty = { name = ""; addresses = [] }
    66  let compare = Stdlib.compare
    7   let to_string p = List.fold_left (fun a e -> a^" <"^Uri.to_string e^">") p.name p.addresses
     7  let name_to_string p = p.name
     8  let to_string p = List.fold_left (fun a e -> Printf.sprintf "%s <%s>" a e) p.name p.addresses
    89  let of_string s = match String.trim s with "" -> empty | s ->
    9     match Re.Str.(split (regexp " *< *") s) with
     10    match Str.(split (regexp " *< *") s) with
    1011    | [] -> empty
    1112    | [n] -> let name = String.trim n in { empty with name }
    1213    | n::adds ->
    1314      let name = String.trim n in
    14       let addresses = List.map (fun f -> Uri.of_string @@ String.(sub f 0 (length f -1))) adds in
     15      let addresses = List.map (fun f -> String.(sub f 0 (length f -1))) adds in
    1516      { name; addresses }
    1617end
    … …  
    2021module Set = struct
    2122  include Set.Make(Person)
    22   let to_string ?(pre="") ?(sep=", ") s =
    23     let str = Person.to_string in
     23  let to_string ?(names_only=false) ?(pre="") ?(sep=", ") s =
     24    let str = if names_only then Person.name_to_string else Person.to_string in
    2425    let j x a = match a, x with "",_ -> str x | _,x when x = Person.empty -> a | _ -> a^sep^str x in
    2526    fold j s pre

  • trunk/lib/store.ml

    r2 r3  
    44        type t
    55        type item_t
    6         type archive_t = { name: string; archivists: Person.Set.t; id: Id.t; kv: string KV.t; store: t }
     6        type archive_t = { id: Id.t; name: string; archivists: Person.Set.t; kv: string KV.t; store: t }
    77        type record_t = Text.t * item_t
    88        val of_path: string -> (archive_t, string) result

  • trunk/lib/string_set.ml

    r2 r3  
    11include Set.Make(String)
    22
    3 let list_of_csv x = Re.Str.(split (regexp " *, *")) (String.trim x)
     3let list_of_csv x = Str.(split (regexp " *, *")) (String.trim x)
    44let of_string x = of_list (list_of_csv x)
    55let to_string ?(pre="") ?(sep=", ") s =

  • trunk/lib/text.ml

    r2 r3  
    11module String_map = Map.Make (String)
    22type t = {
    3     title: string;
    4     uuid: Id.t;
    5     authors: Person.Set.t;
    6     date: Date.t;
    7     string_map: string String_map.t;
    8     stringset_map: String_set.t String_map.t;
    9     body: string;
    10   }
     3                id: Id.t;
     4                title: string;
     5                authors: Person.Set.t;
     6                date: Date.t;
     7                string_map: string String_map.t;
     8                stringset_map: String_set.t String_map.t;
     9                body: string;
     10        }
    1111
    12 let blank ?(uuid=(Id.generate ())) () = {
    13     title = "";
    14     uuid;
    15     authors = Person.Set.empty;
    16     date = Date.({ created = None; edited = None});
    17     string_map = String_map.empty;
    18     stringset_map = String_map.empty;
    19     body = "";
    20   }
     12let blank ?(id=(Id.generate ())) () = {
     13                id;
     14                title = "";
     15                authors = Person.Set.empty;
     16                date = Date.({ created = now (); edited = ""});
     17                string_map = String_map.empty;
     18                stringset_map = String_map.empty;
     19                body = "";
     20        }
    2121
    2222let compare = Stdlib.compare
    … …  
    2929
    3030let with_kv x (k,v) =
    31   let trim = String.trim in
    32   match String.lowercase_ascii k with
    33   | "body" -> { x with body = String.trim v }
    34   | "title"-> { x with title = trim v }
    35   | "id"   -> (match Id.of_string v with Some id -> { x with uuid = id } | None -> x)
    36   | "author"
    37   | "authors" -> { x with authors = Person.Set.of_string (trim v)}
    38   | "date"       -> { x with date = Date.{ x.date with created  = Date.of_string v }}
    39   | "date-edited"-> { x with date = Date.{ x.date with edited    = Date.of_string v }}
    40   | "licences" | "topics" | "keywords" | "series" as k -> with_str_set x k v
    41   | k -> { x with string_map = String_map.add k (trim v) x.string_map }
     31        let trim = String.trim in
     32        match String.lowercase_ascii k with
     33        | "body" -> { x with body = String.trim v }
     34        | "title"-> { x with title = trim v }
     35        | "id"   -> (match v with "" -> x | s -> { x with id = s })
     36        | "author"
     37        | "authors" -> { x with authors = Person.Set.of_string (trim v)}
     38        | "date"                         -> { x with date = Date.{ x.date with created  = Date.of_string v }}
     39        | "date-edited"-> { x with date = Date.{ x.date with edited             = Date.of_string v }}
     40        | "licences" | "topics" | "keywords" | "series" as k -> with_str_set x k v
     41        | k -> { x with string_map = String_map.add k (trim v) x.string_map }
    4242
    43 let kv_of_string line = match Re.Str.(bounded_split (regexp ": *")) line 2 with
    44   | [ key; value ] -> Re.Str.(replace_first (regexp "^#\\+") "" key), value
    45   | [ key ] -> Re.Str.(replace_first (regexp "^#\\+") "" key), ""
    46   | _ -> "",""
     43let kv_of_string line = match Str.(bounded_split (regexp ": *")) line 2 with
     44        | [ key; value ] -> Str.(replace_first (regexp "^#\\+") "" key), value
     45        | [ key ] -> Str.(replace_first (regexp "^#\\+") "" key), ""
     46        | _ -> "",""
    4747
    4848let of_header front_matter =
    49   let fields = List.map kv_of_string (Re.Str.(split (regexp "\n")) front_matter) in
    50   List.fold_left with_kv (blank ~uuid:Id.nil ()) fields
     49        let fields = List.map kv_of_string (Str.(split (regexp "\n")) front_matter) in
     50        List.fold_left with_kv (blank ~id:Id.nil ()) fields
    5151
    5252let front_matter_body_split s =
    53   if Re.Str.(string_match (regexp ".*:.*")) s 0
    54   then match Re.Str.(bounded_split (regexp "^$")) s 2 with
    55       | front::body::[] -> (front, body)
    56       | _ -> ("", s)
    57   else ("", s)
     53        if Str.(string_match (regexp ".*:.*")) s 0
     54        then match Str.(bounded_split (regexp "^$")) s 2 with
     55                        | front::body::[] -> (front, body)
     56                        | _ -> ("", s)
     57        else ("", s)
    5858
    5959let of_string s =
    60   let front_matter, body = front_matter_body_split s in
    61   try
    62     let note = { (of_header front_matter) with body } in
    63     if note.uuid <> Id.nil then Ok note else Error "Missing ID header"
    64   with _ -> Error ("Failed parsing" ^ s)
     60        let front_matter, body = front_matter_body_split s in
     61        try
     62                let note = { (of_header front_matter) with body } in
     63                if note.id <> Id.nil then Ok note else Error "Missing ID header"
     64        with _ -> Error ("Failed parsing" ^ s)
    6565
    6666let to_string x =
    67   let has_len v = String.length v > 0 in
    68   let s field value = if has_len value then field ^ ": " ^ value ^ "\n" else "" in
    69   let a value = if Person.Set.is_empty value then "" else "Authors: " ^ Person.Set.to_string value ^ "\n" in
    70   let d field value = match value with Some _ -> field ^ ": " ^ Date.rfc_string value ^ "\n" | None -> "" in
    71   let rows =
    72     [ s "Title" x.title;
    73       a x.authors;
    74       d "Date" x.date.Date.created;
    75       d "Edited" x.date.Date.edited;
    76       s "Licences" (str_set "licences" x);
    77       s "Topics"   (str_set "topics" x);
    78       s "Keywords" (str_set "keywords" x);
    79       s "Series"   (str_set "series" x);
    80       s "Abstract" (str "abstract" x);
    81       s "ID" (Uuidm.to_string x.uuid);
    82       s "Alias" (str "Alias" x) ]
    83   in
    84   String.concat "" rows ^ "\n" ^ x.body
     67        let has_len v = String.length v > 0 in
     68        let s field value = if has_len value then field ^ ": " ^ value ^ "\n" else "" in
     69        let a value = if Person.Set.is_empty value then "" else "Authors: " ^ Person.Set.to_string value ^ "\n" in
     70        let d field value = match value with "" -> "" | s -> field ^ ": " ^ Date.rfc_string s ^ "\n" in
     71        let rows = [
     72                s "ID" x.id;
     73                d "Date"     x.date.Date.created;
     74                d "Edited"   x.date.Date.edited;
     75                s "Title" x.title;
     76                a x.authors;
     77                s "Licences" (str_set "licences" x);
     78                s "Topics"   (str_set "topics" x);
     79                s "Keywords" (str_set "keywords" x);
     80                s "Series"   (str_set "series" x);
     81                s "Abstract" (str "abstract" x);
     82                s "Alias"    (str "Alias" x)
     83        ] in
     84        String.concat "" rows ^ "\n" ^ x.body
    8585
    8686let string_alias t =
    87   let is_reserved = function
    88     | '!' | '*' | '\'' | '(' | ')' | ';' | ':' | '@' | '&' | '=' | '+' | '$'
    89       | ',' | '/' | '?' | '#' | '[' | ']' | ' ' | '\t' | '\x00' -> true
    90     | _ -> false
    91   in
    92   let b = Buffer.create (String.length t) in
    93   let filter char =
    94     let open Buffer in
    95     if is_reserved char then (try (if nth b (pred (length b)) <> '-' then add_char b '-') with Invalid_argument _ -> prerr_endline "reserved")
    96     else add_char b char
    97   in
    98   String.(iter filter (lowercase_ascii t));
    99   Buffer.contents b
     87        let is_reserved = function
     88                | '!' | '*' | '\'' | '(' | ')' | ';' | ':' | '@' | '&' | '=' | '+' | '$'
     89                        | ',' | '/' | '?' | '#' | '[' | ']' | ' ' | '\t' | '\x00' -> true
     90                | _ -> false
     91        in
     92        let b = Buffer.create (String.length t) in
     93        let filter char =
     94                let open Buffer in
     95                if is_reserved char then (try (if nth b (pred (length b)) <> '-' then add_char b '-') with Invalid_argument _ -> prerr_endline "reserved")
     96                else add_char b char
     97        in
     98        String.(iter filter (lowercase_ascii t));
     99        Buffer.contents b
    100100
    101101let alias t = match str "alias" t with "" -> string_alias t.title | x -> x
    102 let short_id ?(len=8) t = String.sub (Id.to_string t.uuid) 0 len
     102let short_id t = Id.short t.id

  • trunk/lib/topic_set.ml

    r2 r3  
    1 let of_string x = Re.Str.(split (regexp " *> *")) (String.trim x)
     1let of_string x = Str.(split (regexp " *> *")) (String.trim x)
    22
    33let topic x =
Note: See TracChangeset for help on using the changeset viewer.