Changeset 3 in code for trunk/lib/text.ml


Ignore:
Timestamp:
Apr 15, 2022, 1:17:01 PM (3 years ago)
Author:
fox
Message:
  • Removed 'txt init'

Format

  • New B32 ID

Index

  • New option: txt index --print
  • Move scheme to peers
  • Replace peer.*.conf files with index packed locations Instead of adding a URL to peers.*.conf, run txt pull <url>

Conversion

  • Rewritten converters
  • txt-convert looks for a .convert.conf containing key: value lines.
  • Specifiable topic-roots from .convert.conf.
  • Added Topics: key, with comma seperated topics.

If set only those topics will appear in the main index and used as topic roots.
Other topics will have sub-indices generated, but won't be listed in the main index.

  • HTML converter header & footer options
  • HTML-index renamed to HTM-index

Internal

  • Change types: uuid:Uuid -> id:string
  • File_store merges identical texts
  • Use peer ID for store path, store peers' texts in .local/share/texts
  • Simple URN resolution for converter

Continue to next feed if parsing one fails

  • Phasing-out Archive, replaced by improved packs
  • Eliminate Bos, Cohttp, lwt, uri, tls, Re, Ptime, dependencies
  • Lock version for Cmdliner, fix dune-project
  • Optional resursive store
  • Improve header_pack
  • Fix recursive mkdir
File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/lib/text.ml

    r2 r3  
    11module String_map = Map.Make (String)
    22type t = {
    3     title: string;
    4     uuid: Id.t;
    5     authors: Person.Set.t;
    6     date: Date.t;
    7     string_map: string String_map.t;
    8     stringset_map: String_set.t String_map.t;
    9     body: string;
    10   }
     3                id: Id.t;
     4                title: string;
     5                authors: Person.Set.t;
     6                date: Date.t;
     7                string_map: string String_map.t;
     8                stringset_map: String_set.t String_map.t;
     9                body: string;
     10        }
    1111
    12 let blank ?(uuid=(Id.generate ())) () = {
    13     title = "";
    14     uuid;
    15     authors = Person.Set.empty;
    16     date = Date.({ created = None; edited = None});
    17     string_map = String_map.empty;
    18     stringset_map = String_map.empty;
    19     body = "";
    20   }
     12let blank ?(id=(Id.generate ())) () = {
     13                id;
     14                title = "";
     15                authors = Person.Set.empty;
     16                date = Date.({ created = now (); edited = ""});
     17                string_map = String_map.empty;
     18                stringset_map = String_map.empty;
     19                body = "";
     20        }
    2121
    2222let compare = Stdlib.compare
    … …  
    2929
    3030let with_kv x (k,v) =
    31   let trim = String.trim in
    32   match String.lowercase_ascii k with
    33   | "body" -> { x with body = String.trim v }
    34   | "title"-> { x with title = trim v }
    35   | "id"   -> (match Id.of_string v with Some id -> { x with uuid = id } | None -> x)
    36   | "author"
    37   | "authors" -> { x with authors = Person.Set.of_string (trim v)}
    38   | "date"       -> { x with date = Date.{ x.date with created  = Date.of_string v }}
    39   | "date-edited"-> { x with date = Date.{ x.date with edited    = Date.of_string v }}
    40   | "licences" | "topics" | "keywords" | "series" as k -> with_str_set x k v
    41   | k -> { x with string_map = String_map.add k (trim v) x.string_map }
     31        let trim = String.trim in
     32        match String.lowercase_ascii k with
     33        | "body" -> { x with body = String.trim v }
     34        | "title"-> { x with title = trim v }
     35        | "id"   -> (match v with "" -> x | s -> { x with id = s })
     36        | "author"
     37        | "authors" -> { x with authors = Person.Set.of_string (trim v)}
     38        | "date"                         -> { x with date = Date.{ x.date with created  = Date.of_string v }}
     39        | "date-edited"-> { x with date = Date.{ x.date with edited             = Date.of_string v }}
     40        | "licences" | "topics" | "keywords" | "series" as k -> with_str_set x k v
     41        | k -> { x with string_map = String_map.add k (trim v) x.string_map }
    4242
    43 let kv_of_string line = match Re.Str.(bounded_split (regexp ": *")) line 2 with
    44   | [ key; value ] -> Re.Str.(replace_first (regexp "^#\\+") "" key), value
    45   | [ key ] -> Re.Str.(replace_first (regexp "^#\\+") "" key), ""
    46   | _ -> "",""
     43let kv_of_string line = match Str.(bounded_split (regexp ": *")) line 2 with
     44        | [ key; value ] -> Str.(replace_first (regexp "^#\\+") "" key), value
     45        | [ key ] -> Str.(replace_first (regexp "^#\\+") "" key), ""
     46        | _ -> "",""
    4747
    4848let of_header front_matter =
    49   let fields = List.map kv_of_string (Re.Str.(split (regexp "\n")) front_matter) in
    50   List.fold_left with_kv (blank ~uuid:Id.nil ()) fields
     49        let fields = List.map kv_of_string (Str.(split (regexp "\n")) front_matter) in
     50        List.fold_left with_kv (blank ~id:Id.nil ()) fields
    5151
    5252let front_matter_body_split s =
    53   if Re.Str.(string_match (regexp ".*:.*")) s 0
    54   then match Re.Str.(bounded_split (regexp "^$")) s 2 with
    55       | front::body::[] -> (front, body)
    56       | _ -> ("", s)
    57   else ("", s)
     53        if Str.(string_match (regexp ".*:.*")) s 0
     54        then match Str.(bounded_split (regexp "^$")) s 2 with
     55                        | front::body::[] -> (front, body)
     56                        | _ -> ("", s)
     57        else ("", s)
    5858
    5959let of_string s =
    60   let front_matter, body = front_matter_body_split s in
    61   try
    62     let note = { (of_header front_matter) with body } in
    63     if note.uuid <> Id.nil then Ok note else Error "Missing ID header"
    64   with _ -> Error ("Failed parsing" ^ s)
     60        let front_matter, body = front_matter_body_split s in
     61        try
     62                let note = { (of_header front_matter) with body } in
     63                if note.id <> Id.nil then Ok note else Error "Missing ID header"
     64        with _ -> Error ("Failed parsing" ^ s)
    6565
    6666let to_string x =
    67   let has_len v = String.length v > 0 in
    68   let s field value = if has_len value then field ^ ": " ^ value ^ "\n" else "" in
    69   let a value = if Person.Set.is_empty value then "" else "Authors: " ^ Person.Set.to_string value ^ "\n" in
    70   let d field value = match value with Some _ -> field ^ ": " ^ Date.rfc_string value ^ "\n" | None -> "" in
    71   let rows =
    72     [ s "Title" x.title;
    73       a x.authors;
    74       d "Date" x.date.Date.created;
    75       d "Edited" x.date.Date.edited;
    76       s "Licences" (str_set "licences" x);
    77       s "Topics"   (str_set "topics" x);
    78       s "Keywords" (str_set "keywords" x);
    79       s "Series"   (str_set "series" x);
    80       s "Abstract" (str "abstract" x);
    81       s "ID" (Uuidm.to_string x.uuid);
    82       s "Alias" (str "Alias" x) ]
    83   in
    84   String.concat "" rows ^ "\n" ^ x.body
     67        let has_len v = String.length v > 0 in
     68        let s field value = if has_len value then field ^ ": " ^ value ^ "\n" else "" in
     69        let a value = if Person.Set.is_empty value then "" else "Authors: " ^ Person.Set.to_string value ^ "\n" in
     70        let d field value = match value with "" -> "" | s -> field ^ ": " ^ Date.rfc_string s ^ "\n" in
     71        let rows = [
     72                s "ID" x.id;
     73                d "Date"     x.date.Date.created;
     74                d "Edited"   x.date.Date.edited;
     75                s "Title" x.title;
     76                a x.authors;
     77                s "Licences" (str_set "licences" x);
     78                s "Topics"   (str_set "topics" x);
     79                s "Keywords" (str_set "keywords" x);
     80                s "Series"   (str_set "series" x);
     81                s "Abstract" (str "abstract" x);
     82                s "Alias"    (str "Alias" x)
     83        ] in
     84        String.concat "" rows ^ "\n" ^ x.body
    8585
    8686let string_alias t =
    87   let is_reserved = function
    88     | '!' | '*' | '\'' | '(' | ')' | ';' | ':' | '@' | '&' | '=' | '+' | '$'
    89       | ',' | '/' | '?' | '#' | '[' | ']' | ' ' | '\t' | '\x00' -> true
    90     | _ -> false
    91   in
    92   let b = Buffer.create (String.length t) in
    93   let filter char =
    94     let open Buffer in
    95     if is_reserved char then (try (if nth b (pred (length b)) <> '-' then add_char b '-') with Invalid_argument _ -> prerr_endline "reserved")
    96     else add_char b char
    97   in
    98   String.(iter filter (lowercase_ascii t));
    99   Buffer.contents b
     87        let is_reserved = function
     88                | '!' | '*' | '\'' | '(' | ')' | ';' | ':' | '@' | '&' | '=' | '+' | '$'
     89                        | ',' | '/' | '?' | '#' | '[' | ']' | ' ' | '\t' | '\x00' -> true
     90                | _ -> false
     91        in
     92        let b = Buffer.create (String.length t) in
     93        let filter char =
     94                let open Buffer in
     95                if is_reserved char then (try (if nth b (pred (length b)) <> '-' then add_char b '-') with Invalid_argument _ -> prerr_endline "reserved")
     96                else add_char b char
     97        in
     98        String.(iter filter (lowercase_ascii t));
     99        Buffer.contents b
    100100
    101101let alias t = match str "alias" t with "" -> string_alias t.title | x -> x
    102 let short_id ?(len=8) t = String.sub (Id.to_string t.uuid) 0 len
     102let short_id t = Id.short t.id
Note: See TracChangeset for help on using the changeset viewer.