Changeset 253 in code for trunk

Timestamp:

Apr 10, 2020, 5:22:47 PM (5 years ago)

Author:

contact

Message:

Per-entity ring buffers

Instead of having one ring buffer per network, each network has one ring
buffer per entity (channel or nick). This allows history to be more
fair: if there's a lot of activity in a channel, it won't prune activity
in other channels.

We now track history sequence numbers per client and per network in
networkHistory. The overall list of offline clients is still tracked in
network.offlineClients.

When all clients have received history, the ring buffer can be released.

In the future, we should get rid of too-old offline clients to avoid
having to maintain history for them forever. We should also add a
per-user limit on the number of ring buffers.

Location:

trunk

Files:

• downstream.go (modified) (3 diffs)
• upstream.go (modified) (4 diffs)
• user.go (modified) (3 diffs)

trunk/downstream.go

@@ -637 +637 @@
                 return err
         }
-
-        // Only send history if we're the first connected client with that name and
-        // network
-        sendHistory := true
-        dc.user.forEachDownstream(func(conn *downstreamConn) {
-                if dc.clientName == conn.clientName && dc.network == conn.network {
-                        sendHistory = false
-                }
-        })
 
         dc.SendMessage(&irc.Message{
@@ -689 +680 @@
 
         dc.forEachNetwork(func(net *network) {
-                seq, ok := net.history[dc.clientName]
-                if !sendHistory || !ok {
-                        return
-                }
-
-                consumer := net.ring.NewConsumer(seq)
+                // Only send history if we're the first connected client with that name
+                // for the network
+                if _, ok := net.offlineClients[dc.clientName]; ok {
+                        dc.sendNetworkHistory(net)
+                        delete(net.offlineClients, dc.clientName)
+                }
+        })
+
+        return nil
+}
+
+func (dc *downstreamConn) sendNetworkHistory(net *network) {
+        for target, history := range net.history {
+                seq, ok := history.offlineClients[dc.clientName]
+                if !ok {
+                        continue
+                }
+                delete(history.offlineClients, dc.clientName)
+
+                // If all clients have received history, no need to keep the
+                // ring buffer around
+                if len(history.offlineClients) == 0 {
+                        delete(net.history, target)
+                }
+
+                consumer := history.ring.NewConsumer(seq)
 
                 // TODO: this means all history is lost when trying to send it while the
@@ -726 +737 @@
                         dc.SendMessage(dc.marshalMessage(msg, uc))
                 }
-        })
-
-        return nil
+        }
 }
 

trunk/upstream.go

@@ -557 +557 @@
                                 ch.Members[newNick] = membership
                                 uc.appendLog(ch.Name, msg)
+                                uc.appendHistory(ch.Name, msg)
                         }
                 }
 
                 if !me {
-                        uc.network.ring.Produce(msg)
                         uc.forEachDownstream(func(dc *downstreamConn) {
                                 dc.SendMessage(dc.marshalMessage(msg, uc))
@@ -663 +663 @@
 
                                 uc.appendLog(ch.Name, msg)
+                                uc.appendHistory(ch.Name, msg)
                         }
                 }
 
                 if msg.Prefix.Name != uc.nick {
-                        uc.network.ring.Produce(msg)
                         uc.forEachDownstream(func(dc *downstreamConn) {
                                 dc.SendMessage(dc.marshalMessage(msg, uc))
@@ -1295 +1295 @@
 }
 
+// appendHistory appends a message to the history. entity can be empty.
+func (uc *upstreamConn) appendHistory(entity string, msg *irc.Message) {
+        // If no client is offline, no need to append the message to the buffer
+        if len(uc.network.offlineClients) == 0 {
+                return
+        }
+
+        history, ok := uc.network.history[entity]
+        if !ok {
+                history = &networkHistory{
+                        offlineClients: make(map[string]uint64),
+                        ring:           NewRing(uc.srv.RingCap),
+                }
+                uc.network.history[entity] = history
+
+                for clientName, _ := range uc.network.offlineClients {
+                        history.offlineClients[clientName] = 0
+                }
+        }
+
+        history.ring.Produce(msg)
+}
+
 // produce appends a message to the logs, adds it to the history and forwards
 // it to connected downstream connections.
@@ -1305 +1328 @@
         }
 
-        uc.network.ring.Produce(msg)
+        uc.appendHistory(target, msg)
 
         uc.forEachDownstream(func(dc *downstreamConn) {

trunk/user.go

@@ -46 +46 @@
 }
 
+type networkHistory struct {
+        offlineClients map[string]uint64 // indexed by client name
+        ring           *Ring             // can be nil if there are no offline clients
+}
+
 type network struct {
         Network
         user    *user
-        ring    *Ring
         stopped chan struct{}
 
-        conn      *upstreamConn
-        history   map[string]uint64
-        lastError error
+        conn           *upstreamConn
+        history        map[string]*networkHistory // indexed by entity
+        offlineClients map[string]struct{}        // indexed by client name
+        lastError      error
 }
 
 func newNetwork(user *user, record *Network) *network {
         return &network{
-                Network: *record,
-                user:    user,
-                ring:    NewRing(user.srv.RingCap),
-                stopped: make(chan struct{}),
-                history: make(map[string]uint64),
+                Network:        *record,
+                user:           user,
+                stopped:        make(chan struct{}),
+                history:        make(map[string]*networkHistory),
+                offlineClients: make(map[string]struct{}),
         }
 }
@@ -295 +300 @@
                         dc := e.dc
 
-                        dc.forEachNetwork(func(net *network) {
-                                seq := net.ring.Cur()
-                                net.history[dc.clientName] = seq
-                        })
-
                         for i := range u.downstreamConns {
                                 if u.downstreamConns[i] == dc {
@@ -306 +306 @@
                                 }
                         }
+
+                        // Save history if we're the last client with this name
+                        skipHistory := make(map[*network]bool)
+                        u.forEachDownstream(func(conn *downstreamConn) {
+                                if dc.clientName == conn.clientName {
+                                        skipHistory[conn.network] = true
+                                }
+                        })
+
+                        dc.forEachNetwork(func(net *network) {
+                                if skipHistory[net] || skipHistory[nil] {
+                                        return
+                                }
+
+                                net.offlineClients[dc.clientName] = struct{}{}
+                                for _, history := range net.history {
+                                        history.offlineClients[dc.clientName] = history.ring.Cur()
+                                }
+                        })
 
                         u.forEachUpstream(func(uc *upstreamConn) {