-
Notifications
You must be signed in to change notification settings - Fork 2
Expand file tree
/
Copy pathwatch.go
More file actions
113 lines (102 loc) · 2.64 KB
/
watch.go
File metadata and controls
113 lines (102 loc) · 2.64 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
package rink
import (
"context"
"time"
"github.com/luno/jettison/errors"
"github.com/luno/jettison/j"
"github.com/luno/jettison/log"
clientv3 "go.etcd.io/etcd/client/v3"
)
const (
etcdScanPage = 1000
monitorInterval = 5 * time.Minute
)
func knownLeases(ctx context.Context, cli *clientv3.Client) (map[clientv3.LeaseID]int64, error) {
leases, err := cli.Lease.Leases(ctx)
if err != nil {
return nil, errors.Wrap(err, "leases")
}
ret := make(map[clientv3.LeaseID]int64)
for _, l := range leases.Leases {
ttlResp, err := cli.Lease.TimeToLive(ctx, l.ID)
if err != nil {
return nil, err
}
ret[l.ID] = ttlResp.TTL
}
log.Debug(ctx, "found leases", j.KV("lease_count", len(ret)))
return ret, nil
}
func scanForExpiredKeys(ctx context.Context, cli *clientv3.Client,
prefix string,
leases map[clientv3.LeaseID]int64,
) (map[string]clientv3.LeaseID, error) {
var rev int64
expired := make(map[string]clientv3.LeaseID)
var scanned int
for {
resp, err := cli.Get(ctx, prefix,
clientv3.WithMinCreateRev(rev),
clientv3.WithPrefix(),
clientv3.WithLimit(etcdScanPage),
clientv3.WithSort(clientv3.SortByCreateRevision, clientv3.SortAscend),
)
if err != nil {
return nil, errors.Wrap(err, "get")
}
scanned += len(resp.Kvs)
for _, kv := range resp.Kvs {
rev = kv.CreateRevision + 1
keyTTL := leases[clientv3.LeaseID(kv.Lease)]
if keyTTL <= 0 {
expired[string(kv.Key)] = clientv3.LeaseID(kv.Lease)
}
}
if !resp.More {
break
}
}
log.Debug(ctx, "scanned keys", j.KV("count", scanned))
return expired, nil
}
type keyHistory struct {
lastExpired map[string]clientv3.LeaseID
}
func (h *keyHistory) checkForExpiredKeys(ctx context.Context, cli *clientv3.Client, prefix string) error {
leases, err := knownLeases(ctx, cli)
if err != nil {
return err
}
expired, err := scanForExpiredKeys(ctx, cli, prefix, leases)
if err != nil {
return err
}
// Check if keys were expired last time (with the same lease)
// to avoid issues with temporary consistency
for key, lease := range expired {
if lease == h.lastExpired[key] {
err := errors.New("found an expired key", j.MKV{
"key": key, "lease_id": lease,
})
log.Error(ctx, err)
}
}
h.lastExpired = expired
return nil
}
func watchForExpiredKeys(ctx context.Context, cli *clientv3.Client, prefix string) error {
ti := time.NewTicker(monitorInterval)
defer ti.Stop()
var history keyHistory
for {
select {
case <-ti.C:
err := history.checkForExpiredKeys(ctx, cli, prefix)
if err != nil && !errors.Is(err, context.Canceled) {
log.Error(ctx, errors.Wrap(err, "checking for expired keys"))
}
case <-ctx.Done():
return ctx.Err()
}
}
}