commit 08f9923

New function instance_failure().

 1@@ -2261,6 +2261,9 @@ int process_input_message(snac *snac, const xs_dict *msg, const xs_dict *req)
 2         return -1;
 3     }
 4 
 5+    /* this instance is alive */
 6+    instance_failure(actor, 2);
 7+
 8     /* question votes may not have a type */
 9     if (xs_is_null(type))
10         type = "Note";
11@@ -3152,6 +3155,11 @@ void process_queue_item(xs_dict *q_item)
12             return;
13         }
14 
15+        if (instance_failure(inbox, 0)) {
16+            srv_debug(1, xs_fmt("too many failures for instance %s", inbox));
17+            return;
18+        }
19+
20         /* deliver (if previous error status was a timeout, try now longer) */
21         if (p_status == 599)
22             timeout = xs_number_get(xs_dict_get_def(srv_config, "queue_timeout_2", "8"));
23@@ -3163,6 +3171,9 @@ void process_queue_item(xs_dict *q_item)
24 
25         status = send_to_inbox_raw(keyid, seckey, inbox, msg, &payload, &p_size, timeout);
26 
27+        /* register or clear a value for this instance */
28+        instance_failure(inbox, valid_status(status) ? 2 : 1);
29+
30         if (payload) {
31             if (p_size > 64) {
32                 /* trim the message */

 1@@ -112,6 +112,9 @@ int srv_open(const char *basedir, int auto_upgrade)
 2     xs *tmpdir = xs_fmt("%s/tmp", srv_basedir);
 3     mkdirx(tmpdir);
 4 
 5+    xs *faildir = xs_fmt("%s/failure", srv_basedir);
 6+    mkdirx(faildir);
 7+
 8 #ifdef __APPLE__
 9 /* Apple uses st_atimespec instead of st_atim etc */
10 #define st_atim st_atimespec
11@@ -3039,6 +3042,56 @@ xs_list *content_search(snac *user, const char *regex,
12 }
13 
14 
15+int instance_failure(const char *url, int op)
16+/* do some checks and accounting on instance failures */
17+{
18+    int ret = 0;
19+    xs *l = xs_split(url, "/");
20+    const char *hostname = xs_list_get(l, 2);
21+    double mt;
22+
23+    if (!xs_is_string(hostname))
24+        return 0;
25+
26+    xs *md5 = xs_md5_hex(hostname, strlen(hostname));
27+    xs *fn = xs_fmt("%s/%s", srv_basedir, md5);
28+
29+    switch (op) {
30+    case 0: /** check **/
31+        if ((mt = mtime(fn)) != 0.0) {
32+            /* grace time */
33+            double seconds_failing = 30 * (24 * 60 * 60);
34+
35+            if ((double)time(NULL) - mt > seconds_failing)
36+                ret = -1;
37+        }
38+
39+        break;
40+
41+    case 1: /** register a failure **/
42+        if (mtime(fn) == 0.0) {
43+            FILE *f;
44+
45+            /* only create once, as the date will be used */
46+            if ((f = fopen(fn, "w")) != NULL) {
47+                fprintf(f, "%s\n", hostname);
48+                fclose(f);
49+            }
50+        }
51+
52+        break;
53+
54+    case 2: /** clear a failure **/
55+        /* called whenever a message comes from this instance */
56+        unlink(fn);
57+
58+        break;
59+    }
60+
61+    return ret;
62+}
63+
64+
65 /** notifications **/
66 
67 xs_str *notify_check_time(snac *snac, int reset)

1@@ -278,6 +278,8 @@ int content_match(const char *file, const xs_dict *msg);
2 xs_list *content_search(snac *user, const char *regex,
3             int priv, int skip, int show, int max_secs, int *timeout);
4 
5+int instance_failure(const char *url, int op);
6+
7 void enqueue_input(snac *snac, const xs_dict *msg, const xs_dict *req, int retries);
8 void enqueue_shared_input(const xs_dict *msg, const xs_dict *req, int retries);
9 void enqueue_output_raw(const char *keyid, const char *seckey,