commit e030fe6

Use HTTP caching (etag / if-none-match) in RSS downloads.

1@@ -68,7 +68,7 @@ mastoapi.o: mastoapi.c xs.h xs_hex.h xs_openssl.h xs_json.h xs_io.h \
2  xs_time.h xs_glob.h xs_set.h xs_random.h xs_url.h xs_mime.h xs_match.h \
3  xs_unicode.h snac.h http_codes.h
4 rss.o: rss.c xs.h xs_html.h xs_regex.h xs_time.h xs_match.h xs_curl.h \
5- snac.h http_codes.h
6+ xs_openssl.h xs_json.h snac.h http_codes.h
7 sandbox.o: sandbox.c xs.h snac.h http_codes.h
8 snac.o: snac.c xs.h xs_hex.h xs_io.h xs_unicode_tbl.h xs_unicode.h \
9  xs_json.h xs_curl.h xs_openssl.h xs_socket.h xs_unix_socket.h xs_url.h \

1@@ -57,7 +57,7 @@ mastoapi.o: mastoapi.c xs.h xs_hex.h xs_openssl.h xs_json.h xs_io.h \
2  xs_time.h xs_glob.h xs_set.h xs_random.h xs_url.h xs_mime.h xs_match.h \
3  xs_unicode.h snac.h http_codes.h
4 rss.o: rss.c xs.h xs_html.h xs_regex.h xs_time.h xs_match.h xs_curl.h \
5- snac.h http_codes.h
6+ xs_openssl.h xs_json.h snac.h http_codes.h
7 sandbox.o: sandbox.c xs.h snac.h http_codes.h
8 snac.o: snac.c xs.h xs_hex.h xs_io.h xs_unicode_tbl.h xs_unicode.h \
9  xs_json.h xs_curl.h xs_openssl.h xs_socket.h xs_unix_socket.h xs_url.h \

 1@@ -7,6 +7,8 @@
 2 #include "xs_time.h"
 3 #include "xs_match.h"
 4 #include "xs_curl.h"
 5+#include "xs_openssl.h"
 6+#include "xs_json.h"
 7 
 8 #include "snac.h"
 9 
10@@ -117,12 +119,37 @@ void rss_to_timeline(snac *user, const char *url)
11     hdrs = xs_dict_set(hdrs, "accept",     "application/rss+xml");
12     hdrs = xs_dict_set(hdrs, "user-agent", USER_AGENT);
13 
14+    /* get the RSS metadata */
15+    xs *md5 = xs_md5_hex(url, strlen(url));
16+    xs *rss_md_fn = xs_fmt("%s/rss", user->basedir);
17+    mkdirx(rss_md_fn);
18+    rss_md_fn = xs_str_cat(rss_md_fn, "/", md5, ".json");
19+
20+    xs *rss_md = NULL;
21+    const char *etag = NULL;
22+
23+    FILE *f;
24+    if ((f = fopen(rss_md_fn, "r")) != NULL) {
25+        rss_md = xs_json_load(f);
26+        fclose(f);
27+
28+        etag = xs_dict_get(rss_md, "etag");
29+
30+        if (xs_is_string(etag))
31+            hdrs = xs_dict_set(hdrs, "if-none-match", etag);
32+    }
33+
34+    if (rss_md == NULL)
35+        rss_md = xs_dict_new();
36+
37     xs *payload = NULL;
38     int status;
39     int p_size;
40 
41     xs *rsp = xs_http_request("GET", url, hdrs, NULL, 0, &status, &payload, &p_size, 0);
42 
43+    snac_log(user, xs_fmt("parsing RSS %s %d", url, status));
44+
45     if (!valid_status(status) || !xs_is_string(payload))
46         return;
47 
48@@ -131,8 +158,6 @@ void rss_to_timeline(snac *user, const char *url)
49     if (!xs_is_string(ctype) || xs_str_in(ctype, "application/rss+xml") == -1)
50         return;
51 
52-    snac_log(user, xs_fmt("parsing RSS %s", url));
53-
54     /* yes, parsing is done with regexes (now I have two problems blah blah blah) */
55     xs *links = xs_regex_select(payload, "<link>[^<]+</link>");
56     const char *link;
57@@ -208,6 +233,17 @@ void rss_to_timeline(snac *user, const char *url)
58             timeline_add(user, id, obj);
59         }
60     }
61+
62+    /* update the RSS metadata */
63+    etag = xs_dict_get(rsp, "etag");
64+
65+    if (xs_is_string(etag)) {
66+        rss_md = xs_dict_set(rss_md, "etag", etag);
67+        if ((f = fopen(rss_md_fn, "w")) != NULL) {
68+            xs_json_dump(rss_md, 4, f);
69+            fclose(f);
70+        }
71+    }
72 }
73 
74