commit ee87557
Merge pull request 'Improving Mastodon-api experience and Moshidon support' (#489) from draga79/snac2:master into master Reviewed-on: https://codeberg.org/grunfink/snac2/pulls/489
3 files changed,
+430,
-15
+77,
-1
1@@ -118,6 +118,75 @@ int activitypub_request(snac *user, const char *url, xs_dict **data)
2 }
3
4
5+static xs_dict *actor_get_collections(snac *user, xs_dict *actor, int throttle)
6+/* fetches follower/following/statuses counts from an actor's collections and adds them to the actor object */
7+{
8+ /* only fetch if counts are not already present in the actor object */
9+ const xs_number *existing_followers = xs_dict_get(actor, "followers_count");
10+ const xs_number *existing_following = xs_dict_get(actor, "following_count");
11+ const xs_number *existing_statuses = xs_dict_get(actor, "statuses_count");
12+
13+ /* skip if we already have all counts */
14+ if (xs_type(existing_followers) == XSTYPE_NUMBER &&
15+ xs_type(existing_following) == XSTYPE_NUMBER &&
16+ xs_type(existing_statuses) == XSTYPE_NUMBER) {
17+ return actor;
18+ }
19+
20+ /* CRITICAL: duplicate URLs BEFORE any xs_dict_set calls, as xs_dict_set can reallocate the dict */
21+ xs *followers_url = xs_dup(xs_dict_get(actor, "followers"));
22+ xs *following_url = xs_dup(xs_dict_get(actor, "following"));
23+ xs *outbox_url = xs_dup(xs_dict_get(actor, "outbox"));
24+
25+ /* only fetch followers count if not already present */
26+ if (xs_type(existing_followers) != XSTYPE_NUMBER && !xs_is_null(followers_url)) {
27+ xs *followers_coll = NULL;
28+ if (valid_status(activitypub_request(user, followers_url, &followers_coll))) {
29+ const xs_number *total = xs_dict_get(followers_coll, "totalItems");
30+ if (xs_type(total) == XSTYPE_NUMBER) {
31+ xs *total_copy = xs_dup(total);
32+ actor = xs_dict_set(actor, "followers_count", total_copy);
33+ }
34+ }
35+ /* throttle to prevent resource exhaustion on low-power devices */
36+ if (throttle)
37+ usleep(100000); /* 100ms delay between requests */
38+ }
39+
40+ /* only fetch following count if not already present */
41+ if (xs_type(existing_following) != XSTYPE_NUMBER && !xs_is_null(following_url)) {
42+ xs *following_coll = NULL;
43+ if (valid_status(activitypub_request(user, following_url, &following_coll))) {
44+ const xs_number *total = xs_dict_get(following_coll, "totalItems");
45+ if (xs_type(total) == XSTYPE_NUMBER) {
46+ xs *total_copy = xs_dup(total);
47+ actor = xs_dict_set(actor, "following_count", total_copy);
48+ }
49+ }
50+ /* throttle to prevent resource exhaustion on low-power devices */
51+ if (throttle)
52+ usleep(100000); /* 100ms delay between requests */
53+ }
54+
55+ /* only fetch statuses count if not already present */
56+ if (xs_type(existing_statuses) != XSTYPE_NUMBER && !xs_is_null(outbox_url)) {
57+ xs *outbox_coll = NULL;
58+ if (valid_status(activitypub_request(user, outbox_url, &outbox_coll))) {
59+ const xs_number *total = xs_dict_get(outbox_coll, "totalItems");
60+ if (xs_type(total) == XSTYPE_NUMBER) {
61+ xs *total_copy = xs_dup(total);
62+ actor = xs_dict_set(actor, "statuses_count", total_copy);
63+ }
64+ }
65+ /* throttle to prevent resource exhaustion on low-power devices */
66+ if (throttle)
67+ usleep(100000); /* 100ms delay between requests */
68+ }
69+
70+ return actor;
71+}
72+
73+
74 int actor_request(snac *user, const char *actor, xs_dict **data)
75 /* request an actor */
76 {
77@@ -135,6 +204,9 @@ int actor_request(snac *user, const char *actor, xs_dict **data)
78 status = activitypub_request(user, actor, &payload);
79
80 if (valid_status(status)) {
81+ /* fetch collection counts when initially fetching an actor (no throttle) */
82+ payload = actor_get_collections(user, payload, 0);
83+
84 /* renew data */
85 status = actor_add(actor, payload);
86
87@@ -3118,8 +3190,12 @@ void process_user_queue_item(snac *user, xs_dict *q_item)
88 xs *actor_o = NULL;
89 int status;
90
91- if (valid_status((status = activitypub_request(user, actor, &actor_o))))
92+ if (valid_status((status = activitypub_request(user, actor, &actor_o)))) {
93+ /* refresh collection counts with throttling to prevent resource exhaustion */
94+ actor_o = actor_get_collections(user, actor_o, 1);
95+
96 actor_add(actor, actor_o);
97+ }
98 else {
99 if (status == HTTP_STATUS_GONE) {
100 actor_failure(actor, 1);
+325,
-14
1@@ -533,6 +533,117 @@ xs_str *mastoapi_id(const xs_dict *msg)
2 #define MID_TO_MD5(id) (id + 10)
3
4
5+static xs_val *get_count_from_actor(const xs_dict *actor, const char *field_names[], const char *collection_url_field)
6+/* helper to extract count from actor dict using various field name variations or from cached collection */
7+{
8+ xs_val *count = NULL;
9+
10+ /* try direct field name variations first */
11+ for (int i = 0; field_names[i] && xs_type(count) != XSTYPE_NUMBER; i++) {
12+ const xs_number *val = xs_dict_get(actor, field_names[i]);
13+ if (xs_type(val) == XSTYPE_NUMBER)
14+ count = xs_dup(val);
15+ }
16+
17+ /* if not found directly, try to get from cached collection object */
18+ if (xs_type(count) != XSTYPE_NUMBER) {
19+ const char *url = xs_dict_get(actor, collection_url_field);
20+ if (!xs_is_null(url)) {
21+ xs *coll = NULL;
22+ if (valid_status(object_get(url, &coll))) {
23+ const xs_number *total = xs_dict_get(coll, "totalItems");
24+ if (xs_type(total) == XSTYPE_NUMBER)
25+ count = xs_dup(total);
26+ }
27+ }
28+ }
29+
30+ return count;
31+}
32+
33+
34+static const xs_list *get_collection_items(snac *snac, const char *collection_url,
35+ xs_dict **out_collection, xs_dict **out_page)
36+/* fetches items from an ActivityPub collection (outbox, etc) with minimal HTTP requests */
37+{
38+ const xs_list *items = NULL;
39+ xs_dict *collection = NULL;
40+
41+ if (valid_status(activitypub_request(snac, collection_url, &collection))) {
42+ /* check if items are directly embedded */
43+ items = xs_dict_get(collection, "orderedItems");
44+ if (xs_is_null(items))
45+ items = xs_dict_get(collection, "items");
46+
47+ if (!xs_is_null(items)) {
48+ /* items found in main collection - transfer ownership to keep items valid */
49+ if (out_collection)
50+ *out_collection = collection;
51+ return items;
52+ }
53+
54+ /* if no items, try fetching first page (only 1 extra request) */
55+ const char *first_url = xs_dict_get(collection, "first");
56+ if (!xs_is_null(first_url)) {
57+ xs_dict *first_page = NULL;
58+ if (valid_status(activitypub_request(snac, first_url, &first_page))) {
59+ items = xs_dict_get(first_page, "orderedItems");
60+ if (xs_is_null(items))
61+ items = xs_dict_get(first_page, "items");
62+
63+ if (!xs_is_null(items)) {
64+ /* items found in first page - transfer ownership to keep items valid */
65+ if (out_page)
66+ *out_page = first_page;
67+ xs_free(collection);
68+ return items;
69+ }
70+ xs_free(first_page);
71+ }
72+ }
73+ xs_free(collection);
74+ }
75+
76+ return NULL;
77+}
78+
79+
80+static const xs_dict *extract_post_from_item(const xs_val *item)
81+/* extracts the post object from an outbox item, handling Create/Announce wrappers */
82+{
83+ if (xs_type(item) != XSTYPE_DICT)
84+ return NULL;
85+
86+ const char *item_type = xs_dict_get(item, "type");
87+ const xs_dict *post = item;
88+
89+ /* if it's an activity, try to get embedded object */
90+ if (!xs_is_null(item_type) &&
91+ (strcmp(item_type, "Create") == 0 || strcmp(item_type, "Announce") == 0)) {
92+ const xs_val *obj = xs_dict_get(item, "object");
93+
94+ /* only use embedded objects, skip URL references (would need HTTP fetch) */
95+ if (!xs_is_null(obj) && xs_type(obj) == XSTYPE_DICT)
96+ post = obj;
97+ else
98+ return NULL;
99+ }
100+
101+ return post;
102+}
103+
104+
105+static int is_valid_post_type(const char *post_type)
106+/* checks if a type is a valid post type for timeline display */
107+{
108+ return !xs_is_null(post_type) &&
109+ (strcmp(post_type, "Note") == 0 ||
110+ strcmp(post_type, "Article") == 0 ||
111+ strcmp(post_type, "Question") == 0 ||
112+ strcmp(post_type, "Page") == 0);
113+}
114+
115+
116 xs_dict *mastoapi_account(snac *logged, const xs_dict *actor)
117 /* converts an ActivityPub actor to a Mastodon account */
118 {
119@@ -661,9 +772,22 @@ xs_dict *mastoapi_account(snac *logged, const xs_dict *actor)
120 }
121
122 acct = xs_dict_append(acct, "locked", xs_stock(XSTYPE_FALSE));
123- acct = xs_dict_append(acct, "followers_count", xs_stock(0));
124- acct = xs_dict_append(acct, "following_count", xs_stock(0));
125- acct = xs_dict_append(acct, "statuses_count", xs_stock(0));
126+
127+ /* try to get counts from actor object if available (some servers include these) */
128+ const char *fcount_fields[] = { "followersCount", "followers_count", NULL };
129+ const char *gcount_fields[] = { "followingCount", "following_count", NULL };
130+ const char *scount_fields[] = { "statusesCount", "statuses_count", "totalItems", NULL };
131+
132+ xs *followers_count = get_count_from_actor(actor, fcount_fields, "followers");
133+ xs *following_count = get_count_from_actor(actor, gcount_fields, "following");
134+ xs *statuses_count = get_count_from_actor(actor, scount_fields, "outbox");
135+
136+ acct = xs_dict_append(acct, "followers_count",
137+ xs_type(followers_count) == XSTYPE_NUMBER ? followers_count : xs_stock(0));
138+ acct = xs_dict_append(acct, "following_count",
139+ xs_type(following_count) == XSTYPE_NUMBER ? following_count : xs_stock(0));
140+ acct = xs_dict_append(acct, "statuses_count",
141+ xs_type(statuses_count) == XSTYPE_NUMBER ? statuses_count : xs_stock(0));
142
143 xs *fields = xs_list_new();
144 p = xs_dict_get(actor, "attachment");
145@@ -1679,6 +1803,11 @@ int mastoapi_get_handler(const xs_dict *req, const char *q_path,
146 xs *out = NULL;
147 xs *actor = NULL;
148
149+ if (logged_in && strcmp(uid, "familiar_followers") == 0) { /** **/
150+ /* familiar followers endpoint - return empty array */
151+ out = xs_list_new();
152+ }
153+ else
154 if (logged_in && strcmp(uid, "search") == 0) { /** **/
155 /* search for accounts starting with q */
156 const char *aq = xs_dict_get(args, "q");
157@@ -1764,26 +1893,59 @@ int mastoapi_get_handler(const xs_dict *req, const char *q_path,
158 else
159 if (strcmp(opt, "statuses") == 0) { /** **/
160 /* the public list of posts of a user */
161+ const char *limit_s = xs_dict_get(args, "limit");
162+ const char *o_max_id = xs_dict_get(args, "max_id");
163+ int limit = limit_s ? atoi(limit_s) : 20;
164+ xs *max_id = o_max_id ? xs_tolower_i(xs_dup(o_max_id)) : NULL;
165+
166+ srv_debug(1, xs_fmt("account statuses: max_id=%s limit=%d", max_id ? max_id : "(null)", limit));
167+
168 xs *timeline = timeline_simple_list(&snac2, "public", 0, 256, NULL);
169 xs_list *p = timeline;
170 const xs_str *v;
171+ xs_set seen;
172+ int cnt = 0;
173+ int skip_until_max = max_id != NULL;
174
175 out = xs_list_new();
176+ xs_set_init(&seen);
177
178- while (xs_list_iter(&p, &v)) {
179+ while (xs_list_iter(&p, &v) && cnt < limit) {
180 xs *msg = NULL;
181
182 if (valid_status(timeline_get_by_md5(&snac2, v, &msg))) {
183+ const char *msg_id = xs_dict_get(msg, "id");
184+
185 /* add only posts by the author */
186- if (strcmp(xs_dict_get(msg, "type"), "Note") == 0 &&
187+ if (!xs_is_null(msg_id) &&
188+ strcmp(xs_dict_get(msg, "type"), "Note") == 0 &&
189 xs_startswith(xs_dict_get(msg, "id"), snac2.actor) && is_msg_public(msg)) {
190- xs *st = mastoapi_status(&snac2, msg);
191
192- if (st)
193- out = xs_list_append(out, st);
194+ /* if max_id is set, skip entries until we find it */
195+ if (skip_until_max) {
196+ xs *mid = mastoapi_id(msg);
197+ if (strcmp(mid, max_id) == 0) {
198+ skip_until_max = 0;
199+ srv_debug(2, xs_fmt("account statuses: found max_id, starting from next post"));
200+ }
201+ continue;
202+ }
203+
204+ /* deduplicate by message id */
205+ if (xs_set_add(&seen, msg_id) == 1) {
206+ xs *st = mastoapi_status(&snac2, msg);
207+
208+ if (st) {
209+ out = xs_list_append(out, st);
210+ cnt++;
211+ }
212+ }
213 }
214 }
215 }
216+
217+ srv_debug(1, xs_fmt("account statuses: returning %d posts (requested %d)", cnt, limit));
218+ xs_set_free(&seen);
219 }
220 else
221 if (strcmp(opt, "featured_tags") == 0) {
222@@ -1815,6 +1977,11 @@ int mastoapi_get_handler(const xs_dict *req, const char *q_path,
223 if (strcmp(opt, "lists") == 0) {
224 out = mastoapi_account_lists(&snac1, uid);
225 }
226+ else
227+ if (strcmp(opt, "familiar_followers") == 0) {
228+ /* familiar followers - not implemented, return empty array */
229+ out = xs_list_new();
230+ }
231
232 user_free(&snac2);
233 }
234@@ -1827,8 +1994,95 @@ int mastoapi_get_handler(const xs_dict *req, const char *q_path,
235 }
236 else
237 if (strcmp(opt, "statuses") == 0) {
238- /* we don't serve statuses of others; return the empty list */
239+ /* fetch statuses from remote outbox */
240 out = xs_list_new();
241+ const char *outbox_url = xs_dict_get(actor, "outbox");
242+
243+ if (!xs_is_null(outbox_url)) {
244+ /* extract query parameters */
245+ const char *limit_s = xs_dict_get(args, "limit");
246+ const char *exclude_replies_s = xs_dict_get(args, "exclude_replies");
247+ const char *o_max_id = xs_dict_get(args, "max_id");
248+
249+ int limit = 20;
250+ if (!xs_is_null(limit_s))
251+ limit = atoi(limit_s);
252+ if (limit == 0 || limit > 40)
253+ limit = 20;
254+
255+ int exclude_replies = !xs_is_null(exclude_replies_s) &&
256+ strcmp(exclude_replies_s, "true") == 0;
257+
258+ xs *max_id = o_max_id ? xs_tolower_i(xs_dup(o_max_id)) : NULL;
259+ int skip_until_max = max_id != NULL;
260+
261+ srv_debug(1, xs_fmt("remote account statuses: fetching from %s (max_id=%s limit=%d)",
262+ outbox_url, max_id ? max_id : "(null)", limit));
263+
264+ /* fetch first page only - safer for memory on large instances */
265+ xs *outbox_collection = NULL;
266+ xs *first_page = NULL;
267+ const xs_list *items = get_collection_items(&snac1, outbox_url,
268+ &outbox_collection, &first_page);
269+
270+ int count = 0;
271+ int processed = 0;
272+
273+ if (!xs_is_null(items) && xs_type(items) == XSTYPE_LIST) {
274+ int total_items = xs_list_len(items);
275+ srv_debug(1, xs_fmt("remote account statuses: got %d items from outbox", total_items));
276+
277+ const xs_val *item;
278+
279+ xs_list_foreach(items, item) {
280+ processed++;
281+
282+ if (count >= limit)
283+ break;
284+
285+ const xs_dict *post = extract_post_from_item(item);
286+ if (!post)
287+ continue;
288+
289+ const char *post_type = xs_dict_get(post, "type");
290+ const char *in_reply_to = xs_dict_get(post, "inReplyTo");
291+
292+ /* apply filters */
293+ if (exclude_replies && !xs_is_null(in_reply_to))
294+ continue;
295+
296+ if (is_valid_post_type(post_type)) {
297+ /* store object locally so mastoapi_id() can generate valid IDs */
298+ const char *post_id = xs_dict_get(post, "id");
299+ if (!xs_is_null(post_id))
300+ object_add(post_id, post);
301+
302+ /* handle pagination with max_id */
303+ if (skip_until_max) {
304+ xs *mid = mastoapi_id(post);
305+ if (!xs_is_null(mid) && strcmp(mid, max_id) == 0) {
306+ skip_until_max = 0;
307+ srv_debug(2, xs_fmt("remote account statuses: found max_id at position %d", processed));
308+ }
309+ continue;
310+ }
311+
312+ /* pass logged-in user context to enable media proxying if configured */
313+ xs *st = mastoapi_status(&snac1, post);
314+ if (st) {
315+ out = xs_list_append(out, st);
316+ count++;
317+ }
318+ }
319+ }
320+ }
321+ else {
322+ srv_debug(1, xs_fmt("remote account statuses: no items found in outbox"));
323+ }
324+
325+ srv_debug(1, xs_fmt("remote account statuses: processed %d items, returning %d posts (requested %d)",
326+ processed, count, limit));
327+ }
328 }
329 else
330 if (strcmp(opt, "featured_tags") == 0) {
331@@ -1840,6 +2094,11 @@ int mastoapi_get_handler(const xs_dict *req, const char *q_path,
332 if (strcmp(opt, "lists") == 0) {
333 out = mastoapi_account_lists(&snac1, uid);
334 }
335+ else
336+ if (strcmp(opt, "familiar_followers") == 0) {
337+ /* familiar followers - not implemented, return empty array */
338+ out = xs_list_new();
339+ }
340 }
341 }
342
343@@ -2730,8 +2989,9 @@ int mastoapi_post_handler(const xs_dict *req, const char *q_path,
344 const char *i_ctype = xs_dict_get(req, "content-type");
345
346 if (i_ctype && xs_startswith(i_ctype, "application/json")) {
347- if (!xs_is_null(payload))
348+ if (!xs_is_null(payload)) {
349 args = xs_json_loads(payload);
350+ }
351 }
352 else if (i_ctype && xs_startswith(i_ctype, "application/x-www-form-urlencoded"))
353 {
354@@ -2740,11 +3000,24 @@ int mastoapi_post_handler(const xs_dict *req, const char *q_path,
355 args = xs_url_vars(payload);
356 }
357 }
358- else
359+ else if (i_ctype && xs_startswith(i_ctype, "multipart/form-data"))
360+ {
361+ // Handle multipart/form-data by using p_vars (already parsed by httpd)
362+ args = xs_dup(xs_dict_get(req, "p_vars"));
363+ }
364+
365+ /* if args still NULL, try falling back to p_vars or q_vars */
366+ if (args == NULL)
367 args = xs_dup(xs_dict_get(req, "p_vars"));
368
369 if (args == NULL)
370+ args = xs_dup(xs_dict_get(req, "q_vars"));
371+
372+ if (args == NULL) {
373+ srv_debug(1, xs_fmt("mastoapi_post_handler: failed to parse args for %s, content-type: %s",
374+ q_path, i_ctype ? i_ctype : "(null)"));
375 return HTTP_STATUS_BAD_REQUEST;
376+ }
377
378 xs *cmd = xs_replace_n(q_path, "/api", "", 1);
379
380@@ -2926,7 +3199,12 @@ int mastoapi_post_handler(const xs_dict *req, const char *q_path,
381 /* skip the 'fake' part of the id */
382 mid = MID_TO_MD5(mid);
383
384- if (valid_status(timeline_get_by_md5(&snac, mid, &msg))) {
385+ /* try timeline first, then global object store for remote posts */
386+ int found = valid_status(timeline_get_by_md5(&snac, mid, &msg));
387+ if (!found)
388+ found = valid_status(object_get_by_md5(mid, &msg));
389+
390+ if (found) {
391 const char *id = xs_dict_get(msg, "id");
392
393 if (op == NULL) {
394@@ -3599,11 +3877,31 @@ int mastoapi_put_handler(const xs_dict *req, const char *q_path,
395 if (!xs_is_null(payload))
396 args = xs_json_loads(payload);
397 }
398- else
399+ else if (i_ctype && xs_startswith(i_ctype, "application/x-www-form-urlencoded"))
400+ {
401+ // Some apps send form data instead of json so we should cater for those
402+ if (!xs_is_null(payload)) {
403+ args = xs_url_vars(payload);
404+ }
405+ }
406+ else if (i_ctype && xs_startswith(i_ctype, "multipart/form-data"))
407+ {
408+ // Handle multipart/form-data by using p_vars (already parsed by httpd)
409 args = xs_dup(xs_dict_get(req, "p_vars"));
410+ }
411
412+ /* if args still NULL, try falling back to p_vars or q_vars */
413 if (args == NULL)
414+ args = xs_dup(xs_dict_get(req, "p_vars"));
415+
416+ if (args == NULL)
417+ args = xs_dup(xs_dict_get(req, "q_vars"));
418+
419+ if (args == NULL) {
420+ srv_debug(1, xs_fmt("mastoapi_put_handler: failed to parse args for %s, content-type: %s",
421+ q_path, i_ctype ? i_ctype : "(null)"));
422 return HTTP_STATUS_BAD_REQUEST;
423+ }
424
425 xs *cmd = xs_replace_n(q_path, "/api", "", 1);
426
427@@ -3753,11 +4051,24 @@ int mastoapi_patch_handler(const xs_dict *req, const char *q_path,
428 args = xs_url_vars(payload);
429 }
430 }
431- else
432+ else if (i_ctype && xs_startswith(i_ctype, "multipart/form-data"))
433+ {
434+ // Handle multipart/form-data by using p_vars (already parsed by httpd)
435+ args = xs_dup(xs_dict_get(req, "p_vars"));
436+ }
437+
438+ /* if args still NULL, try falling back to p_vars or q_vars */
439+ if (args == NULL)
440 args = xs_dup(xs_dict_get(req, "p_vars"));
441
442 if (args == NULL)
443+ args = xs_dup(xs_dict_get(req, "q_vars"));
444+
445+ if (args == NULL) {
446+ srv_debug(1, xs_fmt("mastoapi_patch_handler: failed to parse args for %s, content-type: %s",
447+ q_path, i_ctype ? i_ctype : "(null)"));
448 return HTTP_STATUS_BAD_REQUEST;
449+ }
450
451 xs *cmd = xs_replace_n(q_path, "/api", "", 1);
452
+28,
-0
1@@ -87,6 +87,34 @@ xs_dict *xs_httpd_request(FILE *f, xs_str **payload, int *p_size)
2 *p_size = atoi(v);
3 *payload = xs_read(f, p_size);
4 }
5+ else if ((v = xs_dict_get(req, "transfer-encoding")) != NULL &&
6+ xs_startswith(v, "chunked")) {
7+ /* handle chunked transfer encoding */
8+ xs_str *body = xs_str_new(NULL);
9+
10+ for (;;) {
11+ xs *line = xs_strip_i(xs_readline(f));
12+
13+ /* parse chunk size (in hex) */
14+ int chunk_size = strtol(line, NULL, 16);
15+
16+ if (chunk_size <= 0)
17+ break;
18+
19+ /* read chunk data */
20+ xs *chunk = xs_read(f, &chunk_size);
21+ if (chunk == NULL)
22+ break;
23+
24+ body = xs_append_m(body, chunk, chunk_size);
25+
26+ /* read trailing \r\n after chunk data */
27+ xs_readline(f);
28+ }
29+
30+ *p_size = xs_size(body) - 1; /* subtract trailing null */
31+ *payload = body;
32+ }
33
34 v = xs_dict_get(req, "content-type");
35