From 24105f6e9759b74c04522de1de12ddb77cfba568 Mon Sep 17 00:00:00 2001 From: postscriptum Date: Thu, 22 May 2025 03:34:48 +0300 Subject: use utf-8 lowercase function for tags #396 --- activitypub.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'activitypub.c') diff --git a/activitypub.c b/activitypub.c index a7e133a..3ff51ad 100644 --- a/activitypub.c +++ b/activitypub.c @@ -903,7 +903,7 @@ xs_str *process_tags(snac *snac, const char *content, xs_list **tag) if (*v == '#') { /* hashtag */ xs *d = xs_dict_new(); - xs *n = xs_tolower_i(xs_dup(v)); + xs *n = xs_utf8_to_lower(xs_dup(v)); xs *h = xs_fmt("%s?t=%s", srv_baseurl, n + 1); xs *l = xs_fmt("%s", h, v); -- cgit From 56816b305155fee2154c7991ba9be8c0e7671307 Mon Sep 17 00:00:00 2001 From: grunfink Date: Thu, 22 May 2025 11:18:48 +0200 Subject: Minor memory leak fixes. --- activitypub.c | 2 +- data.c | 8 ++++---- format.c | 2 +- mastoapi.c | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) (limited to 'activitypub.c') diff --git a/activitypub.c b/activitypub.c index c06d8df..0cc7bcb 100644 --- a/activitypub.c +++ b/activitypub.c @@ -903,7 +903,7 @@ xs_str *process_tags(snac *snac, const char *content, xs_list **tag) if (*v == '#') { /* hashtag */ xs *d = xs_dict_new(); - xs *n = xs_utf8_to_lower(xs_dup(v)); + xs *n = xs_utf8_to_lower(v); xs *h = xs_fmt("%s?t=%s", srv_baseurl, n + 1); xs *l = xs_fmt("%s", h, v); diff --git a/data.c b/data.c index 631a68b..5f890d3 100644 --- a/data.c +++ b/data.c @@ -2247,7 +2247,7 @@ xs_str *tag_fn(const char *tag) if (*tag == '#') tag++; - xs *lw_tag = xs_utf8_to_lower(xs_dup(tag)); + xs *lw_tag = xs_utf8_to_lower(tag); xs *md5 = xs_md5_hex(lw_tag, strlen(lw_tag)); return xs_fmt("%s/tag/%c%c/%s.idx", srv_basedir, md5[0], md5[1], md5); @@ -2832,9 +2832,9 @@ int content_match(const char *file, const xs_dict *msg) srv_debug(1, xs_fmt("content_match: loading regexes from %s", fn)); /* massage content (strip HTML tags, etc.) */ - xs *c = xs_regex_replace(v, "<[^>]+>", " "); - c = xs_regex_replace_i(c, " {2,}", " "); - c = xs_utf8_to_lower(c); + xs *c1 = xs_regex_replace(v, "<[^>]+>", " "); + c1 = xs_regex_replace_i(c1, " {2,}", " "); + xs *c = xs_utf8_to_lower(c1); while (!r && !feof(f)) { xs *rx = xs_strip_i(xs_readline(f)); diff --git a/format.c b/format.c index 0f844cd..9bad391 100644 --- a/format.c +++ b/format.c @@ -444,7 +444,7 @@ xs_str *sanitize(const char *content) if (n & 0x1) { xs *s1 = xs_strip_i(xs_crop_i(xs_dup(v), v[1] == '/' ? 2 : 1, -1)); xs *l1 = xs_split_n(s1, " ", 1); - xs *tag = xs_utf8_to_lower(xs_dup(xs_list_get(l1, 0))); + xs *tag = xs_utf8_to_lower(xs_list_get(l1, 0)); xs *s2 = NULL; int i; diff --git a/mastoapi.c b/mastoapi.c index c46a971..9875b12 100644 --- a/mastoapi.c +++ b/mastoapi.c @@ -1638,7 +1638,7 @@ int mastoapi_get_handler(const xs_dict *req, const char *q_path, const char *aq = xs_dict_get(args, "q"); if (!xs_is_null(aq)) { - xs *q = xs_utf8_to_lower(xs_dup(aq)); + xs *q = xs_utf8_to_lower(aq); out = xs_list_new(); xs *wing = following_list(&snac1); xs *wers = follower_list(&snac1); -- cgit From a1369b39c1bd3d2036af12368997648454ca5564 Mon Sep 17 00:00:00 2001 From: grunfink Date: Wed, 28 May 2025 09:07:19 +0200 Subject: Activated hashtag RSS polling. --- activitypub.c | 4 ++++ html.c | 3 ++- httpd.c | 42 +++++++++++++++++++++++++++++++----------- rss.c | 9 +++++---- snac.h | 2 +- 5 files changed, 43 insertions(+), 17 deletions(-) (limited to 'activitypub.c') diff --git a/activitypub.c b/activitypub.c index 0cc7bcb..2cffeac 100644 --- a/activitypub.c +++ b/activitypub.c @@ -3049,6 +3049,10 @@ void process_queue_item(xs_dict *q_item) } } } + else + if (strcmp(type, "rss_poll") == 0) { + rss_poll_hashtags(); + } else srv_log(xs_fmt("unexpected q_item type '%s'", type)); } diff --git a/html.c b/html.c index 7fcc266..3d213a2 100644 --- a/html.c +++ b/html.c @@ -1601,7 +1601,8 @@ xs_html *html_top_controls(snac *user) xs_html_attr("name", "followed_hashtags"), xs_html_attr("cols", "40"), xs_html_attr("rows", "4"), - xs_html_attr("placeholder", "#cats\n#windowfriday\n#classicalmusic"), + xs_html_attr("placeholder", "#cats\n#windowfriday\n#classicalmusic\nhttps:/" + "/mastodon.social/tags/dogs"), xs_html_text(followed_hashtags)), xs_html_tag("br", NULL), diff --git a/httpd.c b/httpd.c index 41b2515..c94a542 100644 --- a/httpd.c +++ b/httpd.c @@ -705,34 +705,36 @@ static pthread_cond_t sleep_cond; static void *background_thread(void *arg) /* background thread (queue management and other things) */ { - time_t purge_time; + time_t t, purge_time, rss_time; (void)arg; + t = time(NULL); + /* first purge time */ - purge_time = time(NULL) + 10 * 60; + purge_time = t + 10 * 60; + + /* first RSS polling time */ + rss_time = t + 15 * 60; srv_log(xs_fmt("background thread started")); while (p_state->srv_running) { - time_t t; int cnt = 0; p_state->th_state[0] = THST_QUEUE; { xs *list = user_list(); - char *p; const char *uid; /* process queues for all users */ - p = list; - while (xs_list_iter(&p, &uid)) { - snac snac; + xs_list_foreach(list, uid) { + snac user; - if (user_open(&snac, uid)) { - cnt += process_user_queue(&snac); - user_free(&snac); + if (user_open(&user, uid)) { + cnt += process_user_queue(&user); + user_free(&user); } } } @@ -740,8 +742,10 @@ static void *background_thread(void *arg) /* global queue */ cnt += process_queue(); + t = time(NULL); + /* time to purge? */ - if ((t = time(NULL)) > purge_time) { + if (t > purge_time) { /* next purge time is tomorrow */ purge_time = t + 24 * 60 * 60; @@ -750,6 +754,22 @@ static void *background_thread(void *arg) job_post(q_item, 0); } + /* time to poll the RSS? */ + if (t > rss_time) { + /* next RSS poll time */ + int hours = xs_number_get(xs_dict_get_def(srv_config, "rss_poll_hours", "4")); + + /* don't hammer servers too much */ + if (hours < 1) + hours = 1; + + rss_time = t + 60 * 60 * hours; + + xs *q_item = xs_dict_new(); + q_item = xs_dict_append(q_item, "type", "rss_poll"); + job_post(q_item, 0); + } + if (cnt == 0) { /* sleep 3 seconds */ diff --git a/rss.c b/rss.c index b31ffd3..ce3e184 100644 --- a/rss.c +++ b/rss.c @@ -110,6 +110,9 @@ xs_str *rss_from_timeline(snac *user, const xs_list *timeline, void rss_to_timeline(snac *user, const char *url) /* reads an RSS and inserts all ActivityPub posts into the user's timeline */ { + if (!xs_startswith(url, "https:/") && !xs_startswith(url, "http:/")) + return; + xs *hdrs = xs_dict_new(); hdrs = xs_dict_set(hdrs, "accept", "application/rss+xml"); hdrs = xs_dict_set(hdrs, "user-agent", USER_AGENT); @@ -204,10 +207,9 @@ void rss_to_timeline(snac *user, const char *url) } -void rss_process(void) +void rss_poll_hashtags(void) /* parses all RSS from all users */ { -#if 0 xs *list = user_list(); const char *uid; @@ -215,7 +217,7 @@ void rss_process(void) snac user; if (user_open(&user, uid)) { - const xs_list *rss = xs_dict_get(user.config, "rss"); + const xs_list *rss = xs_dict_get(user.config, "followed_hashtags"); if (xs_is_list(rss)) { const char *url; @@ -227,5 +229,4 @@ void rss_process(void) user_free(&user); } } -#endif } diff --git a/snac.h b/snac.h index db51a1c..06a36f1 100644 --- a/snac.h +++ b/snac.h @@ -464,4 +464,4 @@ const char *lang_str(const char *str, const snac *user); xs_str *rss_from_timeline(snac *user, const xs_list *timeline, const char *title, const char *link, const char *desc); void rss_to_timeline(snac *user, const char *url); -void rss_process(void); +void rss_poll_hashtags(void); -- cgit From 0594197af7a4a11795150a85fafa7fa5bc6fc4f5 Mon Sep 17 00:00:00 2001 From: grunfink Date: Wed, 28 May 2025 09:16:23 +0200 Subject: Renamed server knob to 'rss_hashtag_poll_hours'. --- activitypub.c | 2 +- httpd.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'activitypub.c') diff --git a/activitypub.c b/activitypub.c index 2cffeac..8877a27 100644 --- a/activitypub.c +++ b/activitypub.c @@ -3050,7 +3050,7 @@ void process_queue_item(xs_dict *q_item) } } else - if (strcmp(type, "rss_poll") == 0) { + if (strcmp(type, "rss_hashtag_poll") == 0) { rss_poll_hashtags(); } else diff --git a/httpd.c b/httpd.c index c94a542..15634d1 100644 --- a/httpd.c +++ b/httpd.c @@ -757,7 +757,7 @@ static void *background_thread(void *arg) /* time to poll the RSS? */ if (t > rss_time) { /* next RSS poll time */ - int hours = xs_number_get(xs_dict_get_def(srv_config, "rss_poll_hours", "4")); + int hours = xs_number_get(xs_dict_get_def(srv_config, "rss_hashtag_poll_hours", "4")); /* don't hammer servers too much */ if (hours < 1) @@ -766,7 +766,7 @@ static void *background_thread(void *arg) rss_time = t + 60 * 60 * hours; xs *q_item = xs_dict_new(); - q_item = xs_dict_append(q_item, "type", "rss_poll"); + q_item = xs_dict_append(q_item, "type", "rss_hashtag_poll"); job_post(q_item, 0); } -- cgit From 5a2aef8666a82a30dff329992bd41baa53d4e123 Mon Sep 17 00:00:00 2001 From: grunfink Date: Fri, 30 May 2025 12:12:19 +0200 Subject: More notify_webhook work. --- activitypub.c | 33 +++++++++++++++++++++++++++++++++ data.c | 11 ++++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) (limited to 'activitypub.c') diff --git a/activitypub.c b/activitypub.c index 8877a27..fb0c42c 100644 --- a/activitypub.c +++ b/activitypub.c @@ -2765,6 +2765,39 @@ void process_user_queue_item(snac *user, xs_dict *q_item) snac_log(user, xs_fmt("actor_refresh %s %d", actor, status)); } } + else + if (strcmp(type, "notify_webhook") == 0) { + const char *webhook = xs_dict_get(user->config, "notify_webhook"); + + if (xs_is_string(webhook)) { + const xs_dict *msg = xs_dict_get(q_item, "message"); + int retries = xs_number_get(xs_dict_get(q_item, "retries")); + + xs *hdrs = xs_dict_new(); + + hdrs = xs_dict_set(hdrs, "content-type", "application/json"); + hdrs = xs_dict_set(hdrs, "user-agent", USER_AGENT); + + xs *body = xs_json_dumps(msg, 4); + + int status; + xs *rsp = xs_http_request("POST", webhook, hdrs, body, strlen(body), &status, NULL, NULL, 0); + + snac_debug(user, 0, xs_fmt("webhook post %s %d", webhook, status)); + + if (!valid_status(status)) { + retries++; + + if (retries > queue_retry_max) + snac_debug(user, 0, xs_fmt("webhook post giving up %s", webhook)); + else { + snac_debug(user, 0, xs_fmt("webhook post requeue %s %d", webhook, retries)); + + enqueue_notify_webhook(user, msg, retries); + } + } + } + } else snac_log(user, xs_fmt("unexpected user q_item type '%s'", type)); } diff --git a/data.c b/data.c index ddfb443..10084f9 100644 --- a/data.c +++ b/data.c @@ -3529,7 +3529,16 @@ void enqueue_notify_webhook(snac *user, const xs_dict *noti, int retries) const char *webhook = xs_dict_get(user->config, "notify_webhook"); if (xs_is_string(webhook)) { - xs *qmsg = _new_qmsg("notify_webhook", noti, retries); + xs *msg = xs_dup(noti); + + /* add more data */ + msg = xs_dict_set(msg, "target", user->actor); + xs *actor_obj = NULL; + + if (valid_status(object_get(xs_dict_get(noti, "actor"), &actor_obj)) && actor_obj) + msg = xs_dict_set(msg, "account", actor_obj); + + xs *qmsg = _new_qmsg("notify_webhook", msg, retries); const char *ntid = xs_dict_get(qmsg, "ntid"); xs *fn = xs_fmt("%s/queue/%s.json", user->basedir, ntid); -- cgit From f055c8b694a398868d14fd70df61d02429846dae Mon Sep 17 00:00:00 2001 From: grunfink Date: Fri, 30 May 2025 19:34:11 +0200 Subject: Added a new server knob disable_notify_webhook. --- activitypub.c | 2 +- data.c | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'activitypub.c') diff --git a/activitypub.c b/activitypub.c index fb0c42c..ab63abe 100644 --- a/activitypub.c +++ b/activitypub.c @@ -2769,7 +2769,7 @@ void process_user_queue_item(snac *user, xs_dict *q_item) if (strcmp(type, "notify_webhook") == 0) { const char *webhook = xs_dict_get(user->config, "notify_webhook"); - if (xs_is_string(webhook)) { + if (xs_is_string(webhook) && *webhook) { const xs_dict *msg = xs_dict_get(q_item, "message"); int retries = xs_number_get(xs_dict_get(q_item, "retries")); diff --git a/data.c b/data.c index 10084f9..74bc112 100644 --- a/data.c +++ b/data.c @@ -3167,7 +3167,8 @@ void notify_add(snac *snac, const char *type, const char *utype, pthread_mutex_unlock(&data_mutex); } - enqueue_notify_webhook(snac, noti, 0); + if (!xs_is_true(xs_dict_get(srv_config, "disable_notify_webhook"))) + enqueue_notify_webhook(snac, noti, 0); } @@ -3528,7 +3529,7 @@ void enqueue_notify_webhook(snac *user, const xs_dict *noti, int retries) { const char *webhook = xs_dict_get(user->config, "notify_webhook"); - if (xs_is_string(webhook)) { + if (xs_is_string(webhook) && *webhook) { xs *msg = xs_dup(noti); /* add more data */ -- cgit From c8848f6e9f8e9fd9d17290b1ef301d3bf7beccb4 Mon Sep 17 00:00:00 2001 From: grunfink Date: Fri, 30 May 2025 19:59:40 +0200 Subject: More webhook checks. --- activitypub.c | 2 +- data.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'activitypub.c') diff --git a/activitypub.c b/activitypub.c index ab63abe..100db67 100644 --- a/activitypub.c +++ b/activitypub.c @@ -2769,7 +2769,7 @@ void process_user_queue_item(snac *user, xs_dict *q_item) if (strcmp(type, "notify_webhook") == 0) { const char *webhook = xs_dict_get(user->config, "notify_webhook"); - if (xs_is_string(webhook) && *webhook) { + if (xs_is_string(webhook) && xs_match(webhook, "https://*|http://*")) { /** **/ const xs_dict *msg = xs_dict_get(q_item, "message"); int retries = xs_number_get(xs_dict_get(q_item, "retries")); diff --git a/data.c b/data.c index 28f7cd9..6c38631 100644 --- a/data.c +++ b/data.c @@ -3529,7 +3529,7 @@ void enqueue_notify_webhook(snac *user, const xs_dict *noti, int retries) { const char *webhook = xs_dict_get(user->config, "notify_webhook"); - if (xs_is_string(webhook) && *webhook) { + if (xs_is_string(webhook) && xs_match(webhook, "https://*|http://*")) { /** **/ xs *msg = xs_dup(noti); /* add more data */ -- cgit