From 97ff66116b7633066375bbc6cc88be5b8587453b Mon Sep 17 00:00:00 2001 From: grunfink Date: Wed, 28 May 2025 07:48:47 +0200 Subject: New file rss.c. --- Makefile.NetBSD | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'Makefile.NetBSD') diff --git a/Makefile.NetBSD b/Makefile.NetBSD index ecf8205..a2bf83c 100644 --- a/Makefile.NetBSD +++ b/Makefile.NetBSD @@ -6,7 +6,7 @@ LDFLAGS=-lrt all: snac snac: snac.o main.o sandbox.o data.o http.o httpd.o webfinger.o \ - activitypub.o html.o utils.o format.o upgrade.o mastoapi.o + activitypub.o html.o utils.o format.o upgrade.o mastoapi.o rss.o $(CC) $(CFLAGS) -L/usr/pkg/lib *.o -lcurl -lcrypto -pthread $(LDFLAGS) -Wl,-rpath,/usr/lib -Wl,-rpath,/usr/pkg/lib -o $@ @@ -37,12 +37,12 @@ uninstall: activitypub.o: activitypub.c xs.h xs_json.h xs_curl.h xs_mime.h \ xs_openssl.h xs_regex.h xs_time.h xs_set.h xs_match.h xs_unicode.h \ - snac.h http_codes.h + xs_webmention.h snac.h http_codes.h data.o: data.c xs.h xs_hex.h xs_io.h xs_json.h xs_openssl.h xs_glob.h \ xs_set.h xs_time.h xs_regex.h xs_match.h xs_unicode.h xs_random.h \ xs_po.h snac.h http_codes.h format.o: format.c xs.h xs_regex.h xs_mime.h xs_html.h xs_json.h \ - xs_time.h xs_match.h snac.h http_codes.h + xs_time.h xs_match.h xs_unicode.h snac.h http_codes.h html.o: html.c xs.h xs_io.h xs_json.h xs_regex.h xs_set.h xs_openssl.h \ xs_time.h xs_mime.h xs_match.h xs_html.h xs_curl.h xs_unicode.h xs_url.h \ xs_random.h snac.h http_codes.h @@ -55,7 +55,8 @@ main.o: main.c xs.h xs_io.h xs_json.h xs_time.h xs_openssl.h xs_match.h \ snac.h http_codes.h mastoapi.o: mastoapi.c xs.h xs_hex.h xs_openssl.h xs_json.h xs_io.h \ xs_time.h xs_glob.h xs_set.h xs_random.h xs_url.h xs_mime.h xs_match.h \ - snac.h http_codes.h + xs_unicode.h snac.h http_codes.h +rss.o: rss.c xs.h xs_html.h xs_regex.h xs_time.h snac.h http_codes.h sandbox.o: sandbox.c xs.h snac.h http_codes.h snac.o: snac.c xs.h xs_hex.h xs_io.h xs_unicode_tbl.h xs_unicode.h \ xs_json.h xs_curl.h xs_openssl.h xs_socket.h xs_unix_socket.h xs_url.h \ -- cgit From c10f6f0e6c3b82c02cbd47707b399330173cae38 Mon Sep 17 00:00:00 2001 From: grunfink Date: Wed, 28 May 2025 07:59:50 +0200 Subject: Updated dependencies. --- Makefile | 3 ++- Makefile.NetBSD | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'Makefile.NetBSD') diff --git a/Makefile b/Makefile index bb4029c..4a24899 100644 --- a/Makefile +++ b/Makefile @@ -67,7 +67,8 @@ main.o: main.c xs.h xs_io.h xs_json.h xs_time.h xs_openssl.h xs_match.h \ mastoapi.o: mastoapi.c xs.h xs_hex.h xs_openssl.h xs_json.h xs_io.h \ xs_time.h xs_glob.h xs_set.h xs_random.h xs_url.h xs_mime.h xs_match.h \ xs_unicode.h snac.h http_codes.h -rss.o: rss.c xs.h xs_html.h xs_regex.h xs_time.h snac.h http_codes.h +rss.o: rss.c xs.h xs_html.h xs_regex.h xs_time.h xs_match.h xs_curl.h \ + snac.h http_codes.h sandbox.o: sandbox.c xs.h snac.h http_codes.h snac.o: snac.c xs.h xs_hex.h xs_io.h xs_unicode_tbl.h xs_unicode.h \ xs_json.h xs_curl.h xs_openssl.h xs_socket.h xs_unix_socket.h xs_url.h \ diff --git a/Makefile.NetBSD b/Makefile.NetBSD index a2bf83c..9dc534e 100644 --- a/Makefile.NetBSD +++ b/Makefile.NetBSD @@ -56,7 +56,8 @@ main.o: main.c xs.h xs_io.h xs_json.h xs_time.h xs_openssl.h xs_match.h \ mastoapi.o: mastoapi.c xs.h xs_hex.h xs_openssl.h xs_json.h xs_io.h \ xs_time.h xs_glob.h xs_set.h xs_random.h xs_url.h xs_mime.h xs_match.h \ xs_unicode.h snac.h http_codes.h -rss.o: rss.c xs.h xs_html.h xs_regex.h xs_time.h snac.h http_codes.h +rss.o: rss.c xs.h xs_html.h xs_regex.h xs_time.h xs_match.h xs_curl.h \ + snac.h http_codes.h sandbox.o: sandbox.c xs.h snac.h http_codes.h snac.o: snac.c xs.h xs_hex.h xs_io.h xs_unicode_tbl.h xs_unicode.h \ xs_json.h xs_curl.h xs_openssl.h xs_socket.h xs_unix_socket.h xs_url.h \ -- cgit From e030fe6c5054c0a9b76a55adc80bb81d7a73fcc3 Mon Sep 17 00:00:00 2001 From: grunfink Date: Thu, 29 May 2025 18:03:23 +0200 Subject: Use HTTP caching (etag / if-none-match) in RSS downloads. --- Makefile | 2 +- Makefile.NetBSD | 2 +- rss.c | 40 ++++++++++++++++++++++++++++++++++++++-- 3 files changed, 40 insertions(+), 4 deletions(-) (limited to 'Makefile.NetBSD') diff --git a/Makefile b/Makefile index 4a24899..6059fef 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,7 @@ mastoapi.o: mastoapi.c xs.h xs_hex.h xs_openssl.h xs_json.h xs_io.h \ xs_time.h xs_glob.h xs_set.h xs_random.h xs_url.h xs_mime.h xs_match.h \ xs_unicode.h snac.h http_codes.h rss.o: rss.c xs.h xs_html.h xs_regex.h xs_time.h xs_match.h xs_curl.h \ - snac.h http_codes.h + xs_openssl.h xs_json.h snac.h http_codes.h sandbox.o: sandbox.c xs.h snac.h http_codes.h snac.o: snac.c xs.h xs_hex.h xs_io.h xs_unicode_tbl.h xs_unicode.h \ xs_json.h xs_curl.h xs_openssl.h xs_socket.h xs_unix_socket.h xs_url.h \ diff --git a/Makefile.NetBSD b/Makefile.NetBSD index 9dc534e..d19ab2d 100644 --- a/Makefile.NetBSD +++ b/Makefile.NetBSD @@ -57,7 +57,7 @@ mastoapi.o: mastoapi.c xs.h xs_hex.h xs_openssl.h xs_json.h xs_io.h \ xs_time.h xs_glob.h xs_set.h xs_random.h xs_url.h xs_mime.h xs_match.h \ xs_unicode.h snac.h http_codes.h rss.o: rss.c xs.h xs_html.h xs_regex.h xs_time.h xs_match.h xs_curl.h \ - snac.h http_codes.h + xs_openssl.h xs_json.h snac.h http_codes.h sandbox.o: sandbox.c xs.h snac.h http_codes.h snac.o: snac.c xs.h xs_hex.h xs_io.h xs_unicode_tbl.h xs_unicode.h \ xs_json.h xs_curl.h xs_openssl.h xs_socket.h xs_unix_socket.h xs_url.h \ diff --git a/rss.c b/rss.c index ed0bf05..9d582f7 100644 --- a/rss.c +++ b/rss.c @@ -7,6 +7,8 @@ #include "xs_time.h" #include "xs_match.h" #include "xs_curl.h" +#include "xs_openssl.h" +#include "xs_json.h" #include "snac.h" @@ -117,12 +119,37 @@ void rss_to_timeline(snac *user, const char *url) hdrs = xs_dict_set(hdrs, "accept", "application/rss+xml"); hdrs = xs_dict_set(hdrs, "user-agent", USER_AGENT); + /* get the RSS metadata */ + xs *md5 = xs_md5_hex(url, strlen(url)); + xs *rss_md_fn = xs_fmt("%s/rss", user->basedir); + mkdirx(rss_md_fn); + rss_md_fn = xs_str_cat(rss_md_fn, "/", md5, ".json"); + + xs *rss_md = NULL; + const char *etag = NULL; + + FILE *f; + if ((f = fopen(rss_md_fn, "r")) != NULL) { + rss_md = xs_json_load(f); + fclose(f); + + etag = xs_dict_get(rss_md, "etag"); + + if (xs_is_string(etag)) + hdrs = xs_dict_set(hdrs, "if-none-match", etag); + } + + if (rss_md == NULL) + rss_md = xs_dict_new(); + xs *payload = NULL; int status; int p_size; xs *rsp = xs_http_request("GET", url, hdrs, NULL, 0, &status, &payload, &p_size, 0); + snac_log(user, xs_fmt("parsing RSS %s %d", url, status)); + if (!valid_status(status) || !xs_is_string(payload)) return; @@ -131,8 +158,6 @@ void rss_to_timeline(snac *user, const char *url) if (!xs_is_string(ctype) || xs_str_in(ctype, "application/rss+xml") == -1) return; - snac_log(user, xs_fmt("parsing RSS %s", url)); - /* yes, parsing is done with regexes (now I have two problems blah blah blah) */ xs *links = xs_regex_select(payload, "[^<]+"); const char *link; @@ -208,6 +233,17 @@ void rss_to_timeline(snac *user, const char *url) timeline_add(user, id, obj); } } + + /* update the RSS metadata */ + etag = xs_dict_get(rsp, "etag"); + + if (xs_is_string(etag)) { + rss_md = xs_dict_set(rss_md, "etag", etag); + if ((f = fopen(rss_md_fn, "w")) != NULL) { + xs_json_dump(rss_md, 4, f); + fclose(f); + } + } } -- cgit From c976b9a334872e14aadc1f454680599468652231 Mon Sep 17 00:00:00 2001 From: grunfink Date: Thu, 26 Jun 2025 10:39:47 +0200 Subject: Sending notes from the cmdline allows setting an inReplyTo field. --- Makefile | 6 +++--- Makefile.NetBSD | 6 +++--- main.c | 23 +++++++++++++++++++---- 3 files changed, 25 insertions(+), 10 deletions(-) (limited to 'Makefile.NetBSD') diff --git a/Makefile b/Makefile index 6059fef..5de2882 100644 --- a/Makefile +++ b/Makefile @@ -60,10 +60,10 @@ html.o: html.c xs.h xs_io.h xs_json.h xs_regex.h xs_set.h xs_openssl.h \ http.o: http.c xs.h xs_io.h xs_openssl.h xs_curl.h xs_time.h xs_json.h \ snac.h http_codes.h httpd.o: httpd.c xs.h xs_io.h xs_json.h xs_socket.h xs_unix_socket.h \ - xs_httpd.h xs_mime.h xs_time.h xs_openssl.h xs_fcgi.h xs_html.h snac.h \ - http_codes.h + xs_httpd.h xs_mime.h xs_time.h xs_openssl.h xs_fcgi.h xs_html.h \ + xs_webmention.h snac.h http_codes.h main.o: main.c xs.h xs_io.h xs_json.h xs_time.h xs_openssl.h xs_match.h \ - snac.h http_codes.h + xs_random.h snac.h http_codes.h mastoapi.o: mastoapi.c xs.h xs_hex.h xs_openssl.h xs_json.h xs_io.h \ xs_time.h xs_glob.h xs_set.h xs_random.h xs_url.h xs_mime.h xs_match.h \ xs_unicode.h snac.h http_codes.h diff --git a/Makefile.NetBSD b/Makefile.NetBSD index d19ab2d..ac5b052 100644 --- a/Makefile.NetBSD +++ b/Makefile.NetBSD @@ -49,10 +49,10 @@ html.o: html.c xs.h xs_io.h xs_json.h xs_regex.h xs_set.h xs_openssl.h \ http.o: http.c xs.h xs_io.h xs_openssl.h xs_curl.h xs_time.h xs_json.h \ snac.h http_codes.h httpd.o: httpd.c xs.h xs_io.h xs_json.h xs_socket.h xs_unix_socket.h \ - xs_httpd.h xs_mime.h xs_time.h xs_openssl.h xs_fcgi.h xs_html.h snac.h \ - http_codes.h + xs_httpd.h xs_mime.h xs_time.h xs_openssl.h xs_fcgi.h xs_html.h \ + xs_webmention.h snac.h http_codes.h main.o: main.c xs.h xs_io.h xs_json.h xs_time.h xs_openssl.h xs_match.h \ - snac.h http_codes.h + xs_random.h snac.h http_codes.h mastoapi.o: mastoapi.c xs.h xs_hex.h xs_openssl.h xs_json.h xs_io.h \ xs_time.h xs_glob.h xs_set.h xs_random.h xs_url.h xs_mime.h xs_match.h \ xs_unicode.h snac.h http_codes.h diff --git a/main.c b/main.c index f271abc..ce4e7ce 100644 --- a/main.c +++ b/main.c @@ -7,6 +7,7 @@ #include "xs_time.h" #include "xs_openssl.h" #include "xs_match.h" +#include "xs_random.h" #include "snac.h" @@ -779,12 +780,24 @@ int main(int argc, char *argv[]) xs *msg = NULL; xs *c_msg = NULL; xs *attl = xs_list_new(); - char *fn = NULL; + const char *fn = NULL; + const char *in_reply_to = NULL; + const char **next = NULL; /* iterate possible attachments */ while ((fn = GET_ARGV())) { FILE *f; + if (next) { + *next = fn; + next = NULL; + } + else + if (strcmp(fn, "-r") == 0) { + /* next argument is an inReplyTo */ + next = &in_reply_to; + } + else if ((f = fopen(fn, "rb")) != NULL) { /* get the file size and content */ fseek(f, 0, SEEK_END); @@ -794,8 +807,10 @@ int main(int argc, char *argv[]) fclose(f); char *ext = strrchr(fn, '.'); - xs *hash = xs_md5_hex(fn, strlen(fn)); - xs *id = xs_fmt("%s%s", hash, ext); + char rnd[32]; + xs_rnd_buf(rnd, sizeof(rnd)); + xs *hash = xs_md5_hex(rnd, sizeof(rnd)); + xs *id = xs_fmt("post-%s%s", hash, ext ? ext : ""); xs *url = xs_fmt("%s/s/%s", snac.actor, id); /* store */ @@ -856,7 +871,7 @@ int main(int argc, char *argv[]) if (strcmp(cmd, "note_unlisted") == 0) scope = 2; - msg = msg_note(&snac, content, NULL, NULL, attl, scope, getenv("LANG"), NULL); + msg = msg_note(&snac, content, NULL, in_reply_to, attl, scope, getenv("LANG"), NULL); c_msg = msg_create(&snac, msg); -- cgit