From 68f615cc6e6ac177109db5b3be4f9767c9cfd1a8 Mon Sep 17 00:00:00 2001 From: Jeff Halter <868228+jhalter@users.noreply.github.com> Date: Sun, 23 Aug 2026 12:44:53 -0700 Subject: Add feed-backed threaded news imports --- README.md | 25 ++ cmd/mobius-hotline-server/main.go | 17 +- .../mobius/config/config.yaml | 21 +- docs/feed-backed-news.md | 129 ++++++ docs/text-encoding.md | 40 +- go.mod | 16 +- go.sum | 32 +- hotline/config.go | 55 ++- hotline/news.go | 25 +- hotline/time.go | 25 ++ hotline/time_test.go | 13 + internal/mobius/config.go | 37 ++ internal/mobius/config_test.go | 91 ++++ internal/mobius/feed_news.go | 217 ++++++++++ internal/mobius/feed_news_fetch.go | 478 +++++++++++++++++++++ internal/mobius/feed_news_import.go | 238 ++++++++++ internal/mobius/feed_news_test.go | 462 ++++++++++++++++++++ internal/mobius/handlers_news.go | 2 +- internal/mobius/testdata/afterglow-appcast.xml | 22 + internal/mobius/testdata/mobius-releases.atom | 16 + 20 files changed, 1898 insertions(+), 63 deletions(-) create mode 100644 docs/feed-backed-news.md create mode 100644 internal/mobius/feed_news.go create mode 100644 internal/mobius/feed_news_fetch.go create mode 100644 internal/mobius/feed_news_import.go create mode 100644 internal/mobius/feed_news_test.go create mode 100644 internal/mobius/testdata/afterglow-appcast.xml create mode 100644 internal/mobius/testdata/mobius-releases.atom diff --git a/README.md b/README.md index 3a25a5b..d09719b 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,31 @@ Within this directory some files are intended to be edited to customize the serv 🛠️ `config.yaml` - Edit to set your server name, description, and enable tracker registration. +### Feed-backed threaded news + +An advanced `NewsFeeds` configuration can import RSS or Atom entries into an +existing ordinary news category, including Sparkle appcasts and GitHub release +feeds: + +See [Feed-backed threaded news](docs/feed-backed-news.md) for the complete +administrator guide, including configuration rules, retention behavior, +durable state, operational limits, and troubleshooting. + +```yaml +NewsFeeds: + - CategoryPath: ["Software Updates", "Afterglow"] + URL: "https://morphing.cloud/afterglow/appcast.xml" + - CategoryPath: ["Software Updates", "Mobius"] + URL: "https://github.com/jhalter/mobius/releases.atom" +``` + +The full target path must already exist as an ordinary category. Whenever a +client opens it, Mobius conditionally checks the source and saves new entries +as ordinary root articles in `ThreadedNews.yaml`. Existing entries are never +rewritten, and deleted imports stay deleted. Feed failures are logged while the +current local articles remain available. RSS and Atom are detected +automatically; changes to `NewsFeeds` require restarting the server. + ### User accounts diff --git a/cmd/mobius-hotline-server/main.go b/cmd/mobius-hotline-server/main.go index 45b9a08..2511227 100644 --- a/cmd/mobius-hotline-server/main.go +++ b/cmd/mobius-hotline-server/main.go @@ -205,9 +205,24 @@ func main() { slogger.Error("Error loading news", "err", err) os.Exit(1) } - srv.ThreadedNewsMgr = threadedNews reloaders = append(reloaders, namedReloader{"threaded news", threadedNews}) + srv.ThreadedNewsMgr = threadedNews + if len(config.NewsFeeds) > 0 { + feedNews, err := mobius.NewFeedNewsManager( + threadedNews, + config.NewsFeeds, + slogger, + "mobius-hotline-server/"+version, + config.Encoding, + ) + if err != nil { + slogger.Error("Error configuring feed-backed news", "err", err) + os.Exit(1) + } + srv.ThreadedNewsMgr = feedNews + } + srv.AccountManager, err = mobius.NewYAMLAccountManager(path.Join(*configDir, "Users/")) if err != nil { slogger.Error("Error loading accounts", "err", err) diff --git a/cmd/mobius-hotline-server/mobius/config/config.yaml b/cmd/mobius-hotline-server/mobius/config/config.yaml index d7ded22..2fda24c 100644 --- a/cmd/mobius-hotline-server/mobius/config/config.yaml +++ b/cmd/mobius-hotline-server/mobius/config/config.yaml @@ -25,10 +25,11 @@ Trackers: - hotline.kicks-ass.net:5499 # - example-tracker-with-password.com:5499:mypassword -# Text encoding used for file and folder names sent to and from clients. +# Text encoding used for file and folder names and server-generated feed news. +# User-authored chat, news, replies, and usernames remain raw protocol bytes. # Supported values: -# "macintosh" - Translates between Mac Roman and UTF-8 (default, compatible with classic Hotline clients) -# "utf8" - No conversion; passes file names through as-is (for servers with only modern UTF-8 clients) +# "macintosh" - Converts filesystem names and feed news for classic Hotline clients (default) +# "utf8" - Leaves UTF-8 filesystem names and feed news unchanged (modern clients only) # If omitted, defaults to "macintosh". Encoding: macintosh @@ -52,6 +53,18 @@ NewsDateFormat: "" # Optional custom delimiter between flat news postings NewsDelimiter: "" +# Optional mappings from existing ordinary news categories to public RSS or +# Atom feeds. Opening a mapped category checks its source and imports unseen +# entries as ordinary root articles in ThreadedNews.yaml. The complete category +# path must already exist. Feed formats are detected automatically. +# +# NewsFeeds: +# - CategoryPath: ["Software Updates", "Afterglow"] +# URL: "https://morphing.cloud/afterglow/appcast.xml" +# - CategoryPath: ["Software Updates", "Mobius"] +# URL: "https://github.com/jhalter/mobius/releases.atom" +NewsFeeds: [] + # Maximum simultaneous downloads; currently unimplemented MaxDownloads: 0 @@ -67,4 +80,4 @@ IgnoreFiles: - '^@' # Ignore all files starting with "@" # Enable service announcement on local network with Bonjour -EnableBonjour: false \ No newline at end of file +EnableBonjour: false diff --git a/docs/feed-backed-news.md b/docs/feed-backed-news.md new file mode 100644 index 0000000..8edcf79 --- /dev/null +++ b/docs/feed-backed-news.md @@ -0,0 +1,129 @@ +# Feed-backed threaded news + +Mobius can import a public RSS or Atom feed into an existing Hotline news +category. This is useful for project announcements, Sparkle appcasts, and +GitHub release feeds. + +When a user opens a mapped category, Mobius checks its source and adds entries +it has not seen before as ordinary root articles. Imported articles live in +`ThreadedNews.yaml`, alongside locally posted articles and replies. The category +continues to work as ordinary Hotline news: authorized users can post roots, +reply, and delete any article. + +## Configure a category + +First, create the target as an ordinary news category with a Hotline client. +Every component in `CategoryPath`, including the final category, must already +exist before the server starts. + +Then add a mapping to `config.yaml` and restart Mobius: + +```yaml +NewsFeeds: + - CategoryPath: ["Software Updates", "Afterglow"] + URL: "https://morphing.cloud/afterglow/appcast.xml" + + - CategoryPath: ["Software Updates", "Mobius"] + URL: "https://github.com/jhalter/mobius/releases.atom" +``` + +`CategoryPath` is the complete path from the root of threaded news to the +existing category. `URL` must be an absolute public `http` or `https` URL and +cannot contain embedded credentials. Only one feed may map to a category. + +Mobius detects RSS and Atom automatically. Sparkle appcasts are RSS; GitHub's +`releases.atom` endpoints are Atom. JSON Feed and provider-specific APIs are not +supported. + +## Import behavior + +Every article-list request for the mapped category performs a source check. +Mobius sends the saved `ETag` and `Last-Modified` values when the source +provides them, allowing an unchanged source to answer with a small `304 Not +Modified` response. Concurrent requests for the same category share one +in-flight check. + +On a successful response, Mobius: + +1. Finds entries not previously observed from that URL. +2. Converts HTML descriptions to plain text and retains useful source, release + notes, and enclosure links. +3. Imports new entries oldest-first as ordinary root articles. +4. Saves the articles and import metadata together in one atomic update to + `ThreadedNews.yaml`. + +An entry needs a stable identity: a GUID or Atom ID, an HTTP(S) article link, +or an HTTP(S) enclosure URL, in that order. Entries without any of these are +skipped and logged. Mobius intentionally does not derive identities from titles +or bodies because an edited entry could then be mistaken for a new article. + +Once an identity has been imported, later changes to that entry's title, +author, date, or body are ignored. If an imported article is deleted locally, +its identity remains in the seen set, so the feed does not recreate it. Items +also remain in Hotline when they disappear from the source. This prevents a +feed's rolling window from rolling articles out of the Hotline category. + +The initial import can only include entries returned by the source. Mobius does +not paginate GitHub or reconstruct older releases that are already absent from +its Atom feed. + +## Failures and limits + +A network error, timeout, non-success HTTP response, invalid feed, or failed +disk write is logged. Mobius still returns the category's current local article +list, including an empty list before the first successful import. A later +category load tries the source again. Requests time out after 10 seconds and a +response body may be at most 2 MiB. + +The Hotline protocol limits the complete encoded article-list payload for one +category to 65,535 bytes. This listing contains article metadata—not article +bodies—so the practical capacity depends mostly on the number and encoded +length of titles and authors. Mobius checks the full list after each candidate +article. It saves the oldest prefix that fits and stops before an addition +would exceed the limit. It does not save new HTTP validators while unseen +entries remain, so the next category load fetches and retries them. Mobius does +not prune local or imported articles automatically; split a large source across +categories when a category approaches this protocol limit. + +Individual imported titles and authors are limited to 255 encoded bytes, and +article bodies to 65,535 encoded bytes, matching Hotline's field sizes. + +## Durable state and backups + +Feed history is part of each category in `ThreadedNews.yaml` under a YAML-only +`FeedState` key. It records the current source URL, HTTP validators, and hashes +of identities that have already been imported. This metadata is not sent to +Hotline clients. Back up `ThreadedNews.yaml` as usual; there are no additional +feed state or cache files. + +Changing a category's URL clears its HTTP validators but retains its seen +history. Identity hashes include the source URL, so the new source can import +its entries even if it uses the same GUID values. Switching back to a previous +URL does not duplicate entries that URL imported earlier. + +The earlier experimental `FeedNewsState.yaml` and `FeedNewsCache.json` formats +are not migrated or read. If they exist from a development build, Mobius +ignores them; their contents do not participate in this feature. + +## Text encoding + +Feed parsers produce UTF-8 text. Mobius converts each entry once, when it is +imported, according to the server-wide `Encoding` setting. The default +`macintosh` setting converts to Mac Roman and replaces unsupported characters; +`utf8` stores UTF-8 unchanged. Existing imported articles are ordinary raw-byte +news data, so changing `Encoding` affects only future imports and does not +rewrite history. + +## Troubleshooting + +- If startup fails, verify that every mapped path already exists and ends at a + news category rather than a bundle. Also check for duplicate paths and + invalid or credential-bearing URLs. +- If new entries do not appear, open the category and inspect the server log. + Confirm that the URL is public RSS or Atom and that entries have stable IDs, + links, or enclosures. +- If old releases are missing on the first import, inspect the source feed. Its + published window is the initial cutoff; Mobius does not use pagination or a + provider API. +- If a source is unavailable, existing local news remains usable and the next + category load retries it. diff --git a/docs/text-encoding.md b/docs/text-encoding.md index 71bdada..f299b51 100644 --- a/docs/text-encoding.md +++ b/docs/text-encoding.md @@ -2,7 +2,7 @@ ## Background -The Hotline protocol was designed for classic Mac OS, which used an encoding called Mac Roman for text. Modern operating systems use UTF-8 instead. Mobius automatically translates between these two encodings so that classic Hotline clients and modern filesystems can work together. +The Hotline protocol was designed for classic Mac OS, which used an encoding called Mac Roman for text. Modern operating systems use UTF-8 instead. Mobius translates filesystem names and newly imported feed news so that classic Hotline clients and modern UTF-8 sources can work together. Most other protocol text remains raw bytes. By default, Mobius assumes all clients use Mac Roman encoding. If your server exclusively serves modern UTF-8 clients, you can disable the Mac Roman conversion. This document explains how encoding works and how to configure it. @@ -17,9 +17,20 @@ When a Hotline client uploads, downloads, browses, creates, or renames files and This means files on disk always use UTF-8 names, regardless of what encoding the client uses. You can place files with Unicode names in the server's file directory and clients will see them — as long as the characters have Mac Roman equivalents. -### Chat, news, and usernames are NOT translated +### Server-generated feed news is translated -Text in chat messages, news articles, usernames, and private messages is passed through as raw bytes with no encoding conversion. This means: +RSS and Atom sources are UTF-8 server data rather than client-authored Hotline +text. Mobius encodes each imported article's title, author, and body when it is +first added to `ThreadedNews.yaml`. With the default `macintosh` setting, +characters that Mac Roman cannot represent are replaced. With `utf8`, feed text +is stored unchanged. + +Imported entries become ordinary raw-byte news articles. Changing `Encoding` +therefore affects future imports only; it does not rewrite existing articles. + +### Chat, user-authored news, and usernames are NOT translated + +Text in chat messages, existing news articles, usernames, and private messages is passed through as raw bytes with no encoding conversion. This means: - If all your users are on classic Mac clients, they'll see each other's text correctly (all Mac Roman). - If all your users are on modern UTF-8 clients, they'll also see each other's text correctly. @@ -29,13 +40,13 @@ There is no way to configure this behavior — the Hotline protocol has no mecha ## Configuration -The `Encoding` field in `config.yaml` controls how file and folder names are translated: +The `Encoding` field in `config.yaml` controls how file and folder names and server-generated feed news are translated: ```yaml # Default — translates between Mac Roman and UTF-8 (compatible with classic Hotline clients) Encoding: macintosh -# No-op — passes file names through without conversion (for servers with only modern UTF-8 clients) +# No-op — passes file names and feed news through unchanged (modern UTF-8 clients only) Encoding: utf8 ``` @@ -59,12 +70,13 @@ Modern Hotline clients that send UTF-8 for file operations may produce unexpecte ## Summary -| What | Encoding translation? | Notes | -|---------------------------|----------------------|-----------------------------------------------| -| File and folder names | Yes | Mac Roman <-> UTF-8 at the filesystem boundary | -| Chat messages | No | Raw bytes, passed through as-is | -| News articles and titles | No | Raw bytes, passed through as-is | -| Usernames | No | Raw bytes, passed through as-is | -| Private messages | No | Raw bytes, passed through as-is | -| File comments | No | Stored and retrieved as raw bytes | -| Login credentials | No | Obfuscated, no charset conversion | +| What | Encoding translation? | Notes | +| --- | --- | --- | +| File and folder names | Yes | Mac Roman <-> UTF-8 at the filesystem boundary | +| Newly imported feed articles | Yes | UTF-8 source text -> configured encoding at import time | +| Chat messages | No | Raw bytes, passed through as-is | +| Existing and user-authored news articles | No | Raw bytes, passed through as-is | +| Usernames | No | Raw bytes, passed through as-is | +| Private messages | No | Raw bytes, passed through as-is | +| File comments | No | Stored and retrieved as raw bytes | +| Login credentials | No | Obfuscated, no charset conversion | diff --git a/go.mod b/go.mod index b44350f..2905fa4 100644 --- a/go.mod +++ b/go.mod @@ -11,11 +11,14 @@ require ( github.com/aws/aws-sdk-go-v2/service/s3 v1.105.0 github.com/aws/smithy-go v1.27.3 github.com/go-playground/validator/v10 v10.28.0 + github.com/mmcdole/gofeed v1.4.0 github.com/oleksandr/bonjour v0.0.0-20210301155756-30f43c61b915 github.com/redis/go-redis/v9 v9.17.0 github.com/stretchr/testify v1.11.1 - golang.org/x/crypto v0.45.0 - golang.org/x/text v0.31.0 + golang.org/x/crypto v0.53.0 + golang.org/x/net v0.56.0 + golang.org/x/sync v0.21.0 + golang.org/x/text v0.38.0 golang.org/x/time v0.14.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v3 v3.0.1 @@ -43,12 +46,11 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/miekg/dns v1.1.68 // indirect + github.com/mmcdole/goxpp/v2 v2.0.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/stretchr/objx v0.5.3 // indirect github.com/yuin/gopher-lua v1.1.1 // indirect - golang.org/x/mod v0.30.0 // indirect - golang.org/x/net v0.47.0 // indirect - golang.org/x/sync v0.18.0 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/tools v0.39.0 // indirect + golang.org/x/mod v0.36.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/tools v0.45.0 // indirect ) diff --git a/go.sum b/go.sum index 57ca4a9..2db280f 100644 --- a/go.sum +++ b/go.sum @@ -64,6 +64,10 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/miekg/dns v1.1.68 h1:jsSRkNozw7G/mnmXULynzMNIsgY2dHC8LO6U6Ij2JEA= github.com/miekg/dns v1.1.68/go.mod h1:fujopn7TB3Pu3JM69XaawiU0wqjpL9/8xGop5UrTPps= +github.com/mmcdole/gofeed v1.4.0 h1:+efDmI/yJXJgTfa8we5zg9GAKsU+2d7tnpt9QZwvjLQ= +github.com/mmcdole/gofeed v1.4.0/go.mod h1:ngV5MTB7UJko6fH3/fG5AkB/ABUGK1ZTePF9iRhzu/c= +github.com/mmcdole/goxpp/v2 v2.0.0 h1:HrSCflxerUEqZQNq3u7ldtmE/XkwnTx4Zpq2DW4i5rQ= +github.com/mmcdole/goxpp/v2 v2.0.0/go.mod h1:CUduYMnO9JB6Z/uqDn9Ormk/r8E9BsLQxHPWDZ961Os= github.com/oleksandr/bonjour v0.0.0-20210301155756-30f43c61b915 h1:d291KOLbN1GthTPA1fLKyWdclX3k1ZP+CzYtun+a5Es= github.com/oleksandr/bonjour v0.0.0-20210301155756-30f43c61b915/go.mod h1:MGuVJ1+5TX1SCoO2Sx0eAnjpdRytYla2uC1YIZfkC9c= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -76,22 +80,22 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= -golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= -golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= -golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= -golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= +golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= -golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= -golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= diff --git a/hotline/config.go b/hotline/config.go index 8b6adff..b3fdcb8 100644 --- a/hotline/config.go +++ b/hotline/config.go @@ -1,19 +1,44 @@ package hotline +import ( + "fmt" + + "gopkg.in/yaml.v3" +) + +// NewsFeedConfig maps an existing threaded-news category to an RSS or Atom +// feed. Feed entries are imported as ordinary articles in ThreadedNews.yaml. +type NewsFeedConfig struct { + CategoryPath []string `yaml:"CategoryPath" validate:"required,min=1,dive,required"` + URL string `yaml:"URL" validate:"required"` +} + +func (c *NewsFeedConfig) UnmarshalYAML(value *yaml.Node) error { + for i := 0; i+1 < len(value.Content); i += 2 { + key := value.Content[i].Value + if key != "CategoryPath" && key != "URL" { + return fmt.Errorf("unknown NewsFeeds setting %q", key) + } + } + type plainNewsFeedConfig NewsFeedConfig + return value.Decode((*plainNewsFeedConfig)(c)) +} + type Config struct { - Name string `yaml:"Name" validate:"required,max=50"` // Name used for Tracker registration - Description string `yaml:"Description" validate:"required,max=200"` // Description used for Tracker registration - BannerFile string `yaml:"BannerFile" validate:"omitempty,bannerext"` // Path to Banner jpg or gif - FileRoot string `yaml:"FileRoot" validate:"required"` // Path to Files - EnableTrackerRegistration bool `yaml:"EnableTrackerRegistration"` // Toggle Tracker Registration - Trackers []string `yaml:"Trackers" validate:"dive,hostname_port"` // List of trackers that the server should register with - NewsDelimiter string `yaml:"NewsDelimiter"` // String used to separate news posts - NewsDateFormat string `yaml:"NewsDateFormat"` // Go template string to customize news date format - MaxDownloads int `yaml:"MaxDownloads"` // Global simultaneous download limit - MaxDownloadsPerClient int `yaml:"MaxDownloadsPerClient"` // Per client simultaneous download limit - MaxConnectionsPerIP int `yaml:"MaxConnectionsPerIP"` // Max connections per IP - PreserveResourceForks bool `yaml:"PreserveResourceForks"` // Enable preservation of file info and resource forks in sidecar files - IgnoreFiles []string `yaml:"IgnoreFiles"` // List of regular expression for filtering files from the file list - EnableBonjour bool `yaml:"EnableBonjour"` // Enable service announcement on local network with Bonjour - Encoding string `yaml:"Encoding" validate:"omitempty,oneof=macintosh utf8"` // Text encoding for client communication + Name string `yaml:"Name" validate:"required,max=50"` // Name used for Tracker registration + Description string `yaml:"Description" validate:"required,max=200"` // Description used for Tracker registration + BannerFile string `yaml:"BannerFile" validate:"omitempty,bannerext"` // Path to Banner jpg or gif + FileRoot string `yaml:"FileRoot" validate:"required"` // Path to Files + EnableTrackerRegistration bool `yaml:"EnableTrackerRegistration"` // Toggle Tracker Registration + Trackers []string `yaml:"Trackers" validate:"dive,hostname_port"` // List of trackers that the server should register with + NewsDelimiter string `yaml:"NewsDelimiter"` // String used to separate news posts + NewsDateFormat string `yaml:"NewsDateFormat"` // Go template string to customize news date format + MaxDownloads int `yaml:"MaxDownloads"` // Global simultaneous download limit + MaxDownloadsPerClient int `yaml:"MaxDownloadsPerClient"` // Per client simultaneous download limit + MaxConnectionsPerIP int `yaml:"MaxConnectionsPerIP"` // Max connections per IP + PreserveResourceForks bool `yaml:"PreserveResourceForks"` // Enable preservation of file info and resource forks in sidecar files + IgnoreFiles []string `yaml:"IgnoreFiles"` // List of regular expression for filtering files from the file list + EnableBonjour bool `yaml:"EnableBonjour"` // Enable service announcement on local network with Bonjour + Encoding string `yaml:"Encoding" validate:"omitempty,oneof=macintosh utf8"` // Encoding for filesystem names and server-generated feed news + NewsFeeds []NewsFeedConfig `yaml:"NewsFeeds" validate:"dive"` // Existing threaded-news categories populated from remote feeds } diff --git a/hotline/news.go b/hotline/news.go index 033badb..b0e511b 100644 --- a/hotline/news.go +++ b/hotline/news.go @@ -28,14 +28,25 @@ type ThreadedNews struct { Categories map[string]NewsCategoryListData15 `yaml:"Categories"` } +// NewsFeedCategoryState is durable import metadata for a feed-mapped category. +// It is YAML-only: NewsCategoryListData15.Read explicitly serializes the +// Hotline protocol fields and therefore never exposes this state to clients. +type NewsFeedCategoryState struct { + SourceURL string `yaml:"SourceURL"` + ETag string `yaml:"ETag,omitempty"` + LastModified string `yaml:"LastModified,omitempty"` + Imported map[string]uint32 `yaml:"Imported"` +} + type NewsCategoryListData15 struct { - Type [2]byte `yaml:"Type,flow"` // Bundle (2) or category (3) - Name string `yaml:"Name"` - Articles map[uint32]*NewsArtData `yaml:"Articles"` // Optional, if Type is Category - SubCats map[string]NewsCategoryListData15 `yaml:"SubCats"` - GUID [16]byte `yaml:"-"` // What does this do? Undocumented and seeming unused. - AddSN [4]byte `yaml:"-"` // What does this do? Undocumented and seeming unused. - DeleteSN [4]byte `yaml:"-"` // What does this do? Undocumented and seeming unused. + Type [2]byte `yaml:"Type,flow"` // Bundle (2) or category (3) + Name string `yaml:"Name"` + Articles map[uint32]*NewsArtData `yaml:"Articles"` // Optional, if Type is Category + SubCats map[string]NewsCategoryListData15 `yaml:"SubCats"` + FeedState *NewsFeedCategoryState `yaml:"FeedState,omitempty"` + GUID [16]byte `yaml:"-"` // What does this do? Undocumented and seeming unused. + AddSN [4]byte `yaml:"-"` // What does this do? Undocumented and seeming unused. + DeleteSN [4]byte `yaml:"-"` // What does this do? Undocumented and seeming unused. readOffset int // Internal offset to track read progress writeOffset int // Internal offset to track write progress diff --git a/hotline/time.go b/hotline/time.go index b6d2d9c..47972d4 100644 --- a/hotline/time.go +++ b/hotline/time.go @@ -36,6 +36,22 @@ func NewTime(t time.Time) (b Time) { )) } +// NewNewsTime converts a time.Time to the year-relative timestamp encoding +// used by threaded-news article lists. Classic Hotline clients combine the +// leading year field with seconds elapsed since January 1 of that year. +// +// This intentionally differs from NewTime's Mac-epoch encoding, which is +// needed for compatibility with clients such as Pitbull Pro in other protocol +// fields. +func NewNewsTime(t time.Time) (b Time) { + localTime := t.In(time.Local) + startOfYear := time.Date(localTime.Year(), time.January, 1, 0, 0, 0, 0, time.Local) + + binary.BigEndian.PutUint16(b[0:2], uint16(localTime.Year())) + binary.BigEndian.PutUint32(b[4:8], uint32(localTime.Sub(startOfYear).Seconds())) + return b +} + // Time converts the Hotline Time format to a Go time.Time. // The Hotline format stores: Year (2 bytes) + milliseconds (2 bytes, unused) + // seconds since the Mac OS epoch (4 bytes). @@ -45,6 +61,15 @@ func (t Time) Time() time.Time { return macEpoch.Add(time.Duration(seconds) * time.Second) } +// NewsTime decodes the year-relative timestamp format produced by +// NewNewsTime. +func (t Time) NewsTime() time.Time { + year := binary.BigEndian.Uint16(t[0:2]) + seconds := binary.BigEndian.Uint32(t[4:8]) + startOfYear := time.Date(int(year), time.January, 1, 0, 0, 0, 0, time.Local) + return startOfYear.Add(time.Duration(seconds) * time.Second) +} + // Format returns the time formatted according to the layout string. // This is a convenience wrapper around Time().Format(). func (t Time) Format(layout string) string { diff --git a/hotline/time_test.go b/hotline/time_test.go index b52f3ef..f9ee5c4 100644 --- a/hotline/time_test.go +++ b/hotline/time_test.go @@ -47,3 +47,16 @@ func TestNewTime_SecondsSinceMacEpoch(t *testing.T) { t.Errorf("Mac-epoch decode = %v, want %v", decoded, tm) } } + +func TestNewNewsTime_UsesSecondsSinceStartOfYear(t *testing.T) { + want := time.Date(2026, time.August, 22, 18, 9, 55, 0, time.Local) + + encoded := NewNewsTime(want) + seconds := binary.BigEndian.Uint32(encoded[4:8]) + if seconds >= 366*24*60*60 { + t.Fatalf("news seconds field = %d, want a year-relative value", seconds) + } + if got := encoded.NewsTime(); !got.Equal(want) { + t.Errorf("news time round trip = %v, want %v", got, want) + } +} diff --git a/internal/mobius/config.go b/internal/mobius/config.go index 171f6c7..a0924df 100644 --- a/internal/mobius/config.go +++ b/internal/mobius/config.go @@ -2,9 +2,11 @@ package mobius import ( "fmt" + "net/url" "os" "path/filepath" "strings" + "unicode/utf8" "github.com/go-playground/validator/v10" "github.com/jhalter/mobius/hotline" @@ -29,6 +31,10 @@ func LoadConfig(path string) (*hotline.Config, error) { return nil, fmt.Errorf("unmarshal YAML: %v", err) } + if err := normalizeNewsFeedConfig(&config); err != nil { + return nil, err + } + validate := validator.New() if err = validate.RegisterValidation("bannerext", func(fl validator.FieldLevel) bool { filename := fl.Field().String() @@ -56,3 +62,34 @@ func LoadConfig(path string) (*hotline.Config, error) { // only the caller knows how to resolve it (e.g. against the config dir for the OS backend). return &config, nil } + +func normalizeNewsFeedConfig(config *hotline.Config) error { + seenPaths := make(map[string]struct{}, len(config.NewsFeeds)) + for i := range config.NewsFeeds { + feed := &config.NewsFeeds[i] + if len(feed.CategoryPath) == 0 { + return fmt.Errorf("NewsFeeds[%d].CategoryPath is required", i) + } + for j, segment := range feed.CategoryPath { + if segment == "" || !utf8.ValidString(segment) || strings.ContainsRune(segment, '\x00') || len(segment) > 255 { + return fmt.Errorf("NewsFeeds[%d].CategoryPath[%d] must be non-empty valid UTF-8 without NUL bytes and at most 255 bytes", i, j) + } + } + + pathKey := strings.Join(feed.CategoryPath, "\x00") + if _, exists := seenPaths[pathKey]; exists { + return fmt.Errorf("NewsFeeds[%d].CategoryPath duplicates another news feed", i) + } + seenPaths[pathKey] = struct{}{} + + parsedURL, err := url.Parse(feed.URL) + if err != nil || parsedURL.Host == "" || (parsedURL.Scheme != "http" && parsedURL.Scheme != "https") { + return fmt.Errorf("NewsFeeds[%d].URL must be an absolute HTTP or HTTPS URL", i) + } + if parsedURL.User != nil { + return fmt.Errorf("NewsFeeds[%d].URL must not contain credentials", i) + } + } + + return nil +} diff --git a/internal/mobius/config_test.go b/internal/mobius/config_test.go index f090b5c..9985745 100644 --- a/internal/mobius/config_test.go +++ b/internal/mobius/config_test.go @@ -5,6 +5,9 @@ import ( "path/filepath" "strings" "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestLoadConfig_InvalidBannerFileExtension(t *testing.T) { @@ -124,3 +127,91 @@ FileRoot: "files" }) } } + +func TestLoadConfig_NewsFeeds(t *testing.T) { + t.Run("accepts an auto-detected RSS or Atom feed", func(t *testing.T) { + dir := t.TempDir() + configPath := filepath.Join(dir, "config.yaml") + configContent := ` +Name: Test Server +Description: Test Description +FileRoot: Files +NewsFeeds: + - CategoryPath: [Afterglow Releases] + URL: https://morphing.cloud/afterglow/appcast.xml +` + require.NoError(t, os.WriteFile(configPath, []byte(configContent), 0644)) + + config, err := LoadConfig(configPath) + require.NoError(t, err) + require.Len(t, config.NewsFeeds, 1) + assert.Equal(t, []string{"Afterglow Releases"}, config.NewsFeeds[0].CategoryPath) + assert.Equal(t, "https://morphing.cloud/afterglow/appcast.xml", config.NewsFeeds[0].URL) + }) + + tests := []struct { + name string + feedYAML string + wantErr string + }{ + { + name: "rejects non HTTP URL", + feedYAML: ` + - CategoryPath: [News] + URL: file:///tmp/feed.xml`, + wantErr: "absolute HTTP or HTTPS URL", + }, + { + name: "rejects credentials", + feedYAML: ` + - CategoryPath: [News] + URL: https://user:password@example.com/feed.xml`, + wantErr: "must not contain credentials", + }, + { + name: "rejects duplicate category", + feedYAML: ` + - CategoryPath: [News] + URL: https://example.com/one.xml + - CategoryPath: [News] + URL: https://example.com/two.xml`, + wantErr: "duplicates another news feed", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + configPath := filepath.Join(dir, "config.yaml") + configContent := ` +Name: Test Server +Description: Test Description +FileRoot: Files +NewsFeeds:` + tt.feedYAML + "\n" + require.NoError(t, os.WriteFile(configPath, []byte(configContent), 0644)) + + _, err := LoadConfig(configPath) + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + }) + } + + for _, removed := range []string{"Format: rss", "RefreshInterval: 1h", "MaxItems: 25"} { + t.Run("rejects removed setting "+removed, func(t *testing.T) { + dir := t.TempDir() + configPath := filepath.Join(dir, "config.yaml") + configContent := ` +Name: Test Server +Description: Test Description +FileRoot: Files +NewsFeeds: + - CategoryPath: [Releases] + URL: http://example.com/releases.xml + ` + removed + "\n" + require.NoError(t, os.WriteFile(configPath, []byte(configContent), 0644)) + + _, err := LoadConfig(configPath) + require.ErrorContains(t, err, "unknown NewsFeeds setting") + }) + } +} diff --git a/internal/mobius/feed_news.go b/internal/mobius/feed_news.go new file mode 100644 index 0000000..fcc0664 --- /dev/null +++ b/internal/mobius/feed_news.go @@ -0,0 +1,217 @@ +package mobius + +import ( + "context" + "errors" + "fmt" + "log/slog" + "slices" + "strings" + + "github.com/jhalter/mobius/hotline" + "golang.org/x/sync/singleflight" + textencoding "golang.org/x/text/encoding" + "golang.org/x/text/encoding/charmap" +) + +// FeedNewsManager refreshes configured feeds when their existing category is +// opened. Everything except ListArticles remains ordinary threaded-news +// behavior because imported entries are ordinary ThreadedNews.yaml articles. +type FeedNewsManager struct { + base *ThreadedNewsYAML + feeds map[string]configuredNewsFeed + logger *slog.Logger + fetcher *feedFetcher + encoding string + refresh singleflight.Group +} + +type configuredNewsFeed struct { + config hotline.NewsFeedConfig + wirePath []string +} + +var _ hotline.ThreadedNewsMgr = (*FeedNewsManager)(nil) + +func NewFeedNewsManager( + base *ThreadedNewsYAML, + feeds []hotline.NewsFeedConfig, + logger *slog.Logger, + userAgent string, + clientEncoding string, +) (*FeedNewsManager, error) { + if base == nil { + return nil, errors.New("feed news manager requires a threaded news YAML manager") + } + if logger == nil { + logger = slog.Default() + } + + normalized := &hotline.Config{NewsFeeds: slices.Clone(feeds)} + if err := normalizeNewsFeedConfig(normalized); err != nil { + return nil, err + } + + manager := &FeedNewsManager{ + base: base, + feeds: make(map[string]configuredNewsFeed, len(feeds)), + logger: logger, + fetcher: newFeedFetcher(userAgent), + encoding: normalizedFeedEncodingName(clientEncoding), + } + for _, config := range normalized.NewsFeeds { + wirePath := encodeFeedPath(config.CategoryPath, manager.encoding) + key := newsPathKey(wirePath) + if _, exists := manager.feeds[key]; exists { + return nil, fmt.Errorf("NewsFeeds categories collide after %s encoding", manager.encoding) + } + if err := base.validateFeedCategory(wirePath); err != nil { + return nil, fmt.Errorf("NewsFeeds category %q: %w", strings.Join(config.CategoryPath, "/"), err) + } + manager.feeds[key] = configuredNewsFeed{config: config, wirePath: wirePath} + } + return manager, nil +} + +func (m *FeedNewsManager) ListArticles(newsPath []string) (hotline.NewsArtListData, error) { + feed, mapped := m.feeds[newsPathKey(newsPath)] + if !mapped { + return m.base.ListArticles(newsPath) + } + + _, err, _ := m.refresh.Do(newsPathKey(newsPath), func() (any, error) { + return nil, m.refreshFeed(feed) + }) + if err != nil { + m.logger.Warn("Unable to refresh feed-backed news; serving local articles", + "category", strings.Join(feed.config.CategoryPath, "/"), + "err", err, + ) + } + return m.base.ListArticles(newsPath) +} + +func (m *FeedNewsManager) refreshFeed(feed configuredNewsFeed) error { + validators, err := m.base.feedValidators(feed.wirePath, feed.config.URL) + if err != nil { + return err + } + result, err := m.fetcher.fetch(context.Background(), feed.config.URL, validators) + if err != nil { + return err + } + if result.notModified { + return nil + } + if result.skippedNoIdentity > 0 { + m.logger.Warn("Skipped feed entries without a stable identity", + "category", strings.Join(feed.config.CategoryPath, "/"), + "count", result.skippedNoIdentity, + ) + } + + items := make([]normalizedFeedItem, len(result.items)) + for i, item := range result.items { + items[i] = item + items[i].article = encodeFeedArticle(item.article, m.encoding) + } + imported, err := m.base.importFeedArticles( + feed.wirePath, + feed.config.URL, + result.etag, + result.lastModified, + items, + ) + if err != nil { + return err + } + if imported.limitReached { + m.logger.Warn("Stopped importing feed entries at the Hotline article-list size limit", + "category", strings.Join(feed.config.CategoryPath, "/"), + "imported", imported.count, + ) + } + return nil +} + +func (m *FeedNewsManager) GetArticle(newsPath []string, articleID uint32) *hotline.NewsArtData { + return m.base.GetArticle(newsPath, articleID) +} + +func (m *FeedNewsManager) DeleteArticle(newsPath []string, articleID uint32, recursive bool) error { + return m.base.DeleteArticle(newsPath, articleID, recursive) +} + +func (m *FeedNewsManager) PostArticle(newsPath []string, parentArticleID uint32, article hotline.NewsArtData) error { + return m.base.PostArticle(newsPath, parentArticleID, article) +} + +func (m *FeedNewsManager) CreateGrouping(newsPath []string, name string, itemType [2]byte) error { + return m.base.CreateGrouping(newsPath, name, itemType) +} + +func (m *FeedNewsManager) GetCategories(newsPath []string) []hotline.NewsCategoryListData15 { + return m.base.GetCategories(newsPath) +} + +func (m *FeedNewsManager) NewsItem(newsPath []string) hotline.NewsCategoryListData15 { + return m.base.NewsItem(newsPath) +} + +func (m *FeedNewsManager) DeleteNewsItem(newsPath []string) error { + return m.base.DeleteNewsItem(newsPath) +} + +func newsPathKey(path []string) string { + return strings.Join(path, "\x00") +} + +func normalizedFeedEncodingName(value string) string { + if value == "utf8" { + return "utf8" + } + return "macintosh" +} + +func encodeFeedPath(path []string, clientEncoding string) []string { + encoded := slices.Clone(path) + for i := range encoded { + encoded[i] = encodeFeedText(encoded[i], clientEncoding) + } + return encoded +} + +func encodeFeedArticle(article hotline.NewsArtData, clientEncoding string) hotline.NewsArtData { + article.Title = truncateEncodedFeedText(article.Title, maxNewsStringBytes, clientEncoding) + article.Poster = truncateEncodedFeedText(article.Poster, maxNewsStringBytes, clientEncoding) + article.Data = truncateEncodedFeedText(article.Data, maxNewsBodyBytes, clientEncoding) + article.DataFlav = slices.Clone(hotline.NewsFlavor) + return article +} + +func truncateEncodedFeedText(value string, maxBytes int, clientEncoding string) string { + encoded := encodeFeedText(value, clientEncoding) + if len(encoded) <= maxBytes { + return encoded + } + if normalizedFeedEncodingName(clientEncoding) == "utf8" { + return truncateUTF8(encoded, maxBytes) + } + return encoded[:maxBytes] +} + +func encodeFeedText(value, clientEncoding string) string { + if normalizedFeedEncodingName(clientEncoding) == "utf8" { + return value + } + encoded, err := textencoding.ReplaceUnsupported(charmap.Macintosh.NewEncoder()).String(value) + if err == nil { + return encoded + } + return strings.Map(func(r rune) rune { + if r <= 0x7f { + return r + } + return '?' + }, value) +} diff --git a/internal/mobius/feed_news_fetch.go b/internal/mobius/feed_news_fetch.go new file mode 100644 index 0000000..18d25ee --- /dev/null +++ b/internal/mobius/feed_news_fetch.go @@ -0,0 +1,478 @@ +package mobius + +import ( + "bytes" + "context" + "errors" + "fmt" + stdhtml "html" + "io" + "net/http" + "net/url" + "slices" + "sort" + "strings" + "time" + "unicode/utf8" + + "github.com/jhalter/mobius/hotline" + "github.com/mmcdole/gofeed" + xhtml "golang.org/x/net/html" +) + +const ( + feedRequestTimeout = 10 * time.Second + maxFeedBodyBytes = 2 << 20 + maxNewsStringBytes = 255 + maxNewsBodyBytes = 65535 +) + +var errFeedResponseTooLarge = errors.New("feed response exceeds 2 MiB") + +type normalizedFeedItem struct { + identity string + article hotline.NewsArtData + publishedAt time.Time + sourceIndex int +} + +type feedHTTPValidators struct { + etag string + lastModified string +} + +type feedFetchResult struct { + notModified bool + etag string + lastModified string + items []normalizedFeedItem + skippedNoIdentity int +} + +type feedFetcher struct { + client *http.Client + parser *gofeed.Parser + now func() time.Time + userAgent string +} + +func newFeedFetcher(userAgent string) *feedFetcher { + if userAgent == "" { + userAgent = "mobius-hotline-server" + } + return &feedFetcher{ + client: &http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) >= 10 { + return errors.New("stopped after 10 redirects") + } + if req.URL.Scheme != "http" && req.URL.Scheme != "https" { + return fmt.Errorf("redirected to unsupported URL scheme %q", req.URL.Scheme) + } + if req.URL.User != nil { + return errors.New("redirected to a URL containing credentials") + } + return nil + }, + }, + parser: gofeed.NewParser(), + now: time.Now, + userAgent: userAgent, + } +} + +func (f *feedFetcher) fetch(ctx context.Context, sourceURL string, validators feedHTTPValidators) (feedFetchResult, error) { + ctx, cancel := context.WithTimeout(ctx, feedRequestTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, sourceURL, nil) + if err != nil { + return feedFetchResult{}, errors.New("create request from configured feed URL") + } + req.Header.Set("Accept", "application/rss+xml, application/atom+xml, application/xml;q=0.9, text/xml;q=0.8") + req.Header.Set("User-Agent", f.userAgent) + if validators.etag != "" { + req.Header.Set("If-None-Match", validators.etag) + } + if validators.lastModified != "" { + req.Header.Set("If-Modified-Since", validators.lastModified) + } + + response, err := f.client.Do(req) + if err != nil { + return feedFetchResult{}, fmt.Errorf("fetch feed: %w", redactHTTPError(err)) + } + defer func() { _ = response.Body.Close() }() + + if response.StatusCode == http.StatusNotModified { + if validators.etag == "" && validators.lastModified == "" { + return feedFetchResult{}, errors.New("feed returned 304 without a conditional request") + } + return feedFetchResult{notModified: true}, nil + } + if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices { + return feedFetchResult{}, fmt.Errorf("feed returned %s", response.Status) + } + + body, err := io.ReadAll(io.LimitReader(response.Body, maxFeedBodyBytes+1)) + if err != nil { + return feedFetchResult{}, fmt.Errorf("read feed response: %w", err) + } + if len(body) > maxFeedBodyBytes { + return feedFetchResult{}, errFeedResponseTooLarge + } + items, skipped, err := f.parseFeedResponse(body, f.now()) + if err != nil { + return feedFetchResult{}, err + } + + return feedFetchResult{ + etag: response.Header.Get("ETag"), + lastModified: response.Header.Get("Last-Modified"), + items: items, + skippedNoIdentity: skipped, + }, nil +} + +func (f *feedFetcher) parseFeedResponse(body []byte, fetchedAt time.Time) ([]normalizedFeedItem, int, error) { + detected := gofeed.DetectFeedType(bytes.NewReader(body)) + if detected != gofeed.FeedTypeRSS && detected != gofeed.FeedTypeAtom { + return nil, 0, fmt.Errorf("unsupported feed format %q; expected RSS or Atom", detected) + } + feed, err := f.parser.Parse(bytes.NewReader(body)) + if err != nil { + return nil, 0, fmt.Errorf("parse feed type %v: %w", detected, err) + } + items, skipped := normalizeSyndicationFeed(feed, fetchedAt) + return items, skipped, nil +} + +func redactHTTPError(err error) error { + for { + var urlError *url.Error + if !errors.As(err, &urlError) || urlError.Err == nil || urlError.Err == err { + return err + } + err = urlError.Err + } +} + +func normalizeSyndicationFeed(feed *gofeed.Feed, fetchedAt time.Time) ([]normalizedFeedItem, int) { + items := make([]normalizedFeedItem, 0, len(feed.Items)) + seen := make(map[string]struct{}, len(feed.Items)) + skipped := 0 + for sourceIndex, item := range feed.Items { + if item == nil { + continue + } + identity, ok := feedItemIdentity(item) + if !ok { + skipped++ + continue + } + if _, exists := seen[identity]; exists { + continue + } + seen[identity] = struct{}{} + + publishedAt := fetchedAt + if item.PublishedParsed != nil { + publishedAt = *item.PublishedParsed + } else if item.UpdatedParsed != nil { + publishedAt = *item.UpdatedParsed + } + rawBody := firstNonEmpty(strings.TrimSpace(item.Content), strings.TrimSpace(item.Description)) + title := strings.TrimSpace(item.Title) + if title == "" { + title = "(untitled)" + } + poster := feedItemPoster(item, feed, "News Feed") + items = append(items, normalizedFeedItem{ + identity: identity, + article: hotline.NewsArtData{ + Title: title, + Poster: poster, + Date: hotline.NewNewsTime(publishedAt), + DataFlav: slices.Clone(hotline.NewsFlavor), + Data: feedArticleBody(item, rawBody), + }, + publishedAt: publishedAt, + sourceIndex: sourceIndex, + }) + } + + // Most feeds are newest-first. Dates establish the real order; reversing + // source order resolves equal or missing dates so newer items receive newer + // ordinary Hotline article IDs. + sort.SliceStable(items, func(i, j int) bool { + if items[i].publishedAt.Equal(items[j].publishedAt) { + return items[i].sourceIndex > items[j].sourceIndex + } + return items[i].publishedAt.Before(items[j].publishedAt) + }) + return items, skipped +} + +func feedItemIdentity(item *gofeed.Item) (string, bool) { + if value := strings.TrimSpace(item.GUID); value != "" { + return "id:" + value, true + } + if value := firstHTTPURL(append([]string{item.Link}, item.Links...)...); value != "" { + return "link:" + value, true + } + for _, enclosure := range item.Enclosures { + if enclosure != nil { + if value := safeHTTPURL(enclosure.URL); value != "" { + return "enclosure:" + value, true + } + } + } + return "", false +} + +func feedItemPoster(item *gofeed.Item, feed *gofeed.Feed, fallback string) string { + for _, author := range item.Authors { + if author != nil && strings.TrimSpace(author.Name) != "" { + return strings.TrimSpace(author.Name) + } + } + if item.Author != nil && strings.TrimSpace(item.Author.Name) != "" { + return strings.TrimSpace(item.Author.Name) + } + for _, author := range feed.Authors { + if author != nil && strings.TrimSpace(author.Name) != "" { + return strings.TrimSpace(author.Name) + } + } + if feed.Author != nil && strings.TrimSpace(feed.Author.Name) != "" { + return strings.TrimSpace(feed.Author.Name) + } + if strings.TrimSpace(feed.Title) != "" { + return strings.TrimSpace(feed.Title) + } + return fallback +} + +func feedArticleBody(item *gofeed.Item, rawBody string) string { + body := markupToPlainText(rawBody) + links := make([]string, 0, len(item.Enclosures)+2) + if source := firstHTTPURL(append([]string{item.Link}, item.Links...)...); source != "" { + links = append(links, "Source: "+source) + } + if releaseNotes := extensionValue(item, "sparkle", "releaseNotesLink"); releaseNotes != "" { + if releaseNotes = safeHTTPURL(releaseNotes); releaseNotes != "" { + links = appendUnique(links, "Release notes: "+releaseNotes) + } + } + for _, enclosure := range item.Enclosures { + if enclosure == nil { + continue + } + if download := safeHTTPURL(enclosure.URL); download != "" { + links = appendUnique(links, "Download: "+download) + } + } + if len(links) > 0 { + if body != "" { + body += "\r\r" + } + body += strings.Join(links, "\r") + } + return body +} + +func extensionValue(item *gofeed.Item, namespace, name string) string { + if item.Extensions == nil || item.Extensions[namespace] == nil { + return "" + } + values := item.Extensions[namespace][name] + if len(values) == 0 { + return "" + } + return strings.TrimSpace(values[0].Value) +} + +func markupToPlainText(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "" + } + if !strings.Contains(value, "<") { + return normalizePlainText(value) + } + + tokenizer := xhtml.NewTokenizer(strings.NewReader(value)) + var out strings.Builder + var anchors []string + skipDepth := 0 + for { + tokenType := tokenizer.Next() + switch tokenType { + case xhtml.ErrorToken: + return normalizePlainText(out.String()) + case xhtml.StartTagToken, xhtml.SelfClosingTagToken: + token := tokenizer.Token() + tag := strings.ToLower(token.Data) + if tag == "script" || tag == "style" { + if tokenType == xhtml.StartTagToken { + skipDepth++ + } + continue + } + if skipDepth > 0 { + continue + } + switch tag { + case "br": + out.WriteByte('\n') + case "p", "div", "h1", "h2", "h3", "h4", "h5", "h6", "ul", "ol", "blockquote", "pre": + writeLineBreak(&out, 2) + case "li": + writeLineBreak(&out, 1) + out.WriteString("- ") + case "a": + anchors = append(anchors, safeHTTPURL(attributeValue(token.Attr, "href"))) + } + case xhtml.EndTagToken: + token := tokenizer.Token() + tag := strings.ToLower(token.Data) + if tag == "script" || tag == "style" { + if skipDepth > 0 { + skipDepth-- + } + continue + } + if skipDepth > 0 { + continue + } + if tag == "a" && len(anchors) > 0 { + href := anchors[len(anchors)-1] + anchors = anchors[:len(anchors)-1] + if href != "" { + out.WriteString(" (") + out.WriteString(href) + out.WriteByte(')') + } + } + switch tag { + case "p", "div", "h1", "h2", "h3", "h4", "h5", "h6", "ul", "ol", "blockquote", "pre": + writeLineBreak(&out, 2) + case "li": + writeLineBreak(&out, 1) + } + case xhtml.TextToken: + if skipDepth > 0 { + continue + } + text := strings.Join(strings.Fields(stdhtml.UnescapeString(string(tokenizer.Text()))), " ") + if text == "" { + continue + } + if needsSpace(out.String()) { + out.WriteByte(' ') + } + out.WriteString(text) + } + } +} + +func normalizePlainText(value string) string { + value = strings.ReplaceAll(value, "\r\n", "\n") + value = strings.ReplaceAll(value, "\r", "\n") + lines := strings.Split(value, "\n") + out := make([]string, 0, len(lines)) + blank := false + for _, line := range lines { + line = strings.Join(strings.Fields(line), " ") + if line == "" { + if len(out) > 0 && !blank { + out = append(out, "") + blank = true + } + continue + } + out = append(out, line) + blank = false + } + for len(out) > 0 && out[len(out)-1] == "" { + out = out[:len(out)-1] + } + return strings.Join(out, "\r") +} + +func writeLineBreak(out *strings.Builder, count int) { + value := out.String() + newlines := 0 + for i := len(value) - 1; i >= 0 && value[i] == '\n'; i-- { + newlines++ + } + for newlines < count { + out.WriteByte('\n') + newlines++ + } +} + +func needsSpace(value string) bool { + if value == "" { + return false + } + last := value[len(value)-1] + return last != ' ' && last != '\n' && last != '\t' && last != '-' +} + +func attributeValue(attrs []xhtml.Attribute, name string) string { + for _, attr := range attrs { + if strings.EqualFold(attr.Key, name) { + return strings.TrimSpace(attr.Val) + } + } + return "" +} + +func safeHTTPURL(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "" + } + parsed, err := url.Parse(value) + if err != nil || parsed.Host == "" || (parsed.Scheme != "http" && parsed.Scheme != "https") { + return "" + } + return parsed.String() +} + +func firstHTTPURL(values ...string) string { + for _, value := range values { + if parsed := safeHTTPURL(value); parsed != "" { + return parsed + } + } + return "" +} + +func appendUnique(values []string, value string) []string { + if !slices.Contains(values, value) { + return append(values, value) + } + return values +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if value != "" { + return value + } + } + return "" +} + +func truncateUTF8(value string, maxBytes int) string { + if len(value) <= maxBytes { + return value + } + value = value[:maxBytes] + for !utf8.ValidString(value) { + value = value[:len(value)-1] + } + return value +} diff --git a/internal/mobius/feed_news_import.go b/internal/mobius/feed_news_import.go new file mode 100644 index 0000000..4baacd9 --- /dev/null +++ b/internal/mobius/feed_news_import.go @@ -0,0 +1,238 @@ +package mobius + +import ( + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "io" + "math" + "slices" + + "github.com/jhalter/mobius/hotline" +) + +var errNewsArticleListTooLarge = errors.New("Hotline news article list exceeds 65,535 bytes") + +type feedImportResult struct { + count int + limitReached bool +} + +func (n *ThreadedNewsYAML) validateFeedCategory(newsPath []string) error { + n.mu.Lock() + defer n.mu.Unlock() + _, _, category, ok := n.feedCategoryLocked(newsPath) + if !ok { + return errors.New("target category does not exist") + } + if category.Type != hotline.NewsCategory { + return errors.New("target must be an ordinary news category") + } + return nil +} + +func (n *ThreadedNewsYAML) feedValidators(newsPath []string, sourceURL string) (feedHTTPValidators, error) { + n.mu.Lock() + defer n.mu.Unlock() + _, _, category, ok := n.feedCategoryLocked(newsPath) + if !ok { + return feedHTTPValidators{}, errors.New("feed target category no longer exists") + } + if category.Type != hotline.NewsCategory { + return feedHTTPValidators{}, errors.New("feed target is no longer an ordinary news category") + } + if category.FeedState == nil || category.FeedState.SourceURL != sourceURL { + return feedHTTPValidators{}, nil + } + return feedHTTPValidators{ + etag: category.FeedState.ETag, + lastModified: category.FeedState.LastModified, + }, nil +} + +func (n *ThreadedNewsYAML) importFeedArticles( + newsPath []string, + sourceURL string, + etag string, + lastModified string, + items []normalizedFeedItem, +) (feedImportResult, error) { + n.mu.Lock() + defer n.mu.Unlock() + + parent, name, original, ok := n.feedCategoryLocked(newsPath) + if !ok { + return feedImportResult{}, errors.New("feed target category no longer exists") + } + if original.Type != hotline.NewsCategory { + return feedImportResult{}, errors.New("feed target is no longer an ordinary news category") + } + + working := cloneNewsCategory(original) + if working.Articles == nil { + working.Articles = make(map[uint32]*hotline.NewsArtData) + } + if working.FeedState == nil { + working.FeedState = &hotline.NewsFeedCategoryState{Imported: make(map[string]uint32)} + } + if working.FeedState.Imported == nil { + working.FeedState.Imported = make(map[string]uint32) + } + + result := feedImportResult{} + for _, item := range items { + identityHash := feedIdentityHash(sourceURL, item.identity) + if _, seen := working.FeedState.Imported[identityHash]; seen { + continue + } + + articleID, previousID, err := nextNewsArticleID(working.Articles) + if err != nil { + return feedImportResult{}, err + } + article := item.article + article.PrevArt = [4]byte{} + article.NextArt = [4]byte{} + article.ParentArt = [4]byte{} + article.FirstChildArt = [4]byte{} + article.DataFlav = slices.Clone(hotline.NewsFlavor) + + var oldPreviousNext [4]byte + if previousID != 0 { + oldPreviousNext = working.Articles[previousID].NextArt + binary.BigEndian.PutUint32(article.PrevArt[:], previousID) + binary.BigEndian.PutUint32(working.Articles[previousID].NextArt[:], articleID) + } + working.Articles[articleID] = &article + + if err := validateNewsArticleListSize(working.Articles); err != nil { + delete(working.Articles, articleID) + if previousID != 0 { + working.Articles[previousID].NextArt = oldPreviousNext + } + if errors.Is(err, errNewsArticleListTooLarge) { + result.limitReached = true + break + } + return feedImportResult{}, err + } + + working.FeedState.Imported[identityHash] = articleID + result.count++ + } + + if !result.limitReached { + working.FeedState.SourceURL = sourceURL + working.FeedState.ETag = etag + working.FeedState.LastModified = lastModified + } + + if result.count == 0 && feedStateEqual(original.FeedState, working.FeedState) { + return result, nil + } + parent[name] = working + if err := n.writeFile(); err != nil { + parent[name] = original + return feedImportResult{}, fmt.Errorf("persist imported feed articles: %w", err) + } + return result, nil +} + +func (n *ThreadedNewsYAML) feedCategoryLocked(newsPath []string) (map[string]hotline.NewsCategoryListData15, string, hotline.NewsCategoryListData15, bool) { + if len(newsPath) == 0 { + return nil, "", hotline.NewsCategoryListData15{}, false + } + categories := n.ThreadedNews.Categories + for i, segment := range newsPath { + category, ok := categories[segment] + if !ok { + return nil, "", hotline.NewsCategoryListData15{}, false + } + if i == len(newsPath)-1 { + return categories, segment, category, true + } + categories = category.SubCats + if categories == nil { + return nil, "", hotline.NewsCategoryListData15{}, false + } + } + return nil, "", hotline.NewsCategoryListData15{}, false +} + +func cloneNewsCategory(category hotline.NewsCategoryListData15) hotline.NewsCategoryListData15 { + clone := category + clone.Articles = make(map[uint32]*hotline.NewsArtData, len(category.Articles)) + for id, article := range category.Articles { + if article == nil { + clone.Articles[id] = nil + continue + } + articleClone := *article + articleClone.DataFlav = slices.Clone(article.DataFlav) + clone.Articles[id] = &articleClone + } + clone.SubCats = make(map[string]hotline.NewsCategoryListData15, len(category.SubCats)) + for name, subcategory := range category.SubCats { + clone.SubCats[name] = cloneNewsCategory(subcategory) + } + if category.FeedState != nil { + state := *category.FeedState + state.Imported = make(map[string]uint32, len(category.FeedState.Imported)) + for identity, id := range category.FeedState.Imported { + state.Imported[identity] = id + } + clone.FeedState = &state + } + return clone +} + +func nextNewsArticleID(articles map[uint32]*hotline.NewsArtData) (uint32, uint32, error) { + var previousID uint32 + for id := range articles { + if id > previousID { + previousID = id + } + } + if previousID == math.MaxUint32 { + return 0, 0, errors.New("news category exhausted article IDs") + } + return previousID + 1, previousID, nil +} + +func validateNewsArticleListSize(articles map[uint32]*hotline.NewsArtData) error { + category := hotline.NewsCategoryListData15{Articles: articles} + list, err := category.GetNewsArtListData() + if err != nil { + return err + } + encoded, err := io.ReadAll(&list) + if err != nil { + return err + } + if len(encoded) > math.MaxUint16 { + return errNewsArticleListTooLarge + } + return nil +} + +func feedIdentityHash(sourceURL, identity string) string { + hash := sha256.Sum256([]byte(sourceURL + "\x00" + identity)) + return hex.EncodeToString(hash[:]) +} + +func feedStateEqual(a, b *hotline.NewsFeedCategoryState) bool { + if a == nil || b == nil { + return a == b + } + if a.SourceURL != b.SourceURL || a.ETag != b.ETag || a.LastModified != b.LastModified || len(a.Imported) != len(b.Imported) { + return false + } + for identity, id := range a.Imported { + if b.Imported[identity] != id { + return false + } + } + return true +} diff --git a/internal/mobius/feed_news_test.go b/internal/mobius/feed_news_test.go new file mode 100644 index 0000000..037d1a0 --- /dev/null +++ b/internal/mobius/feed_news_test.go @@ -0,0 +1,462 @@ +package mobius + +import ( + "bytes" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/jhalter/mobius/hotline" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + textencoding "golang.org/x/text/encoding" + "golang.org/x/text/encoding/charmap" +) + +const testFeedURL = "https://example.com/releases.xml" + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { + return f(request) +} + +func testFeedResponse(status int, body string, headers map[string]string) *http.Response { + response := &http.Response{ + StatusCode: status, + Status: fmt.Sprintf("%d %s", status, http.StatusText(status)), + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(body)), + } + for key, value := range headers { + response.Header.Set(key, value) + } + return response +} + +func testNewsFeedConfig(url string, path ...string) hotline.NewsFeedConfig { + return hotline.NewsFeedConfig{CategoryPath: path, URL: url} +} + +func addTestNewsCategory(t *testing.T, base *ThreadedNewsYAML, path []string, name string) { + t.Helper() + require.NoError(t, base.CreateGrouping(path, name, hotline.NewsCategory)) +} + +func newTestFeedNewsManager( + t *testing.T, + base *ThreadedNewsYAML, + config hotline.NewsFeedConfig, + transport http.RoundTripper, + encoding string, +) *FeedNewsManager { + t.Helper() + manager, err := NewFeedNewsManager( + base, + []hotline.NewsFeedConfig{config}, + slog.New(slog.NewTextHandler(io.Discard, nil)), + "mobius-test", + encoding, + ) + require.NoError(t, err) + if transport != nil { + manager.fetcher.client = &http.Client{Transport: transport} + } + return manager +} + +func feedState(t *testing.T, base *ThreadedNewsYAML, path ...string) *hotline.NewsFeedCategoryState { + t.Helper() + state := base.NewsItem(path).FeedState + require.NotNil(t, state) + return state +} + +func TestFeedNewsManager_ImportsAfterglowAsOrdinaryNews(t *testing.T) { + base := newTestThreadedNews(t) + addTestNewsCategory(t, base, nil, "Afterglow Releases") + requests := 0 + manager := newTestFeedNewsManager(t, base, testNewsFeedConfig(testFeedURL, "Afterglow Releases"), roundTripFunc(func(request *http.Request) (*http.Response, error) { + requests++ + assert.Equal(t, "application/rss+xml, application/atom+xml, application/xml;q=0.9, text/xml;q=0.8", request.Header.Get("Accept")) + assert.Equal(t, "mobius-test", request.Header.Get("User-Agent")) + return testFeedResponse(http.StatusOK, feedFixture(t, "afterglow-appcast.xml"), map[string]string{ + "ETag": `"afterglow-23"`, + "Last-Modified": "Sat, 22 Aug 2026 18:09:55 GMT", + }), nil + }), "utf8") + + list, err := manager.ListArticles([]string{"Afterglow Releases"}) + require.NoError(t, err) + assert.Equal(t, 1, list.Count) + assert.Equal(t, 1, requests) + + state := feedState(t, base, "Afterglow Releases") + assert.Equal(t, testFeedURL, state.SourceURL) + assert.Equal(t, `"afterglow-23"`, state.ETag) + require.Len(t, state.Imported, 1) + var rootID uint32 + for _, id := range state.Imported { + rootID = id + } + root := manager.GetArticle([]string{"Afterglow Releases"}, rootID) + require.NotNil(t, root) + assert.Equal(t, "Afterglow 1.0rc4", root.Title) + assert.Equal(t, "Afterglow", root.Poster) + assert.Contains(t, root.Data, "Fixed main-window resizing.") + assert.Contains(t, root.Data, "Release notes: https://morphing.cloud/afterglow/1.0rc4-notes.html") + assert.Contains(t, root.Data, "Download: https://morphing.cloud/afterglow/Afterglow-v1.0rc4.zip") + assert.NotContains(t, root.Data, "

") + expectedTime := time.Date(2026, time.August, 22, 18, 9, 55, 0, time.FixedZone("appcast", -7*60*60)) + assert.True(t, hotline.Time(root.Date).NewsTime().Equal(expectedTime)) + + // Imported roots are completely ordinary: local roots, replies, and deletion + // all use the conventional manager. + require.NoError(t, manager.PostArticle([]string{"Afterglow Releases"}, 0, hotline.NewsArtData{Title: "Local root"})) + require.NoError(t, manager.PostArticle([]string{"Afterglow Releases"}, rootID, hotline.NewsArtData{Title: "Reply"})) + require.NoError(t, manager.DeleteArticle([]string{"Afterglow Releases"}, rootID, true)) + assert.Nil(t, manager.GetArticle([]string{"Afterglow Releases"}, rootID)) + + // The durable seen marker survives deletion, so the same feed entry stays + // deleted even though the source still returns it. + list, err = manager.ListArticles([]string{"Afterglow Releases"}) + require.NoError(t, err) + assert.Equal(t, 2, list.Count) + assert.Nil(t, manager.GetArticle([]string{"Afterglow Releases"}, rootID)) + assert.Equal(t, 2, requests) + + yamlData, err := os.ReadFile(base.filePath) + require.NoError(t, err) + assert.Contains(t, string(yamlData), "FeedState:") + assert.Contains(t, string(yamlData), "Imported:") + assert.NoFileExists(t, filepath.Join(filepath.Dir(base.filePath), "FeedNewsState.yaml")) + assert.NoFileExists(t, filepath.Join(filepath.Dir(base.filePath), "FeedNewsCache.json")) +} + +func TestFeedNewsManager_AutoDetectsAtomAndIgnoresUpdates(t *testing.T) { + base := newTestThreadedNews(t) + addTestNewsCategory(t, base, nil, "Mobius Releases") + var requests int + manager := newTestFeedNewsManager(t, base, testNewsFeedConfig(testFeedURL, "Mobius Releases"), roundTripFunc(func(request *http.Request) (*http.Response, error) { + requests++ + if requests == 1 { + return testFeedResponse(http.StatusOK, feedFixture(t, "mobius-releases.atom"), map[string]string{"ETag": `"v1"`}), nil + } + assert.Equal(t, `"v1"`, request.Header.Get("If-None-Match")) + return testFeedResponse(http.StatusOK, atomBatchFixture( + atomEntry("v0.22.0 renamed", "tag:github.com,2008:Repository/272052223/v0.22.0", "2026-06-12T16:03:33Z"), + atomEntry("v0.23.0", "tag:github.com,2008:Repository/272052223/v0.23.0", "2026-07-01T12:00:00Z"), + ), map[string]string{"ETag": `"v2"`}), nil + }), "utf8") + + _, err := manager.ListArticles([]string{"Mobius Releases"}) + require.NoError(t, err) + first := manager.GetArticle([]string{"Mobius Releases"}, 1) + require.NotNil(t, first) + assert.Equal(t, "v0.22.0", first.Title) + + list, err := manager.ListArticles([]string{"Mobius Releases"}) + require.NoError(t, err) + assert.Equal(t, 2, list.Count) + assert.Equal(t, "v0.22.0", manager.GetArticle([]string{"Mobius Releases"}, 1).Title, "same-identity changes must be ignored") + assert.Equal(t, "v0.23.0", manager.GetArticle([]string{"Mobius Releases"}, 2).Title) + assert.Equal(t, `"v2"`, feedState(t, base, "Mobius Releases").ETag) +} + +func TestFeedNewsManager_RestartUsesConditionalGETWithoutDuplicates(t *testing.T) { + base := newTestThreadedNews(t) + addTestNewsCategory(t, base, nil, "Releases") + config := testNewsFeedConfig(testFeedURL, "Releases") + manager := newTestFeedNewsManager(t, base, config, roundTripFunc(func(*http.Request) (*http.Response, error) { + return testFeedResponse(http.StatusOK, rssFixture("One", "one", "Sat, 22 Aug 2026 18:09:55 -0700"), map[string]string{ + "ETag": `"one"`, + "Last-Modified": "Sat, 22 Aug 2026 18:09:55 GMT", + }), nil + }), "utf8") + _, err := manager.ListArticles([]string{"Releases"}) + require.NoError(t, err) + + reloaded, err := NewThreadedNewsYAML(base.filePath) + require.NoError(t, err) + restarted := newTestFeedNewsManager(t, reloaded, config, roundTripFunc(func(request *http.Request) (*http.Response, error) { + assert.Equal(t, `"one"`, request.Header.Get("If-None-Match")) + assert.Equal(t, "Sat, 22 Aug 2026 18:09:55 GMT", request.Header.Get("If-Modified-Since")) + return testFeedResponse(http.StatusNotModified, "", nil), nil + }), "utf8") + before, err := os.ReadFile(base.filePath) + require.NoError(t, err) + list, err := restarted.ListArticles([]string{"Releases"}) + require.NoError(t, err) + after, err := os.ReadFile(base.filePath) + require.NoError(t, err) + assert.Equal(t, 1, list.Count) + assert.Equal(t, before, after, "304 responses must not rewrite ThreadedNews.yaml") +} + +func TestFeedNewsManager_SourceURLChangesKeepSeenHistory(t *testing.T) { + base := newTestThreadedNews(t) + addTestNewsCategory(t, base, nil, "Releases") + path := []string{"Releases"} + urlA := "https://example.com/a.xml" + urlB := "https://example.com/b.xml" + fixture := rssFixture("Release", "shared-id", "Sat, 22 Aug 2026 18:09:55 -0700") + + for _, sourceURL := range []string{urlA, urlB, urlA} { + manager := newTestFeedNewsManager(t, base, testNewsFeedConfig(sourceURL, path...), roundTripFunc(func(request *http.Request) (*http.Response, error) { + assert.Empty(t, request.Header.Get("If-None-Match"), "a URL switch must clear validators") + return testFeedResponse(http.StatusOK, fixture, map[string]string{"ETag": `"etag"`}), nil + }), "utf8") + _, err := manager.ListArticles(path) + require.NoError(t, err) + } + + list, err := base.ListArticles(path) + require.NoError(t, err) + assert.Equal(t, 2, list.Count, "the same ID is distinct across URLs but not reimported after switching back") + state := feedState(t, base, path...) + assert.Equal(t, urlA, state.SourceURL) + assert.Len(t, state.Imported, 2) +} + +func TestFeedNewsManager_FailuresServeLocalAndEachLoadRetries(t *testing.T) { + base := newTestThreadedNews(t) + addTestNewsCategory(t, base, nil, "Releases") + var requests atomic.Int32 + manager := newTestFeedNewsManager(t, base, testNewsFeedConfig(testFeedURL, "Releases"), roundTripFunc(func(*http.Request) (*http.Response, error) { + if requests.Add(1) == 1 { + return nil, errors.New("offline") + } + return testFeedResponse(http.StatusOK, rssFixture("Recovered", "recovered", "Sat, 22 Aug 2026 18:09:55 -0700"), nil), nil + }), "utf8") + + list, err := manager.ListArticles([]string{"Releases"}) + require.NoError(t, err) + assert.Zero(t, list.Count) + list, err = manager.ListArticles([]string{"Releases"}) + require.NoError(t, err) + assert.Equal(t, 1, list.Count) + assert.Equal(t, int32(2), requests.Load()) +} + +func TestFeedNewsManager_CoalescesConcurrentLoads(t *testing.T) { + base := newTestThreadedNews(t) + addTestNewsCategory(t, base, nil, "Releases") + var requests atomic.Int32 + requestStarted := make(chan struct{}) + releaseRequest := make(chan struct{}) + var startedOnce sync.Once + manager := newTestFeedNewsManager(t, base, testNewsFeedConfig(testFeedURL, "Releases"), roundTripFunc(func(*http.Request) (*http.Response, error) { + requests.Add(1) + startedOnce.Do(func() { close(requestStarted) }) + <-releaseRequest + return testFeedResponse(http.StatusOK, rssFixture("One", "one", "Sat, 22 Aug 2026 18:09:55 -0700"), nil), nil + }), "utf8") + + const callers = 12 + start := make(chan struct{}) + var ready sync.WaitGroup + var done sync.WaitGroup + ready.Add(callers) + done.Add(callers) + for range callers { + go func() { + defer done.Done() + ready.Done() + <-start + _, err := manager.ListArticles([]string{"Releases"}) + assert.NoError(t, err) + }() + } + ready.Wait() + close(start) + <-requestStarted + time.Sleep(25 * time.Millisecond) + close(releaseRequest) + done.Wait() + assert.Equal(t, int32(1), requests.Load()) +} + +func TestFeedNewsManager_SkipsEntriesWithoutIdentityAndRejectsJSON(t *testing.T) { + t.Run("missing identity", func(t *testing.T) { + base := newTestThreadedNews(t) + addTestNewsCategory(t, base, nil, "Releases") + fixture := `ReleasesNo identityNotes` + manager := newTestFeedNewsManager(t, base, testNewsFeedConfig(testFeedURL, "Releases"), roundTripFunc(func(*http.Request) (*http.Response, error) { + return testFeedResponse(http.StatusOK, fixture, map[string]string{"ETag": `"empty"`}), nil + }), "utf8") + list, err := manager.ListArticles([]string{"Releases"}) + require.NoError(t, err) + assert.Zero(t, list.Count) + assert.Empty(t, feedState(t, base, "Releases").Imported) + }) + + t.Run("JSON", func(t *testing.T) { + base := newTestThreadedNews(t) + addTestNewsCategory(t, base, nil, "Releases") + manager := newTestFeedNewsManager(t, base, testNewsFeedConfig(testFeedURL, "Releases"), roundTripFunc(func(*http.Request) (*http.Response, error) { + return testFeedResponse(http.StatusOK, `{"version":"https://jsonfeed.org/version/1.1","items":[]}`, nil), nil + }), "utf8") + list, err := manager.ListArticles([]string{"Releases"}) + require.NoError(t, err) + assert.Zero(t, list.Count) + assert.Nil(t, base.NewsItem([]string{"Releases"}).FeedState) + }) +} + +func TestFeedNewsManager_EncodingIsAppliedOnceAtImport(t *testing.T) { + base := newTestThreadedNews(t) + addTestNewsCategory(t, base, nil, "Releases") + title := "Café — Snowman ☃" + fixture := rssFixture(title, "unicode", "Sat, 22 Aug 2026 18:09:55 -0700") + manager := newTestFeedNewsManager(t, base, testNewsFeedConfig(testFeedURL, "Releases"), roundTripFunc(func(*http.Request) (*http.Response, error) { + return testFeedResponse(http.StatusOK, fixture, nil), nil + }), "macintosh") + _, err := manager.ListArticles([]string{"Releases"}) + require.NoError(t, err) + + wantTitle, err := textencoding.ReplaceUnsupported(charmap.Macintosh.NewEncoder()).String(title) + require.NoError(t, err) + article := base.GetArticle([]string{"Releases"}, 1) + require.NotNil(t, article) + assert.Equal(t, wantTitle, article.Title) + yamlData, err := os.ReadFile(base.filePath) + require.NoError(t, err) + assert.Contains(t, string(yamlData), "!!binary", "MacRoman bytes that are invalid UTF-8 must round-trip through YAML") + + reloaded, err := NewThreadedNewsYAML(base.filePath) + require.NoError(t, err) + assert.Equal(t, wantTitle, reloaded.GetArticle([]string{"Releases"}, 1).Title) + + // Changing server encoding later affects future imports only. + utf8Manager := newTestFeedNewsManager(t, reloaded, testNewsFeedConfig(testFeedURL, "Releases"), roundTripFunc(func(*http.Request) (*http.Response, error) { + return testFeedResponse(http.StatusOK, rssFixture("UTF-8 ☃", "unicode-2", "Sun, 23 Aug 2026 18:09:55 -0700"), nil), nil + }), "utf8") + _, err = utf8Manager.ListArticles([]string{"Releases"}) + require.NoError(t, err) + assert.Equal(t, wantTitle, reloaded.GetArticle([]string{"Releases"}, 1).Title) + assert.Equal(t, "UTF-8 ☃", reloaded.GetArticle([]string{"Releases"}, 2).Title) +} + +func TestFeedNewsManager_ArticleListLimitPersistsFittingPrefixAndRetries(t *testing.T) { + base := newTestThreadedNews(t) + addTestNewsCategory(t, base, nil, "Releases") + fixture := rssBatchFixture("large", 300, 255) + var requests atomic.Int32 + manager := newTestFeedNewsManager(t, base, testNewsFeedConfig(testFeedURL, "Releases"), roundTripFunc(func(request *http.Request) (*http.Response, error) { + requests.Add(1) + assert.Empty(t, request.Header.Get("If-None-Match"), "validators must not advance while unseen entries remain") + return testFeedResponse(http.StatusOK, fixture, map[string]string{"ETag": `"too-large"`}), nil + }), "utf8") + + list, err := manager.ListArticles([]string{"Releases"}) + require.NoError(t, err) + assert.Greater(t, list.Count, 0) + assert.Less(t, list.Count, 300) + encoded, err := io.ReadAll(&list) + require.NoError(t, err) + assert.LessOrEqual(t, len(encoded), 65535) + state := feedState(t, base, "Releases") + assert.Empty(t, state.ETag) + assert.Len(t, state.Imported, list.Count) + + second, err := manager.ListArticles([]string{"Releases"}) + require.NoError(t, err) + assert.Equal(t, list.Count, second.Count) + assert.Equal(t, int32(2), requests.Load(), "the first unseen item must be retried on the next load") +} + +func TestFeedNewsManager_ValidatesExistingOrdinaryTarget(t *testing.T) { + base := newTestThreadedNews(t) + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + _, err := NewFeedNewsManager(base, []hotline.NewsFeedConfig{testNewsFeedConfig(testFeedURL, "Missing")}, logger, "test", "utf8") + require.ErrorContains(t, err, "does not exist") + _, err = NewFeedNewsManager(base, []hotline.NewsFeedConfig{testNewsFeedConfig(testFeedURL, "Archive")}, logger, "test", "utf8") + require.ErrorContains(t, err, "ordinary news category") + _, err = NewFeedNewsManager(base, []hotline.NewsFeedConfig{ + testNewsFeedConfig("https://example.com/one.xml", "General"), + testNewsFeedConfig("https://example.com/two.xml", "General"), + }, logger, "test", "utf8") + require.ErrorContains(t, err, "duplicates") +} + +func TestFeedNewsManager_WriteFailureRollsBackImport(t *testing.T) { + base := newTestThreadedNews(t) + addTestNewsCategory(t, base, nil, "Releases") + manager := newTestFeedNewsManager(t, base, testNewsFeedConfig(testFeedURL, "Releases"), roundTripFunc(func(*http.Request) (*http.Response, error) { + return testFeedResponse(http.StatusOK, rssFixture("One", "one", "Sat, 22 Aug 2026 18:09:55 -0700"), nil), nil + }), "utf8") + base.filePath = filepath.Join(t.TempDir(), "missing", "ThreadedNews.yaml") + + list, err := manager.ListArticles([]string{"Releases"}) + require.NoError(t, err, "import failures serve local news") + assert.Zero(t, list.Count) + item := base.NewsItem([]string{"Releases"}) + assert.Empty(t, item.Articles) + assert.Nil(t, item.FeedState) +} + +func TestFeedFetcherLimitsAndConditional304(t *testing.T) { + t.Run("oversized response", func(t *testing.T) { + fetcher := newFeedFetcher("test") + fetcher.client = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return testFeedResponse(http.StatusOK, strings.Repeat("x", maxFeedBodyBytes+1), nil), nil + })} + _, err := fetcher.fetch(t.Context(), testFeedURL, feedHTTPValidators{}) + assert.ErrorIs(t, err, errFeedResponseTooLarge) + }) + + t.Run("unprompted 304", func(t *testing.T) { + fetcher := newFeedFetcher("test") + fetcher.client = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return testFeedResponse(http.StatusNotModified, "", nil), nil + })} + _, err := fetcher.fetch(t.Context(), testFeedURL, feedHTTPValidators{}) + require.ErrorContains(t, err, "without a conditional request") + }) +} + +func feedFixture(t *testing.T, name string) string { + t.Helper() + data, err := os.ReadFile(filepath.Join("testdata", name)) + require.NoError(t, err) + return string(data) +} + +func rssFixture(title, guid, published string) string { + return fmt.Sprintf(`Releases%s%s%sNotes for %s.

]]>`, title, guid, published, title) +} + +func atomEntry(title, id, updated string) string { + return fmt.Sprintf(`%s%s%sNotesMobius`, id, updated, title) +} + +func atomBatchFixture(entries ...string) string { + return `Releases` + strings.Join(entries, "") + `` +} + +func rssBatchFixture(prefix string, count, titleSize int) string { + var builder strings.Builder + builder.WriteString(`Releases`) + for i := 0; i < count; i++ { + title := strings.Repeat("x", titleSize) + fmt.Fprintf(&builder, `%s%s-%dNotes`, title, prefix, i) + } + builder.WriteString(``) + return builder.String() +} + +func TestEncodeFeedTextDoesNotSplitUTF8(t *testing.T) { + value := strings.Repeat("é", 200) + encoded := truncateEncodedFeedText(value, 255, "utf8") + assert.True(t, bytes.Equal([]byte(encoded), []byte(strings.Repeat("é", 127)))) +} diff --git a/internal/mobius/handlers_news.go b/internal/mobius/handlers_news.go index 29b0ae2..368375e 100644 --- a/internal/mobius/handlers_news.go +++ b/internal/mobius/handlers_news.go @@ -342,7 +342,7 @@ func HandlePostNewsArt(cc *hotline.ClientConn, t *hotline.Transaction) (res []ho hotline.NewsArtData{ Title: string(t.GetField(hotline.FieldNewsArtTitle).Data), Poster: string(cc.GetUserName()), - Date: hotline.NewTime(time.Now()), + Date: hotline.NewNewsTime(time.Now()), DataFlav: hotline.NewsFlavor, Data: string(t.GetField(hotline.FieldNewsArtData).Data), }, diff --git a/internal/mobius/testdata/afterglow-appcast.xml b/internal/mobius/testdata/afterglow-appcast.xml new file mode 100644 index 0000000..b31831e --- /dev/null +++ b/internal/mobius/testdata/afterglow-appcast.xml @@ -0,0 +1,22 @@ + + + + Afterglow + https://morphing.cloud/afterglow/appcast.xml + Most recent updates to Afterglow + + Afterglow 1.0rc4 + Sat, 22 Aug 2026 18:09:55 -0700 + 23 + https://morphing.cloud/afterglow/1.0rc4-notes.html + Fixes

+ + ]]> + + + + diff --git a/internal/mobius/testdata/mobius-releases.atom b/internal/mobius/testdata/mobius-releases.atom new file mode 100644 index 0000000..f928f60 --- /dev/null +++ b/internal/mobius/testdata/mobius-releases.atom @@ -0,0 +1,16 @@ + + + tag:github.com,2008:https://github.com/jhalter/mobius/releases + + + Release notes from mobius + 2026-06-12T15:55:05Z + + tag:github.com,2008:Repository/272052223/v0.22.0 + 2026-06-12T16:03:33Z + + v0.22.0 + <h2>Changelog</h2><ul><li>Fix data races</li><li>Improve graceful shutdown</li></ul> + jhalter + + -- cgit