feat: dynamic switch between nitter instances and clearly mark genera…

…ted profiles as so (closes #50 #49 )

.dockerignore

@@ -16,3 +16,4 @@ fly.toml
 .vs
 *.sqlite
 coverage*
+rsslay

.env.sample

@@ -6,9 +6,10 @@ SECRET=test
 DB_DIR="/db/rsslay.sqlite"
 DEFAULT_PROFILE_PICTURE_URL="https://i.imgur.com/MaceU96.png"
 SECRET="CHANGE_ME"
-VERSION=0.4.6
+VERSION=0.4.7
 REPLAY_TO_RELAYS=false
 RELAYS_TO_PUBLISH_TO=""
+NITTER_INSTANCES=""
 DEFAULT_WAIT_TIME_BETWEEN_BATCHES=60000
 DEFAULT_WAIT_TIME_FOR_RELAY_RESPONSE=3000
 MAX_EVENTS_TO_REPLAY=10

.gitignore

@@ -5,3 +5,4 @@ db
 dist
 *.sqlite
 coverage*
+rsslay

Dockerfile

@@ -28,9 +28,10 @@ ENV PORT="8080"
 ENV DB_DIR="/db/rsslay.sqlite"
 ENV DEFAULT_PROFILE_PICTURE_URL="https://i.imgur.com/MaceU96.png"
 ENV SECRET="CHANGE_ME"
-ENV VERSION=0.4.6
+ENV VERSION=0.4.7
 ENV REPLAY_TO_RELAYS=false
 ENV RELAYS_TO_PUBLISH_TO=""
+ENV NITTER_INSTANCES=""
 ENV DEFAULT_WAIT_TIME_BETWEEN_BATCHES=60000
 ENV DEFAULT_WAIT_TIME_FOR_RELAY_RESPONSE=3000
 ENV MAX_EVENTS_TO_REPLAY=10

Dockerfile.fly

@@ -30,9 +30,10 @@ ENV PORT="8080"
 ENV DB_DIR="/db/rsslay.sqlite"
 ENV DEFAULT_PROFILE_PICTURE_URL="https://i.imgur.com/MaceU96.png"
 ENV SECRET="CHANGE_ME"
-ENV VERSION=0.4.6
+ENV VERSION=0.4.7
 ENV REPLAY_TO_RELAYS=false
 ENV RELAYS_TO_PUBLISH_TO=""
+ENV NITTER_INSTANCES=""
 ENV DEFAULT_WAIT_TIME_BETWEEN_BATCHES=60000
 ENV DEFAULT_WAIT_TIME_FOR_RELAY_RESPONSE=3000
 ENV MAX_EVENTS_TO_REPLAY=10

Dockerfile.railwayapp

@@ -6,6 +6,7 @@ ARG DEFAULT_PROFILE_PICTURE_URL
 ARG SECRET
 ARG REPLAY_TO_RELAYS
 ARG RELAYS_TO_PUBLISH_TO
+ARG NITTER_INSTANCES
 ARG DEFAULT_WAIT_TIME_BETWEEN_BATCHES
 ARG DEFAULT_WAIT_TIME_FOR_RELAY_RESPONSE
 ARG MAX_EVENTS_TO_REPLAY
@@ -43,6 +44,7 @@ ARG DEFAULT_PROFILE_PICTURE_URL
 ARG SECRET
 ARG REPLAY_TO_RELAYS
 ARG RELAYS_TO_PUBLISH_TO
+ARG NITTER_INSTANCES
 ARG DEFAULT_WAIT_TIME_BETWEEN_BATCHES
 ARG DEFAULT_WAIT_TIME_FOR_RELAY_RESPONSE
 ARG MAX_EVENTS_TO_REPLAY
@@ -66,9 +68,10 @@ ENV PORT=$PORT
 ENV DB_DIR=$DB_DIR
 ENV DEFAULT_PROFILE_PICTURE_URL=$DEFAULT_PROFILE_PICTURE_URL
 ENV SECRET=$SECRET
-ENV VERSION=0.4.6
+ENV VERSION=0.4.7
 ENV REPLAY_TO_RELAYS=false
 ENV RELAYS_TO_PUBLISH_TO=""
+ENV NITTER_INSTANCES=""
 ENV DEFAULT_WAIT_TIME_BETWEEN_BATCHES=60000
 ENV DEFAULT_WAIT_TIME_FOR_RELAY_RESPONSE=3000
 ENV MAX_EVENTS_TO_REPLAY=10

README.md

@@ -44,6 +44,18 @@ This is needed nowadays, with further improvements in relays implementations or
 
 Maybe in the future with other implementations we can avoid that, but nowadays its needed.
 
+## Feeds from Twitter via Nitter instances
+
+The [Nitter](https://github.com/zedeus/nitter) project is well integrated into `rsslay` and it performs special handling of this kind of feeds.
+Currently `rsslay` is doing the following with Nitter feeds:
+- Upgrading to `https` all instances that may be misconfigured.
+- Format some responses and retweets in a more "user-friendly" way.
+- As there are many instances available out there, if one is unreachable at the moment of parsing, a pool of instances is used (configurable):
+  - [birdsite.xanny.family](https://birdsite.xanny.family/)
+  - [notabird.site](https://notabird.site/)
+  - [nitter.moomoo.me](https://nitter.moomoo.me/)
+  - [nitter.fly.dev](https://nitter.fly.dev/)
+
 ## Running the project
 
 Running `rsslay` its easy, checkout [the wiki entry for it](https://github.com/piraces/rsslay/wiki/Running-the-project).

cmd/rsslay/main.go

@@ -43,6 +43,7 @@ type Relay struct {
 	Version                         string   `envconfig:"VERSION" default:"unknown"`
 	ReplayToRelays                  bool     `envconfig:"REPLAY_TO_RELAYS" default:"false"`
 	RelaysToPublish                 []string `envconfig:"RELAYS_TO_PUBLISH_TO" default:""`
+	NitterInstances                 []string `envconfig:"NITTER_INSTANCES" default:""`
 	DefaultWaitTimeBetweenBatches   int64    `envconfig:"DEFAULT_WAIT_TIME_BETWEEN_BATCHES" default:"60000"`
 	DefaultWaitTimeForRelayResponse int64    `envconfig:"DEFAULT_WAIT_TIME_FOR_RELAY_RESPONSE" default:"3000"`
 	MaxEventsToReplay               int      `envconfig:"MAX_EVENTS_TO_REPLAY" default:"20"`
@@ -145,7 +146,7 @@ func (r *Relay) UpdateListeningFilters() {
 		for _, filter := range filters {
 			if filter.Kinds == nil || slices.Contains(filter.Kinds, nostr.KindTextNote) {
 				for _, pubkey := range filter.Authors {
-					parsedFeed, entity := events.GetParsedFeedForPubKey(pubkey, r.db, r.DeleteFailingFeeds)
+					parsedFeed, entity := events.GetParsedFeedForPubKey(pubkey, r.db, r.DeleteFailingFeeds, r.NitterInstances)
 					if parsedFeed == nil {
 						continue
 					}
@@ -216,7 +217,7 @@ func (b store) QueryEvents(filter *nostr.Filter) ([]nostr.Event, error) {
 	}
 
 	for _, pubkey := range filter.Authors {
-		parsedFeed, entity := events.GetParsedFeedForPubKey(pubkey, relayInstance.db, relayInstance.DeleteFailingFeeds)
+		parsedFeed, entity := events.GetParsedFeedForPubKey(pubkey, relayInstance.db, relayInstance.DeleteFailingFeeds, relayInstance.NitterInstances)
 
 		if parsedFeed == nil {
 			continue

fly.toml

@@ -9,6 +9,7 @@ app = "rsslay"
   DEFAULT_PROFILE_PICTURE_URL = "https://i.imgur.com/MaceU96.png"
   REPLAY_TO_RELAYS = "true"
   RELAYS_TO_PUBLISH_TO = "wss://relay.nostrgraph.net,wss://e.nos.lol,wss://nostr.mom,wss://relay.nostr.band,wss://nos.lol,wss://nostr.mutinywallet.com"
+  NITTER_INSTANCES = "birdsite.xanny.family,notabird.site,nitter.moomoo.me,nitter.fly.dev"
   DEFAULT_WAIT_TIME_BETWEEN_BATCHES = "60000"
   DEFAULT_WAIT_TIME_FOR_RELAY_RESPONSE = "1000"
   MAX_EVENTS_TO_REPLAY = "10"
@@ -19,7 +20,7 @@ app = "rsslay"
   INFO_RELAY_NAME = "rsslay public instance"
   INFO_CONTACT = "mailto:raul@piraces.dev"
   MAX_CONTENT_LENGTH = 5000
-  VERSION = "0.4.6"
+  VERSION = "0.4.7"
   LOG_LEVEL = "INFO"
   DELETE_FAILING_FEEDS = "false"
 

internal/handlers/handlers.go

@@ -248,7 +248,8 @@ func createFeedEntry(r *http.Request, db *sql.DB, secret *string) *Entry {
 		return &entry
 	}
 
-	if _, err := feed.ParseFeed(feedUrl); err != nil {
+	parsedFeed, err := feed.ParseFeed(feedUrl)
+	if err != nil {
 		entry.ErrorCode = http.StatusBadRequest
 		entry.Error = true
 		entry.ErrorMessage = "Bad feed: " + err.Error()
@@ -265,22 +266,23 @@ func createFeedEntry(r *http.Request, db *sql.DB, secret *string) *Entry {
 	}
 
 	publicKey = strings.TrimSpace(publicKey)
-	defer insertFeed(err, feedUrl, publicKey, sk, db)
+	isNitterFeed := strings.Contains(parsedFeed.Description, "Twitter feed")
+	defer insertFeed(err, feedUrl, publicKey, sk, isNitterFeed, db)
 
 	entry.Url = feedUrl
 	entry.PubKey = publicKey
 	entry.NPubKey, _ = nip19.EncodePublicKey(publicKey)
 	return &entry
 }
 
-func insertFeed(err error, feedUrl string, publicKey string, sk string, db *sql.DB) {
+func insertFeed(err error, feedUrl string, publicKey string, sk string, nitter bool, db *sql.DB) {
 	row := db.QueryRow("SELECT privatekey, url FROM feeds WHERE publickey=$1", publicKey)
 
 	var entity feed.Entity
 	err = row.Scan(&entity.PrivateKey, &entity.URL)
 	if err != nil && err == sql.ErrNoRows {
 		log.Printf("[DEBUG] not found feed at url %q as publicKey %s", feedUrl, publicKey)
-		if _, err := db.Exec(`INSERT INTO feeds (publickey, privatekey, url) VALUES (?, ?, ?)`, publicKey, sk, feedUrl); err != nil {
+		if _, err := db.Exec(`INSERT INTO feeds (publickey, privatekey, url, nitter) VALUES (?, ?, ?, ?)`, publicKey, sk, feedUrl, nitter); err != nil {
 			log.Printf("[ERROR] failure: %v", err)
 		} else {
 			log.Printf("[DEBUG] saved feed at url %q as publicKey %s", feedUrl, publicKey)

pkg/events/events.go

@@ -6,15 +6,16 @@ import (
 	"github.com/piraces/rsslay/pkg/feed"
 	"github.com/piraces/rsslay/pkg/helpers"
 	"log"
+	"net/url"
 	"strings"
 )
 
-func GetParsedFeedForPubKey(pubKey string, db *sql.DB, deleteFailingFeeds bool) (*gofeed.Feed, feed.Entity) {
+func GetParsedFeedForPubKey(pubKey string, db *sql.DB, deleteFailingFeeds bool, nitterInstances []string) (*gofeed.Feed, feed.Entity) {
 	pubKey = strings.TrimSpace(pubKey)
-	row := db.QueryRow("SELECT privatekey, url FROM feeds WHERE publickey=$1", pubKey)
+	row := db.QueryRow("SELECT privatekey, url, nitter FROM feeds WHERE publickey=$1", pubKey)
 
 	var entity feed.Entity
-	err := row.Scan(&entity.PrivateKey, &entity.URL)
+	err := row.Scan(&entity.PrivateKey, &entity.URL, &entity.Nitter)
 	if err != nil && err == sql.ErrNoRows {
 		return nil, entity
 	} else if err != nil {
@@ -31,6 +32,19 @@ func GetParsedFeedForPubKey(pubKey string, db *sql.DB, deleteFailingFeeds bool)
 	}
 
 	parsedFeed, err := feed.ParseFeed(entity.URL)
+	if err != nil && entity.Nitter {
+		log.Printf("[DEBUG] failed to parse feed at url %q: %v. Now iterating through other Nitter instances", entity.URL, err)
+		for i, instance := range nitterInstances {
+			newUrl, _ := setHostname(entity.URL, instance)
+			log.Printf("[DEBUG] attempt %d: use %q instead of %q", i, newUrl, entity.URL)
+			parsedFeed, err = feed.ParseFeed(newUrl)
+			if err == nil {
+				log.Printf("[DEBUG] attempt %d: success with %q", i, newUrl)
+				break
+			}
+		}
+	}
+
 	if err != nil {
 		log.Printf("[DEBUG] failed to parse feed at url %q: %v", entity.URL, err)
 		if deleteFailingFeeds {
@@ -39,5 +53,28 @@ func GetParsedFeedForPubKey(pubKey string, db *sql.DB, deleteFailingFeeds bool)
 		return nil, entity
 	}
 
+	if strings.Contains(parsedFeed.Description, "Twitter feed") && !entity.Nitter {
+		updateDatabaseEntry(&entity, db)
+		entity.Nitter = true
+	}
+
 	return parsedFeed, entity
 }
+
+func updateDatabaseEntry(entity *feed.Entity, db *sql.DB) {
+	log.Printf("[DEBUG] attempting to set feed at url %q with publicKey %s as nitter instance", entity.URL, entity.PublicKey)
+	if _, err := db.Exec(`UPDATE feeds SET nitter = ? WHERE publickey = ?`, 1, entity.PublicKey); err != nil {
+		log.Printf("[ERROR] failure while updating record on db to set as nitter feed: %v", err)
+	} else {
+		log.Printf("[DEBUG] set feed at url %q with publicKey %s as nitter instance", entity.URL, entity.PublicKey)
+	}
+}
+
+func setHostname(addr, hostname string) (string, error) {
+	u, err := url.Parse(addr)
+	if err != nil {
+		return "", err
+	}
+	u.Host = hostname
+	return u.String(), nil
+}