File tree Expand file tree Collapse file tree
Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -178,9 +178,11 @@ def hasHashtag(self, hashtag):
178178 def getLinks (self ):
179179 urls = []
180180 content = self .getText ()
181+ is_bluesky = 'platform' in self .data and self .data ['platform' ] == 'bluesky'
182+ has_html_content = 'content' in self .data and '<' in self .data ['content' ]
181183
182184 # For Bluesky posts - extract URLs from structured data first
183- if 'platform' in self . data and self . data [ 'platform' ] == 'bluesky' and 'raw_record' in self .data :
185+ if is_bluesky and 'raw_record' in self .data :
184186 raw_record = self .data ['raw_record' ]
185187
186188 # Extract from embed.external.uri
@@ -196,13 +198,15 @@ def getLinks(self):
196198 urls .append (feature ['uri' ])
197199
198200 # For Mastodon posts with HTML content
199- if 'content' in self . data and '<' in self . data [ 'content' ] :
201+ if has_html_content :
200202 extractor = URLExtractor ()
201203 extractor .feed (self .data ['content' ])
202204 urls .extend (extractor .urls )
203205
204206 # Fallback: extract URLs from plain text content using regex
205- if content :
207+ # Skip if we already found URLs from structured data (Bluesky) or HTML (Mastodon)
208+ # This avoids extracting HTML-encoded duplicates or truncated display text
209+ if content and not ((is_bluesky or has_html_content ) and len (urls ) > 0 ):
206210 import re
207211 # Match URLs in plain text (http/https URLs)
208212 url_pattern = r'https?://[^\s<>"]+|[a-zA-Z0-9][-a-zA-Z0-9]*\.(?:[a-zA-Z]{2,}(?:\.[a-zA-Z]{2,})?)[^\s<>"]*'
You can’t perform that action at this time.
0 commit comments