🎯 Exemples recommandés
Balanced sample collections from various categories for you to explore
Interactions WebMentions entre Sites
Exemples du protocole WebMentions pour les interactions entre sites, systèmes de commentaires et réseaux sociaux
💻 WebMentions Hello World javascript
🟢 simple
⭐⭐
Implémentation WebMentions de base avec envoi et réception de mentions entre sites
⏱️ 25 min
🏷️ webmentions, indieweb, social, interactions
Prerequisites:
JavaScript, Fetch API, HTML parsing, HTTP protocols
// WebMentions Hello World - Basic Cross-Site Interactions
// Simple implementation for sending and receiving webmentions
class WebMentionsHelloWorld {
constructor(websiteUrl) {
this.websiteUrl = websiteUrl
this.mentions = [] // In-memory storage (use database in production)
this.sentMentions = new Map() // Track sent mentions
}
// Discover WebMention endpoint for a URL
async discoverWebMentionEndpoint(targetUrl) {
try {
// Fetch the target URL
const response = await fetch(targetUrl, {
method: 'GET',
headers: {
'Accept': 'text/html,application/xhtml+xml'
}
})
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`)
}
const html = await response.text()
// Parse HTML to find WebMention endpoint
const parser = new DOMParser()
const doc = parser.parseFromString(html, 'text/html')
// Check for <link> tags with rel="webmention"
const linkTags = doc.querySelectorAll('link[rel~="webmention"]')
if (linkTags.length > 0) {
return linkTags[0].getAttribute('href')
}
// Check for <a> tags with rel="webmention"
const anchorTags = doc.querySelectorAll('a[rel~="webmention"]')
if (anchorTags.length > 0) {
return anchorTags[0].getAttribute('href')
}
// Check HTTP Link header
const linkHeader = response.headers.get('Link')
if (linkHeader) {
const links = this.parseLinkHeader(linkHeader)
const webmentionLink = links.find(link => link.rel.includes('webmention'))
if (webmentionLink) {
return webmentionLink.href
}
}
return null
} catch (error) {
console.error('Error discovering WebMention endpoint:', error)
return null
}
}
// Parse HTTP Link header
parseLinkHeader(linkHeader) {
const links = []
const parts = linkHeader.split(',')
for (const part of parts) {
const match = part.match(/<([^>]+)>\s*;\s*rel="([^"]+)"/)
if (match) {
links.push({
href: match[1],
rel: match[2].split(/\s+/)
})
}
}
return links
}
// Send a WebMention
async sendWebMention(sourceUrl, targetUrl) {
try {
console.log(`Sending WebMention from ${sourceUrl} to ${targetUrl}`)
// Discover the WebMention endpoint
const endpoint = await this.discoverWebMentionEndpoint(targetUrl)
if (!endpoint) {
throw new Error('No WebMention endpoint found for target URL')
}
// Make the WebMention request
const formData = new FormData()
formData.append('source', sourceUrl)
formData.append('target', targetUrl)
const response = await fetch(endpoint, {
method: 'POST',
body: formData,
headers: {
'Accept': 'application/json'
}
})
const result = await response.json()
if (response.ok && result.status === 'accepted') {
// Store sent mention
this.sentMentions.set(`${sourceUrl}:${targetUrl}`, {
source: sourceUrl,
target: targetUrl,
endpoint: endpoint,
status: 'accepted',
sentAt: new Date(),
response: result
})
console.log('WebMention sent successfully:', result)
return {
success: true,
status: result.status,
message: result.message || 'WebMention accepted'
}
} else {
throw new Error(result.message || 'WebMention rejected')
}
} catch (error) {
console.error('Error sending WebMention:', error)
return {
success: false,
error: error.message
}
}
}
// Receive and verify a WebMention
async receiveWebMention(sourceUrl, targetUrl) {
try {
console.log(`Processing WebMention from ${sourceUrl} to ${targetUrl}`)
// Verify the source actually mentions the target
const mentionsTarget = await this.verifyMention(sourceUrl, targetUrl)
if (!mentionsTarget) {
throw new Error('Source does not mention target URL')
}
// Extract content from source
const content = await this.extractContent(sourceUrl)
// Create mention object
const mention = {
id: this.generateMentionId(),
source: sourceUrl,
target: targetUrl,
content: content,
author: content.author || 'Anonymous',
published: content.published || new Date().toISOString(),
receivedAt: new Date(),
verified: true,
approved: this.shouldAutoApprove(content)
}
// Store the mention
this.mentions.push(mention)
console.log('WebMention received and processed:', mention)
return {
success: true,
mention: mention
}
} catch (error) {
console.error('Error processing WebMention:', error)
return {
success: false,
error: error.message
}
}
}
// Verify that source URL actually mentions target URL
async verifyMention(sourceUrl, targetUrl) {
try {
const response = await fetch(sourceUrl, {
method: 'GET',
headers: {
'Accept': 'text/html,application/xhtml+xml'
}
})
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`)
}
const html = await response.text()
const parser = new DOMParser()
const doc = parser.parseFromString(html, 'text/html')
// Check for links to target URL
const links = doc.querySelectorAll(`a[href="${targetUrl}"], a[href="${targetUrl}/"]`)
if (links.length > 0) {
return true
}
// Check for text mentions of the URL
const textContent = doc.body.textContent || ''
return textContent.includes(targetUrl) || textContent.includes(targetUrl.replace(/^https?:\/\//, ''))
} catch (error) {
console.error('Error verifying mention:', error)
return false
}
}
// Extract structured content from source URL
async extractContent(sourceUrl) {
try {
const response = await fetch(sourceUrl, {
method: 'GET',
headers: {
'Accept': 'text/html,application/xhtml+xml'
}
})
const html = await response.text()
const parser = new DOMParser()
const doc = parser.parseFromString(html, 'text/html')
// Try to extract structured data (microformats2, schema.org, etc.)
const content = {
title: this.extractTitle(doc),
author: this.extractAuthor(doc),
published: this.extractPublished(doc),
content: this.extractContentText(doc),
url: sourceUrl
}
return content
} catch (error) {
console.error('Error extracting content:', error)
return {
title: 'Unknown',
author: 'Anonymous',
published: new Date().toISOString(),
content: 'Failed to extract content',
url: sourceUrl
}
}
}
extractTitle(doc) {
// Try various title selectors
const selectors = [
'meta[property="og:title"]',
'meta[name="twitter:title"]',
'.p-name', // microformats2
'title',
'h1'
]
for (const selector of selectors) {
const element = doc.querySelector(selector)
if (element) {
const content = element.getAttribute('content') || element.textContent
if (content) return content.trim()
}
}
return 'Untitled'
}
extractAuthor(doc) {
// Try various author selectors
const selectors = [
'.p-author .p-name', // microformats2
'meta[property="article:author"]',
'meta[name="author"]',
'.author',
'[rel="author"]'
]
for (const selector of selectors) {
const element = doc.querySelector(selector)
if (element) {
const content = element.getAttribute('content') || element.textContent
if (content) return content.trim()
}
}
return null
}
extractPublished(doc) {
// Try various date selectors
const selectors = [
'.dt-published', // microformats2
'meta[property="article:published_time"]',
'meta[name="date"]',
'time[datetime]'
]
for (const selector of selectors) {
const element = doc.querySelector(selector)
if (element) {
const content = element.getAttribute('content') || element.getAttribute('datetime') || element.textContent
if (content) return content.trim()
}
}
return new Date().toISOString()
}
extractContentText(doc) {
// Try to find the main content
const selectors = [
'.e-content', // microformats2
'meta[property="og:description"]',
'meta[name="description"]',
'article',
'main'
]
for (const selector of selectors) {
const element = doc.querySelector(selector)
if (element) {
const content = element.getAttribute('content') || element.textContent
if (content) {
const text = content.trim()
// Return first 200 characters
return text.length > 200 ? text.substring(0, 200) + '...' : text
}
}
}
// Fallback to body text
const bodyText = doc.body.textContent || ''
return bodyText.trim().substring(0, 200) + '...'
}
// Auto-approval logic
shouldAutoApprove(content) {
// Simple auto-approval rules
const suspiciousWords = ['spam', 'casino', 'viagra', 'lottery']
const contentLower = content.content.toLowerCase()
return !suspiciousWords.some(word => contentLower.includes(word))
}
// Generate unique mention ID
generateMentionId() {
return 'wm_' + Date.now() + '_' + Math.random().toString(36).substr(2, 9)
}
// Get all mentions for a target URL
getMentions(targetUrl, options = {}) {
const {
approved = true,
limit = null,
offset = 0
} = options
let mentions = this.mentions.filter(mention => mention.target === targetUrl)
if (approved !== null) {
mentions = mentions.filter(mention => mention.approved === approved)
}
// Sort by received date (newest first)
mentions.sort((a, b) => new Date(b.receivedAt) - new Date(a.receivedAt))
// Apply pagination
if (limit) {
mentions = mentions.slice(offset, offset + limit)
}
return mentions
}
// Get mentions statistics
getMentionStats() {
return {
total: this.mentions.length,
approved: this.mentions.filter(m => m.approved).length,
pending: this.mentions.filter(m => !m.approved).length,
sent: this.sentMentions.size,
byTarget: this.groupMentionsByTarget()
}
}
groupMentionsByTarget() {
const grouped = {}
for (const mention of this.mentions) {
if (!grouped[mention.target]) {
grouped[mention.target] = []
}
grouped[mention.target].push(mention)
}
return grouped
}
}
// Example HTML structure with WebMention support
// <!DOCTYPE html>
// <html lang="en">
// <head>
// <meta charset="UTF-8">
// <meta name="viewport" content="width=device-width, initial-scale=1.0">
// <title>My Blog Post</title>
// <!-- WebMention endpoint declaration -->
// <link rel="webmention" href="https://myblog.com/webmention">
//
// <!-- Open Graph and other meta tags -->
// <meta property="og:title" content="My Blog Post">
// <meta property="og:description" content="An interesting blog post about WebMentions">
// </head>
// <body>
// <article class="h-entry">
// <header>
// <h1 class="p-name">My Blog Post</h1>
// <p class="p-author h-card">
// By <span class="p-name">John Doe</span>
// </p>
// <time class="dt-published" datetime="2024-01-15T10:00:00Z">
// January 15, 2024
// </time>
// </header>
//
// <div class="e-content">
// <p>This is an interesting blog post about WebMentions.
// You can link to it from your own blog, and I'll receive a notification.</p>
//
// <p>Check out this related post: <a href="https://example.com/related-post">Related Post</a></p>
// </div>
// </article>
//
// <!-- WebMentions section -->
// <section class="webmentions">
// <h2>WebMentions</h2>
// <div id="webmentions-list">
// <!-- Mentions will be loaded here -->
// </div>
// </section>
//
// <script src="webmentions.js"></script>
// <script>
// // Initialize WebMentions
// const webmentions = new WebMentionsHelloWorld('https://myblog.com');
//
// // Load mentions for current page
// async function loadMentions() {
// const currentUrl = window.location.href;
// const mentions = webmentions.getMentions(currentUrl);
//
// displayMentions(mentions);
// }
//
// function displayMentions(mentions) {
// const container = document.getElementById('webmentions-list');
//
// if (mentions.length === 0) {
// container.innerHTML = '<p>No WebMentions yet.</p>';
// return;
// }
//
// container.innerHTML = mentions.map(mention => `
// <article class="webmention" data-id="${mention.id}">
// <div class="webmention-header">
// <strong>${mention.author}</strong>
// <span class="webmention-date">
// ${new Date(mention.receivedAt).toLocaleDateString()}
// </span>
// </div>
// <div class="webmention-content">
// <p>${mention.content}</p>
// </div>
// <div class="webmention-source">
// <a href="${mention.source}" target="_blank" rel="noopener">
// View original
// </a>
// </div>
// </article>
// `).join('');
// }
//
// // Send a WebMention (example)
// async function sendWebMention(sourceUrl, targetUrl) {
// const result = await webmentions.sendWebMention(sourceUrl, targetUrl);
//
// if (result.success) {
// alert('WebMention sent successfully!');
// } else {
// alert('Error sending WebMention: ' + result.error);
// }
// }
//
// // Load mentions when page loads
// document.addEventListener('DOMContentLoaded', loadMentions);
// </script>
// </body>
// </html>
// Server-side WebMention endpoint example (Node.js/Express)
/*
const express = require('express');
const bodyParser = require('body-parser');
const { URL } = require('url');
const app = express();
app.use(bodyParser.urlencoded({ extended: true }));
// WebMention endpoint
app.post('/webmention', async (req, res) => {
try {
const { source, target } = req.body;
if (!source || !target) {
return res.status(400).json({
error: 'Missing source or target URL'
});
}
// Verify URLs
try {
new URL(source);
new URL(target);
} catch (e) {
return res.status(400).json({
error: 'Invalid source or target URL'
});
}
// Process the WebMention
const result = await processWebMention(source, target);
if (result.success) {
res.status(200).json({
status: 'accepted',
message: 'WebMention accepted'
});
} else {
res.status(400).json({
error: result.error
});
}
} catch (error) {
console.error('WebMention processing error:', error);
res.status(500).json({
error: 'Internal server error'
});
}
});
async function processWebMention(source, target) {
// Verify that source mentions target
const sourceContent = await fetch(source);
const sourceText = await sourceContent.text();
if (!sourceText.includes(target)) {
return { success: false, error: 'Source does not mention target' };
}
// Store the WebMention (in a real app, save to database)
console.log(`WebMention received: ${source} -> ${target}`);
return { success: true };
}
app.listen(3000, () => {
console.log('WebMention server running on port 3000');
});
*/
// Usage example:
// const webmentions = new WebMentionsHelloWorld('https://myblog.com')
//
// // Send a WebMention
// webmentions.sendWebMention(
// 'https://myblog.com/reply-to-post',
// 'https://otherblog.com/original-post'
// )
//
// // Get mentions for a page
// const mentions = webmentions.getMentions('https://myblog.com/my-post')
// console.log('Mentions:', mentions)
export default WebMentionsHelloWorld
💻 Système de Commentaires Avancé typescript
🟡 intermediate
⭐⭐⭐⭐
Système complet de commentaires utilisant WebMentions avec modération, filtrage de spam et contenu riche
⏱️ 40 min
🏷️ webmentions, comments, social, moderation
Prerequisites:
TypeScript, HTML parsing, Content extraction, Spam detection
// Advanced WebMentions Comment System
// Rich content support, moderation, spam filtering, and engagement tracking
interface WebMention {
id: string
source: string
target: string
content: WebMentionContent
author: WebMentionAuthor
published: string
receivedAt: Date
verified: boolean
approved: boolean
spamScore: number
moderationStatus: 'pending' | 'approved' | 'rejected' | 'spam'
replies: string[]
reactions: {
likes: number
reposts: number
replies: number
}
metadata: {
fetchDate?: Date
lastChecked?: Date
checkCount: number
}
}
interface WebMentionContent {
title: string
text: string
html?: string
excerpt: string
url: string
type: 'post' | 'reply' | 'like' | 'repost' | 'mention'
images?: string[]
videos?: string[]
language?: string
}
interface WebMentionAuthor {
name: string
url?: string
photo?: string
email?: string
social?: {
twitter?: string
mastodon?: string
github?: string
}
}
interface ModerationRule {
id: string
name: string
type: 'spam' | 'approval' | 'rejection'
conditions: {
keywords?: string[]
minLength?: number
maxLength?: number
allowedDomains?: string[]
blockedDomains?: string[]
requireHttps?: boolean
requireName?: boolean
}
actions: {
autoApprove?: boolean
autoReject?: boolean
markSpam?: boolean
requireModeration?: boolean
}
}
class WebMentionsCommentSystem {
private mentions: Map<string, WebMention> = new Map()
private moderationRules: ModerationRule[] = []
private spamFilters: SpamFilter[] = []
private contentParsers: ContentParser[] = []
constructor(
private websiteUrl: string,
private options: {
autoApprove?: boolean
enableSpamFilter?: boolean
enableRichContent?: boolean
moderationRules?: ModerationRule[]
} = {}
) {
this.setupDefaultModerationRules()
this.setupDefaultSpamFilters()
this.setupDefaultContentParsers()
}
// Process incoming WebMention
async processWebMention(sourceUrl: string, targetUrl: string): Promise<{
success: boolean
mention?: WebMention
error?: string
status?: string
}> {
try {
console.log(`Processing WebMention: ${sourceUrl} -> ${targetUrl}`)
// Generate unique ID
const mentionId = this.generateMentionId(sourceUrl, targetUrl)
// Check if mention already exists
if (this.mentions.has(mentionId)) {
return {
success: false,
error: 'WebMention already processed',
status: 'duplicate'
}
}
// Verify the mention
const verificationResult = await this.verifyWebMention(sourceUrl, targetUrl)
if (!verificationResult.valid) {
return {
success: false,
error: verificationResult.error || 'Invalid WebMention',
status: 'invalid'
}
}
// Extract content
const content = await this.extractWebMentionContent(sourceUrl)
// Parse content type
const mentionType = this.detectMentionType(content, sourceUrl, targetUrl)
// Extract author information
const author = await this.extractAuthorInfo(sourceUrl)
// Create mention object
const mention: WebMention = {
id: mentionId,
source: sourceUrl,
target: targetUrl,
content: {
...content,
type: mentionType
},
author,
published: content.published || new Date().toISOString(),
receivedAt: new Date(),
verified: true,
approved: false,
spamScore: 0,
moderationStatus: 'pending',
replies: [],
reactions: {
likes: 0,
reposts: 0,
replies: 0
},
metadata: {
fetchDate: new Date(),
checkCount: 1
}
}
// Apply spam filtering
if (this.options.enableSpamFilter) {
mention.spamScore = await this.calculateSpamScore(mention)
mention.moderationStatus = this.determineModerationStatus(mention)
}
// Apply moderation rules
this.applyModerationRules(mention)
// Auto-approve if enabled and no spam detected
if (this.options.autoApprove && mention.spamScore < 50 && mention.moderationStatus === 'pending') {
mention.approved = true
mention.moderationStatus = 'approved'
}
// Store mention
this.mentions.set(mentionId, mention)
console.log(`WebMention processed successfully: ${mentionId}`)
return {
success: true,
mention,
status: mention.moderationStatus
}
} catch (error) {
console.error('Error processing WebMention:', error)
return {
success: false,
error: error.message,
status: 'error'
}
}
}
// Verify WebMention
private async verifyWebMention(sourceUrl: string, targetUrl: string): Promise<{
valid: boolean
error?: string
}> {
try {
// Validate URLs
const source = new URL(sourceUrl)
const target = new URL(targetUrl)
// Check if target belongs to our website
if (!target.origin.includes(new URL(this.websiteUrl).origin)) {
return {
valid: false,
error: 'Target URL does not belong to this website'
}
}
// Fetch source content
const response = await fetch(sourceUrl, {
method: 'GET',
headers: {
'Accept': 'text/html,application/xhtml+xml',
'User-Agent': 'WebMentions-Comment-System/1.0'
}
})
if (!response.ok) {
return {
valid: false,
error: `HTTP ${response.status}: ${response.statusText}`
}
}
const html = await response.text()
// Verify that source mentions target
const mentionsTarget = this.checkForMention(html, targetUrl)
if (!mentionsTarget) {
return {
valid: false,
error: 'Source does not mention target URL'
}
}
// Additional verification checks
const verificationResults = await Promise.all([
this.checkSourceReputation(sourceUrl),
this.validateContent(html),
this.checkRateLimit(sourceUrl)
])
const hasInvalidResult = verificationResults.some(result => !result.valid)
if (hasInvalidResult) {
const invalidResult = verificationResults.find(result => !result.valid)
return {
valid: false,
error: invalidResult.error
}
}
return { valid: true }
} catch (error) {
return {
valid: false,
error: error.message
}
}
}
// Check if source mentions target
private checkForMention(html: string, targetUrl: string): boolean {
const parser = new DOMParser()
const doc = parser.parseFromString(html, 'text/html')
// Check for direct links
const links = doc.querySelectorAll(`a[href="${targetUrl}"], a[href="${targetUrl}/"]`)
if (links.length > 0) return true
// Check for URL in text content
const textContent = doc.body.textContent || ''
if (textContent.includes(targetUrl)) return true
// Check for URL without protocol
const targetWithoutProtocol = targetUrl.replace(/^https?:\/\//, '')
if (textContent.includes(targetWithoutProtocol)) return true
return false
}
// Extract content from source
private async extractWebMentionContent(sourceUrl: string): Promise<WebMentionContent> {
try {
const response = await fetch(sourceUrl, {
method: 'GET',
headers: {
'Accept': 'text/html,application/xhtml+xml'
}
})
const html = await response.text()
const parser = new DOMParser()
const doc = parser.parseFromString(html, 'text/html')
// Extract structured data using various parsers
let content: WebMentionContent = {
title: '',
text: '',
excerpt: '',
url: sourceUrl,
type: 'mention',
language: doc.documentElement.getAttribute('lang') || 'en'
}
for (const parser of this.contentParsers) {
try {
const parsed = await parser.parse(doc, sourceUrl)
if (parsed) {
content = { ...content, ...parsed }
break // Use first successful parser
}
} catch (error) {
console.warn('Content parser failed:', error)
}
}
// Fallback extraction
if (!content.title || !content.text) {
content.title = content.title || this.extractTitle(doc)
content.text = content.text || this.extractText(doc)
content.excerpt = this.createExcerpt(content.text)
}
return content
} catch (error) {
console.error('Error extracting content:', error)
return {
title: 'Unknown',
text: 'Failed to extract content',
excerpt: 'Failed to extract content',
url: sourceUrl,
type: 'mention'
}
}
}
// Extract author information
private async extractAuthorInfo(sourceUrl: string): Promise<WebMentionAuthor> {
try {
const response = await fetch(sourceUrl)
const html = await response.text()
const parser = new DOMParser()
const doc = parser.parseFromString(html, 'text/html')
const author: WebMentionAuthor = {
name: 'Anonymous'
}
// Try microformats2 first
const authorCard = doc.querySelector('.h-card')
if (authorCard) {
author.name = this.getTextFromSelector(authorCard, '.p-name') || author.name
author.url = this.getAttrFromSelector(authorCard, '.u-url') || author.url
author.photo = this.getAttrFromSelector(authorCard, '.u-photo') || author.photo
author.email = this.getTextFromSelector(authorCard, '.u-email') || author.email
}
// Fallback to other meta tags
if (!author.name) {
author.name = this.getMetaContent(doc, 'author') ||
this.getMetaContent(doc, 'article:author') ||
'Anonymous'
}
// Extract social profiles
author.social = {
twitter: this.extractSocialProfile(doc, 'twitter'),
mastodon: this.extractSocialProfile(doc, 'mastodon'),
github: this.extractSocialProfile(doc, 'github')
}
return author
} catch (error) {
console.error('Error extracting author:', error)
return { name: 'Anonymous' }
}
}
// Detect mention type
private detectMentionType(content: WebMentionContent, sourceUrl: string, targetUrl: string): WebMentionContent['type'] {
// Check for in-reply-to relationship
if (content.text.toLowerCase().includes('reply') ||
content.text.toLowerCase().includes('re:') ||
this.isReplyTo(sourceUrl, targetUrl)) {
return 'reply'
}
// Check for like/repost indicators
if (content.text.toLowerCase().includes('like') ||
content.text.toLowerCase().includes('❤️') ||
content.text.toLowerCase().includes('👍')) {
return 'like'
}
if (content.text.toLowerCase().includes('repost') ||
content.text.toLowerCase().includes('🔄')) {
return 'repost'
}
// Default to post/mention
return content.text.length > 200 ? 'post' : 'mention'
}
// Calculate spam score
private async calculateSpamScore(mention: WebMention): Promise<number> {
let spamScore = 0
for (const filter of this.spamFilters) {
try {
const score = await filter.analyze(mention)
spamScore += score
} catch (error) {
console.warn('Spam filter failed:', error)
}
}
return Math.min(100, Math.max(0, spamScore))
}
// Determine moderation status
private determineModerationStatus(mention: WebMention): WebMention['moderationStatus'] {
if (mention.spamScore >= 80) {
return 'spam'
} else if (mention.spamScore >= 60) {
return 'rejected'
} else if (mention.spamScore >= 30) {
return 'pending'
} else {
return 'approved'
}
}
// Apply moderation rules
private applyModerationRules(mention: WebMention): void {
for (const rule of this.moderationRules) {
if (this.evaluateRule(rule, mention)) {
// Apply rule actions
if (rule.actions.autoApprove) {
mention.approved = true
mention.moderationStatus = 'approved'
}
if (rule.actions.autoReject) {
mention.approved = false
mention.moderationStatus = 'rejected'
}
if (rule.actions.markSpam) {
mention.spamScore = 100
mention.moderationStatus = 'spam'
}
}
}
}
// Evaluate moderation rule
private evaluateRule(rule: ModerationRule, mention: WebMention): boolean {
const conditions = rule.conditions
// Check keywords
if (conditions.keywords) {
const content = mention.content.text.toLowerCase()
const hasKeywords = conditions.keywords.some(keyword =>
content.includes(keyword.toLowerCase())
)
if (hasKeywords) return true
}
// Check content length
if (conditions.minLength && mention.content.text.length < conditions.minLength) {
return true
}
if (conditions.maxLength && mention.content.text.length > conditions.maxLength) {
return true
}
// Check domain restrictions
try {
const sourceDomain = new URL(mention.source).hostname
if (conditions.allowedDomains && !conditions.allowedDomains.includes(sourceDomain)) {
return true
}
if (conditions.blockedDomains && conditions.blockedDomains.includes(sourceDomain)) {
return true
}
if (conditions.requireHttps && !mention.source.startsWith('https://')) {
return true
}
} catch (error) {
return true // Invalid URL
}
// Check author requirements
if (conditions.requireName && (!mention.author.name || mention.author.name === 'Anonymous')) {
return true
}
return false
}
// Get mentions for target
getMentions(targetUrl: string, options: {
approved?: boolean
type?: WebMentionContent['type']
limit?: number
offset?: number
sortBy?: 'date' | 'relevance'
sortOrder?: 'asc' | 'desc'
} = {}): WebMention[] {
let mentions = Array.from(this.mentions.values())
.filter(mention => mention.target === targetUrl)
// Apply filters
if (options.approved !== undefined) {
mentions = mentions.filter(mention => mention.approved === options.approved)
}
if (options.type) {
mentions = mentions.filter(mention => mention.content.type === options.type)
}
// Sort
const sortBy = options.sortBy || 'date'
const sortOrder = options.sortOrder || 'desc'
mentions.sort((a, b) => {
let comparison = 0
if (sortBy === 'date') {
comparison = new Date(a.receivedAt).getTime() - new Date(b.receivedAt).getTime()
} else if (sortBy === 'relevance') {
// Simple relevance scoring based on content type and reactions
const scoreA = this.calculateRelevanceScore(a)
const scoreB = this.calculateRelevanceScore(b)
comparison = scoreA - scoreB
}
return sortOrder === 'desc' ? -comparison : comparison
})
// Apply pagination
if (options.offset) {
mentions = mentions.slice(options.offset)
}
if (options.limit) {
mentions = mentions.slice(0, options.limit)
}
return mentions
}
// Get mention statistics
getMentionStats(targetUrl: string) {
const mentions = this.getMentions(targetUrl)
const approved = this.getMentions(targetUrl, { approved: true })
const pending = this.getMentions(targetUrl, { approved: false })
const stats = {
total: mentions.length,
approved: approved.length,
pending: pending.length,
spam: mentions.filter(m => m.moderationStatus === 'spam').length,
rejected: mentions.filter(m => m.moderationStatus === 'rejected').length,
byType: this.groupByType(mentions),
byDate: this.groupByDate(mentions),
averageSpamScore: mentions.length > 0 ?
mentions.reduce((sum, m) => sum + m.spamScore, 0) / mentions.length : 0
}
return stats
}
// Utility methods
private generateMentionId(source: string, target: string): string {
return btoa(`${source}|${target}`).replace(/[+/=]/g, '').substring(0, 16)
}
private createExcerpt(text: string, length: number = 200): string {
const cleaned = text.replace(/<[^>]*>/g, '').trim()
return cleaned.length > length ? cleaned.substring(0, length) + '...' : cleaned
}
private extractTitle(doc: Document): string {
const selectors = ['meta[property="og:title"]', 'title', 'h1']
for (const selector of selectors) {
const element = doc.querySelector(selector)
if (element) {
const content = element.getAttribute('content') || element.textContent
if (content) return content.trim()
}
}
return 'Untitled'
}
private extractText(doc: Document): string {
const selectors = [
'.e-content',
'article',
'main',
'[role="main"]',
'body'
]
for (const selector of selectors) {
const element = doc.querySelector(selector)
if (element) {
return element.textContent || ''
}
}
return ''
}
private getTextFromSelector(element: Element, selector: string): string | null {
const el = element.querySelector(selector)
return el ? el.textContent?.trim() || null : null
}
private getAttrFromSelector(element: Element, selector: string): string | null {
const el = element.querySelector(selector)
return el ? el.getAttribute('href') || el.getAttribute('src') : null
}
private getMetaContent(doc: Document, name: string): string | null {
const selector = `meta[name="${name}"], meta[property="${name}"]`
const element = doc.querySelector(selector)
return element ? element.getAttribute('content') : null
}
private extractSocialProfile(doc: Document, platform: string): string | null {
const selectors = [
`a[href*="${platform}.com"]`,
`a[href*="${platform}.org"]`,
`[rel="me"][href*="${platform}"]`
]
for (const selector of selectors) {
const element = doc.querySelector(selector)
if (element) {
return element.getAttribute('href')
}
}
return null
}
private calculateRelevanceScore(mention: WebMention): number {
let score = 0
// Content type scoring
const typeScores = { reply: 10, post: 8, like: 3, repost: 5, mention: 2 }
score += typeScores[mention.content.type] || 0
// Reaction scoring
score += mention.reactions.likes * 2
score += mention.reactions.reposts * 3
score += mention.reactions.replies * 4
// Content quality scoring
if (mention.content.text.length > 100) score += 2
if (mention.author.name !== 'Anonymous') score += 1
if (mention.author.photo) score += 1
return score
}
private groupByType(mentions: WebMention[]): Record<string, number> {
return mentions.reduce((groups, mention) => {
const type = mention.content.type
groups[type] = (groups[type] || 0) + 1
return groups
}, {} as Record<string, number>)
}
private groupByDate(mentions: WebMention[]): Record<string, number> {
return mentions.reduce((groups, mention) => {
const date = new Date(mention.receivedAt).toISOString().split('T')[0]
groups[date] = (groups[date] || 0) + 1
return groups
}, {} as Record<string, number>)
}
// Setup default configurations
private setupDefaultModerationRules(): void {
this.moderationRules = [
{
id: 'short-content',
name: 'Reject very short content',
type: 'rejection',
conditions: {
minLength: 10
},
actions: {
autoReject: true
}
},
{
id: 'spam-keywords',
name: 'Block spam keywords',
type: 'spam',
conditions: {
keywords: ['viagra', 'casino', 'lottery', 'spam', 'buy now']
},
actions: {
markSpam: true
}
},
{
id: 'require-https',
name: 'Require HTTPS sources',
type: 'rejection',
conditions: {
requireHttps: true
},
actions: {
requireModeration: true
}
}
]
}
private setupDefaultSpamFilters(): void {
this.spamFilters = [
new KeywordSpamFilter(),
new LinkSpamFilter(),
new RepeatedContentFilter(),
new LanguageFilter()
]
}
private setupDefaultContentParsers(): void {
this.contentParsers = [
new MicroformatsParser(),
new OpenGraphParser(),
new JSONLDParser(),
new TwitterCardParser()
]
}
// Additional verification methods
private async checkSourceReputation(sourceUrl: string): Promise<{ valid: boolean; error?: string }> {
// Implement source reputation checking
return { valid: true }
}
private async validateContent(html: string): Promise<{ valid: boolean; error?: string }> {
// Implement content validation
return { valid: true }
}
private async checkRateLimit(sourceUrl: string): Promise<{ valid: boolean; error?: string }> {
// Implement rate limiting
return { valid: true }
}
private isReplyTo(sourceUrl: string, targetUrl: string): boolean {
// Check if source is a reply to target
return sourceUrl.includes('/reply/') || sourceUrl.includes('?in-reply-to=')
}
}
// Supporting classes
class SpamFilter {
async analyze(mention: WebMention): Promise<number> {
return 0
}
}
class KeywordSpamFilter extends SpamFilter {
private spamKeywords = [
'click here', 'buy now', 'free money', 'make money fast',
'viagra', 'casino', 'lottery', 'winner', 'congratulations'
]
async analyze(mention: WebMention): Promise<number> {
const text = mention.content.text.toLowerCase()
let score = 0
for (const keyword of this.spamKeywords) {
if (text.includes(keyword)) {
score += 20
}
}
return score
}
}
class LinkSpamFilter extends SpamFilter {
async analyze(mention: WebMention): Promise<number> {
const linkRegex = /https?:\/\/[^\s]+/g
const links = mention.content.text.match(linkRegex) || []
if (links.length > 5) return 30
if (links.length > 3) return 15
return 0
}
}
class RepeatedContentFilter extends SpamFilter {
async analyze(mention: WebMention): Promise<number> {
// Check for repeated characters or suspicious patterns
const repeatedChars = /(.)\1{4,}/.test(mention.content.text)
if (repeatedChars) return 25
const excessiveCaps = (mention.content.text.match(/[A-Z]/g) || []).length / mention.content.text.length
if (excessiveCaps > 0.5) return 20
return 0
}
}
class LanguageFilter extends SpamFilter {
async analyze(mention: WebMention): Promise<number> {
// Check for language mismatches or suspicious text
const suspiciousWords = ['asdf', 'qwerty', 'test', 'demo']
const text = mention.content.text.toLowerCase()
for (const word of suspiciousWords) {
if (text.includes(word)) {
return 15
}
}
return 0
}
}
class ContentParser {
async parse(doc: Document, sourceUrl: string): Promise<Partial<WebMentionContent>> {
return {}
}
}
class MicroformatsParser extends ContentParser {
async parse(doc: Document, sourceUrl: string): Promise<Partial<WebMentionContent>> {
const hEntry = doc.querySelector('.h-entry')
if (!hEntry) return {}
const content: Partial<WebMentionContent> = {
title: this.getText(hEntry, '.p-name'),
text: this.getText(hEntry, '.e-content'),
url: this.getAttr(hEntry, '.u-url') || sourceUrl
}
const published = this.getAttr(hEntry, '.dt-published')
if (published) {
content.published = published
}
return content
}
private getText(element: Element, selector: string): string {
const el = element.querySelector(selector)
return el ? el.textContent?.trim() || '' : ''
}
private getAttr(element: Element, selector: string): string {
const el = element.querySelector(selector)
return el ? (el.getAttribute('href') || el.getAttribute('datetime') || '') : ''
}
}
class OpenGraphParser extends ContentParser {
async parse(doc: Document): Promise<Partial<WebMentionContent>> {
const content: Partial<WebMentionContent> = {}
const title = doc.querySelector('meta[property="og:title"]')
if (title) {
content.title = title.getAttribute('content') || ''
}
const description = doc.querySelector('meta[property="og:description"]')
if (description) {
content.text = description.getAttribute('content') || ''
}
return content
}
}
class JSONLDParser extends ContentParser {
async parse(doc: Document): Promise<Partial<WebMentionContent>> {
const scripts = doc.querySelectorAll('script[type="application/ld+json"]')
for (const script of scripts) {
try {
const data = JSON.parse(script.textContent || '')
if (data['@type'] === 'BlogPosting' || data['@type'] === 'Article') {
return {
title: data.headline || '',
text: data.description || data.text || '',
published: data.datePublished || ''
}
}
} catch (error) {
continue
}
}
return {}
}
}
class TwitterCardParser extends ContentParser {
async parse(doc: Document): Promise<Partial<WebMentionContent>> {
const content: Partial<WebMentionContent> = {}
const title = doc.querySelector('meta[name="twitter:title"]')
if (title) {
content.title = title.getAttribute('content') || ''
}
const description = doc.querySelector('meta[name="twitter:description"]')
if (description) {
content.text = description.getAttribute('content') || ''
}
return content
}
}
export default WebMentionsCommentSystem
💻 Agrégateur de Réseaux Sociaux typescript
🔴 complex
⭐⭐⭐⭐⭐
Agréger le contenu de plusieurs plateformes sociales en utilisant WebMentions et ActivityPub
⏱️ 50 min
🏷️ webmentions, social media, aggregator, activitypub
Prerequisites:
Advanced TypeScript, Social media APIs, ActivityPub protocol, Data aggregation
// Social Network Aggregator with WebMentions and ActivityPub
// Aggregate content from Twitter, Mastodon, and other federated platforms
interface SocialPost {
id: string
platform: 'twitter' | 'mastodon' | 'bluesky' | 'web' | 'activitypub'
url: string
content: string
author: SocialAuthor
publishedAt: Date
likes?: number
shares?: number
replies?: number
hashtags?: string[]
mentions?: string[]
media?: SocialMedia[]
replyTo?: string
thread?: SocialPost[]
webmentions?: WebMention[]
}
interface SocialAuthor {
id: string
name: string
username: string
avatar?: string
url?: string
verified?: boolean
followers?: number
bio?: string
location?: string
website?: string
social?: {
twitter?: string
mastodon?: string
bluesky?: string
github?: string
}
}
interface SocialMedia {
type: 'image' | 'video' | 'audio' | 'link'
url: string
thumbnail?: string
description?: string
width?: number
height?: number
duration?: number
}
interface WebMention {
id: string
source: string
target: string
content: string
author: SocialAuthor
publishedAt: Date
type: 'like' | 'repost' | 'reply' | 'mention'
}
class SocialNetworkAggregator {
private posts: Map<string, SocialPost> = new Map()
private authors: Map<string, SocialAuthor> = new Map()
private webmentionProcessor: WebMentionsCommentSystem
private activityPubClient: ActivityPubClient
constructor(
private config: {
websiteUrl: string
platforms: SocialPlatform[]
enableWebmentions: boolean
enableActivityPub: boolean
aggregationRules: AggregationRule[]
}
) {
this.webmentionProcessor = new WebMentionsCommentSystem(this.config.websiteUrl)
this.activityPubClient = new ActivityPubClient(this.config.websiteUrl)
}
// Aggregate content from all configured platforms
async aggregateContent(targetUrl: string): Promise<{
posts: SocialPost[]
authors: SocialAuthor[]
statistics: AggregationStats
}> {
console.log(`Aggregating content for: ${targetUrl}`)
const allPosts: SocialPost[] = []
const allAuthors: Map<string, SocialAuthor> = new Map()
// Aggregate from each platform
for (const platform of this.config.platforms) {
try {
const posts = await this.aggregateFromPlatform(platform, targetUrl)
allPosts.push(...posts)
// Collect authors
posts.forEach(post => {
if (!allAuthors.has(post.author.id)) {
allAuthors.set(post.author.id, post.author)
}
})
} catch (error) {
console.error(`Error aggregating from ${platform.name}:`, error)
}
}
// Process WebMentions
if (this.config.enableWebmentions) {
const webmentionPosts = await this.aggregateWebmentions(targetUrl)
allPosts.push(...webmentionPosts)
}
// Process ActivityPub interactions
if (this.config.enableActivityPub) {
const activityPubPosts = await this.aggregateActivityPub(targetUrl)
allPosts.push(...activityPubPosts)
}
// Apply aggregation rules
const filteredPosts = this.applyAggregationRules(allPosts)
// Sort and rank posts
const rankedPosts = this.rankPosts(filteredPosts)
// Calculate statistics
const statistics = this.calculateStatistics(rankedPosts)
return {
posts: rankedPosts,
authors: Array.from(allAuthors.values()),
statistics
}
}
// Aggregate from specific platform
private async aggregateFromPlatform(platform: SocialPlatform, targetUrl: string): Promise<SocialPost[]> {
switch (platform.type) {
case 'twitter':
return this.aggregateTwitter(platform, targetUrl)
case 'mastodon':
return this.aggregateMastodon(platform, targetUrl)
case 'bluesky':
return this.aggregateBluesky(platform, targetUrl)
case 'activitypub':
return this.aggregateActivityPubInstance(platform, targetUrl)
default:
return []
}
}
// Aggregate Twitter content
private async aggregateTwitter(platform: SocialPlatform, targetUrl: string): Promise<SocialPost[]> {
console.log('Aggregating from Twitter...')
// Search for tweets mentioning the target URL
const searchQuery = this.buildSearchQuery(targetUrl)
const tweets = await this.searchTwitterTweets(searchQuery, platform.apiKey)
return tweets.map(tweet => this.convertTwitterToSocialPost(tweet))
}
// Aggregate Mastodon content
private async aggregateMastodon(platform: SocialPlatform, targetUrl: string): Promise<SocialPost[]> {
console.log('Aggregating from Mastodon...')
const posts: SocialPost[] = []
// Search across multiple Mastodon instances
for (const instance of platform.instances || []) {
try {
const searchResults = await this.searchMastodonInstance(instance, targetUrl)
posts.push(...searchResults.map(post => this.convertMastodonToSocialPost(post, instance)))
} catch (error) {
console.warn(`Failed to search Mastodon instance ${instance}:`, error)
}
}
return posts
}
// Aggregate Bluesky content
private async aggregateBluesky(platform: SocialPlatform, targetUrl: string): Promise<SocialPost[]> {
console.log('Aggregating from Bluesky...')
const searchResults = await this.searchBluesky(targetUrl, platform.apiKey)
return searchResults.map(post => this.convertBlueskyToSocialPost(post))
}
// Aggregate WebMentions
private async aggregateWebmentions(targetUrl: string): Promise<SocialPost[]> {
console.log('Aggregating WebMentions...')
const mentions = this.webmentionProcessor.getMentions(targetUrl, { approved: true })
return mentions.map(mention => ({
id: mention.id,
platform: 'web' as const,
url: mention.source,
content: mention.content.text,
author: this.convertWebmentionAuthor(mention.author),
publishedAt: new Date(mention.published),
type: 'mention',
webmentions: [mention]
}))
}
// Aggregate ActivityPub content
private async aggregateActivityPub(targetUrl: string): Promise<SocialPost[]> {
console.log('Aggregating from ActivityPub...')
const activities = await this.activityPubClient.searchInteractions(targetUrl)
return activities.map(activity => this.convertActivityPubToSocialPost(activity))
}
// Search Twitter for tweets
private async searchTwitterTweets(query: string, apiKey: string): Promise<TwitterTweet[]> {
try {
const response = await fetch(`https://api.twitter.com/2/tweets/search/recent?query=${encodeURIComponent(query)}&expansions=author_id,attachments.media&tweet.fields=public_metrics,created_at&user.fields=username,name,profile_image_url,verified,public_metrics`, {
headers: {
'Authorization': `Bearer ${apiKey}`
}
})
if (!response.ok) {
throw new Error(`Twitter API error: ${response.statusText}`)
}
const data = await response.json()
return data.data || []
} catch (error) {
console.error('Twitter search error:', error)
return []
}
}
// Search Mastodon instance
private async searchMastodonInstance(instance: string, targetUrl: string): Promise<MastodonPost[]> {
try {
const response = await fetch(`https://${instance}/api/v2/search?q=${encodeURIComponent(targetUrl)}&resolve=true&type=statuses`, {
headers: {
'Accept': 'application/json'
}
})
if (!response.ok) {
throw new Error(`Mastodon API error: ${response.statusText}`)
}
const data = await response.json()
return data.statuses || []
} catch (error) {
console.error(`Mastodon search error for ${instance}:`, error)
return []
}
}
// Search Bluesky
private async searchBluesky(query: string, apiKey: string): Promise<BlueskyPost[]> {
try {
const response = await fetch(`https://api.bsky.app/xrpc/app.bsky.feed.searchPosts?q=${encodeURIComponent(query)}`, {
headers: {
'Authorization': `Bearer ${apiKey}`
}
})
if (!response.ok) {
throw new Error(`Bluesky API error: ${response.statusText}`)
}
const data = await response.json()
return data.posts || []
} catch (error) {
console.error('Bluesky search error:', error)
return []
}
}
// Build search query for platform
private buildSearchQuery(targetUrl: string): string {
// Extract domain for better searching
const url = new URL(targetUrl)
const domain = url.hostname
// Build comprehensive search query
const queryTerms = [
`\"${targetUrl.replace(/^https?:\/\//, '')}\"`, // Exact URL
domain, // Domain name
url.pathname, // Path
'lang:en' // English results only
]
// Add negative terms to filter out noise
const negativeTerms = ['-filter:replies', '-filter:retweets']
return [...queryTerms, ...negativeTerms].join(' ')
}
// Convert platform-specific formats to SocialPost
private convertTwitterToSocialPost(tweet: TwitterTweet): SocialPost {
return {
id: tweet.id,
platform: 'twitter',
url: `https://twitter.com/i/web/status/${tweet.id}`,
content: tweet.text,
author: {
id: tweet.author_id,
name: tweet.author.name,
username: tweet.author.username,
avatar: tweet.author.profile_image_url,
url: `https://twitter.com/${tweet.author.username}`,
verified: tweet.author.verified,
followers: tweet.author.public_metrics?.followers_count
},
publishedAt: new Date(tweet.created_at),
likes: tweet.public_metrics?.like_count,
shares: tweet.public_metrics?.retweet_count,
replies: tweet.public_metrics?.reply_count,
hashtags: this.extractHashtags(tweet.text),
mentions: this.extractMentions(tweet.text),
media: this.convertTwitterMedia(tweet.attachments)
}
}
private convertMastodonToSocialPost(post: MastodonPost, instance: string): SocialPost {
return {
id: post.id,
platform: 'mastodon',
url: post.url,
content: post.content,
author: {
id: post.account.id,
name: post.account.display_name,
username: post.account.username,
avatar: post.account.avatar,
url: post.account.url,
followers: post.account.followers_count,
bio: post.account.note
},
publishedAt: new Date(post.created_at),
likes: post.favourites_count,
shares: post.reblogs_count,
replies: post.replies_count,
hashtags: post.tags?.map(tag => tag.name),
mentions: post.mentions?.map(mention => mention.acct),
media: this.convertMastodonMedia(post.media_attachments),
replyTo: post.in_reply_to_id
}
}
private convertBlueskyToSocialPost(post: BlueskyPost): SocialPost {
return {
id: post.uri,
platform: 'bluesky',
url: `https://bsky.app/profile/${post.author.handle}/post/${post.uri.split('/').pop()}`,
content: post.record.text,
author: {
id: post.author.did,
name: post.author.displayName,
username: post.author.handle,
avatar: post.author.avatar,
url: `https://bsky.app/profile/${post.author.handle}`,
followers: post.author.followersCount
},
publishedAt: new Date(post.record.createdAt),
likes: post.likeCount,
replies: post.replyCount,
reposts: post.repostCount,
hashtags: this.extractBlueskyTags(post.record.text),
media: this.convertBlueskyMedia(post.embed)
}
}
private convertWebmentionAuthor(author: any): SocialAuthor {
return {
id: author.url || 'unknown',
name: author.name || 'Anonymous',
username: author.name?.toLowerCase().replace(/\s+/g, '') || 'anonymous',
avatar: author.photo,
url: author.url,
social: author.social
}
}
private convertActivityPubToSocialPost(activity: ActivityPubActivity): SocialPost {
const object = activity.object as ActivityPubObject
return {
id: activity.id,
platform: 'activitypub',
url: object.url,
content: object.content,
author: {
id: object.attributedTo.id,
name: object.attributedTo.name,
username: object.attributedTo.preferredUsername,
avatar: object.attributedTo.icon?.url,
url: object.attributedTo.url,
followers: object.attributedTo.followersCount
},
publishedAt: new Date(object.published),
likes: object.likeCount || 0,
shares: object.shareCount || 0,
replies: object.replyCount || 0,
hashtags: object.tag?.filter(tag => tag.type === 'Hashtag').map(tag => tag.name),
mentions: object.tag?.filter(tag => tag.type === 'Mention').map(tag => tag.name)
}
}
// Media conversion methods
private convertTwitterMedia(attachments: TwitterMedia[]): SocialMedia[] {
if (!attachments) return []
return attachments.map(media => ({
type: media.type === 'photo' ? 'image' : 'video',
url: media.url,
description: media.alt_text,
width: media.width,
height: media.height
}))
}
private convertMastodonMedia(attachments: MastodonMedia[]): SocialMedia[] {
if (!attachments) return []
return attachments.map(media => ({
type: media.type,
url: media.url,
thumbnail: media.preview_url,
description: media.description,
width: media.meta?.original?.width,
height: media.meta?.original?.height,
duration: media.meta?.duration
}))
}
private convertBlueskyMedia(embed: any): SocialMedia[] {
if (!embed) return []
if (embed.$type === 'app.bsky.embed.images.view') {
return embed.images.map((img: any) => ({
type: 'image',
url: img.fullsize,
thumbnail: img.thumb,
description: img.alt,
width: img.aspectRatio?.width,
height: img.aspectRatio?.height
}))
}
return []
}
// Text processing utilities
private extractHashtags(text: string): string[] {
const hashtags = text.match(/#[\w]+/g)
return hashtags ? hashtags.map(tag => tag.substring(1)) : []
}
private extractMentions(text: string): string[] {
const mentions = text.match(/@[\w]+/g)
return mentions ? mentions.map(mention => mention.substring(1)) : []
}
private extractBlueskyTags(text: string): string[] {
// Bluesky doesn't use hashtags in the same way
return []
}
// Apply aggregation rules
private applyAggregationRules(posts: SocialPost[]): SocialPost[] {
let filteredPosts = [...posts]
for (const rule of this.config.aggregationRules) {
filteredPosts = this.applyRule(filteredPosts, rule)
}
// Remove duplicates
filteredPosts = this.deduplicatePosts(filteredPosts)
return filteredPosts
}
private applyRule(posts: SocialPost[], rule: AggregationRule): SocialPost[] {
return posts.filter(post => {
// Apply rule logic
if (rule.minFollowers && post.author.followers && post.author.followers < rule.minFollowers) {
return false
}
if (rule.blockedWords) {
const text = post.content.toLowerCase()
if (rule.blockedWords.some(word => text.includes(word.toLowerCase()))) {
return false
}
}
if (rule.requireVerified && !post.author.verified) {
return false
}
if (rule.maxAge) {
const age = Date.now() - post.publishedAt.getTime()
if (age > rule.maxAge) {
return false
}
}
return true
})
}
// Remove duplicate posts
private deduplicatePosts(posts: SocialPost[]): SocialPost[] {
const seen = new Set<string>()
return posts.filter(post => {
// Create a unique key based on content and author
const key = `${post.author.username}:${post.content.substring(0, 100)}`
if (seen.has(key)) {
return false
}
seen.add(key)
return true
})
}
// Rank posts by relevance and engagement
private rankPosts(posts: SocialPost[]): SocialPost[] {
return posts.sort((a, b) => {
const scoreA = this.calculatePostScore(a)
const scoreB = this.calculatePostScore(b)
return scoreB - scoreA
})
}
private calculatePostScore(post: SocialPost): number {
let score = 0
// Base score for content
score += post.content.length > 100 ? 10 : 5
// Engagement metrics
score += (post.likes || 0) * 2
score += (post.shares || 0) * 3
score += (post.replies || 0) * 4
// Author credibility
if (post.author.verified) score += 20
if (post.author.followers && post.author.followers > 1000) score += 10
// Recency (more recent posts get higher scores)
const ageHours = (Date.now() - post.publishedAt.getTime()) / (1000 * 60 * 60)
score += Math.max(0, 20 - ageHours)
// Platform weight
const platformWeights = { twitter: 1.2, mastodon: 1.0, bluesky: 1.1, web: 0.8, activitypub: 1.0 }
score *= platformWeights[post.platform] || 1.0
return score
}
// Calculate aggregation statistics
private calculateStatistics(posts: SocialPost[]): AggregationStats {
const stats: AggregationStats = {
totalPosts: posts.length,
byPlatform: this.groupByPlatform(posts),
byDate: this.groupByDate(posts),
topAuthors: this.getTopAuthors(posts),
engagementMetrics: this.calculateEngagementMetrics(posts),
hashtags: this.getTopHashtags(posts),
timeRange: this.getTimeRange(posts)
}
return stats
}
private groupByPlatform(posts: SocialPost[]): Record<string, number> {
return posts.reduce((groups, post) => {
groups[post.platform] = (groups[post.platform] || 0) + 1
return groups
}, {} as Record<string, number>)
}
private groupByDate(posts: SocialPost[]): Record<string, number> {
return posts.reduce((groups, post) => {
const date = post.publishedAt.toISOString().split('T')[0]
groups[date] = (groups[date] || 0) + 1
return groups
}, {} as Record<string, number>)
}
private getTopAuthors(posts: SocialPost[]): Array<{ author: SocialAuthor; postCount: number }> {
const authorCounts = posts.reduce((counts, post) => {
counts[post.author.id] = {
author: post.author,
postCount: (counts[post.author.id]?.postCount || 0) + 1
}
return counts
}, {} as Record<string, { author: SocialAuthor; postCount: number }>)
return Object.values(authorCounts)
.sort((a, b) => b.postCount - a.postCount)
.slice(0, 10)
}
private calculateEngagementMetrics(posts: SocialPost[]) {
const totals = posts.reduce((metrics, post) => {
metrics.totalLikes += post.likes || 0
metrics.totalShares += post.shares || 0
metrics.totalReplies += post.replies || 0
return metrics
}, { totalLikes: 0, totalShares: 0, totalReplies: 0 })
return {
...totals,
averageLikes: totals.totalLikes / posts.length,
averageShares: totals.totalShares / posts.length,
averageReplies: totals.totalReplies / posts.length
}
}
private getTopHashtags(posts: SocialPost[]): Array<{ hashtag: string; count: number }> {
const hashtagCounts = posts.reduce((counts, post) => {
if (post.hashtags) {
post.hashtags.forEach(hashtag => {
counts[hashtag] = (counts[hashtag] || 0) + 1
})
}
return counts
}, {} as Record<string, number>)
return Object.entries(hashtagCounts)
.map(([hashtag, count]) => ({ hashtag, count }))
.sort((a, b) => b.count - a.count)
.slice(0, 20)
}
private getTimeRange(posts: SocialPost[]): { earliest: Date; latest: Date } {
if (posts.length === 0) {
return {
earliest: new Date(),
latest: new Date()
}
}
const dates = posts.map(post => post.publishedAt)
return {
earliest: new Date(Math.min(...dates.map(d => d.getTime()))),
latest: new Date(Math.max(...dates.map(d => d.getTime())))
}
}
}
// Supporting interfaces and types
interface SocialPlatform {
type: 'twitter' | 'mastodon' | 'bluesky' | 'activitypub'
name: string
apiKey?: string
instances?: string[] // For Mastodon
}
interface AggregationRule {
name: string
minFollowers?: number
blockedWords?: string[]
requireVerified?: boolean
maxAge?: number // milliseconds
}
interface AggregationStats {
totalPosts: number
byPlatform: Record<string, number>
byDate: Record<string, number>
topAuthors: Array<{ author: SocialAuthor; postCount: number }>
engagementMetrics: {
totalLikes: number
totalShares: number
totalReplies: number
averageLikes: number
averageShares: number
averageReplies: number
}
hashtags: Array<{ hashtag: string; count: number }>
timeRange: { earliest: Date; latest: Date }
}
// Placeholder interfaces for platform-specific data
interface TwitterTweet {
id: string
text: string
author_id: string
created_at: string
public_metrics?: {
like_count: number
retweet_count: number
reply_count: number
}
author: {
id: string
name: string
username: string
profile_image_url: string
verified: boolean
public_metrics?: {
followers_count: number
}
}
attachments?: TwitterMedia[]
}
interface TwitterMedia {
media_key: string
type: 'photo' | 'video'
url: string
preview_image_url?: string
width?: number
height?: number
alt_text?: string
}
interface MastodonPost {
id: string
url: string
content: string
created_at: string
account: {
id: string
display_name: string
username: string
avatar: string
url: string
followers_count: number
note: string
}
favourites_count: number
reblogs_count: number
replies_count: number
tags?: Array<{ name: string }>
mentions?: Array<{ acct: string }>
media_attachments?: MastodonMedia[]
in_reply_to_id?: string
}
interface MastodonMedia {
id: string
type: 'image' | 'video' | 'gifv' | 'audio'
url: string
preview_url?: string
description?: string
meta?: {
original?: {
width: number
height: number
}
duration?: number
}
}
interface BlueskyPost {
uri: string
author: {
did: string
handle: string
displayName?: string
avatar?: string
followersCount?: number
}
record: {
text: string
createdAt: string
}
likeCount: number
replyCount: number
repostCount: number
embed?: any
}
interface ActivityPubClient {
searchInteractions(targetUrl: string): Promise<ActivityPubActivity[]>
}
interface ActivityPubActivity {
id: string
type: string
object: ActivityPubObject
}
interface ActivityPubObject {
id: string
url?: string
content: string
published: string
attributedTo: {
id: string
name: string
preferredUsername: string
icon?: { url: string }
url: string
followersCount?: number
}
likeCount?: number
shareCount?: number
replyCount?: number
tag?: Array<{ type: string; name: string }>
}
export default SocialNetworkAggregator