Nes
/
neo-scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update Mastodon.ts 

Cleaned up code a bit.
master
Nes370 2026-10-11 16:41:41 -07:00
parent a6a4308276
commit 8004d0a183
1 changed files with 44 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
src/engines/Mastodon.ts
View File

@ -1,77 +1,45 @@
import {
ScrapeEngineBase,
ScrapeResult,
ScrapedPost,
ScrapeEngineFeature,
ContentType,
} from "../ScrapeEngine.js";
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapeEngineFeature, ContentType} from "../ScrapeEngine.js";
interface Attachment {
url: string; // original media URL, ?sensitive=true stripped
url: string; // original media URL
contentType: ContentType;
}
/**
* Handles Mastodon-compatible instances, including forks such as baraag.net.
*
* Mastodon renders media in three different shapes:
*
* 1. Images:
* <a class="media-gallery__item-thumbnail" href=".../original/....png">
* <img src=".../small/....png">
* </a>
* → read the anchor href.
*
* 2. GIFVs:
* <div class="media-gallery__gifv autoplay">
* <video class="media-gallery__item-gifv-thumbnail"
* src=".../original/....mp4" autoplay loop></video>
* </div>
* → read the <video> src (already the original .mp4).
*
* 3. Videos:
* <div role="menuitem" class="video-player">
* <video src=".../original/....mp4" poster=".../small/....png"></video>
* </div>
* → read the <video> src and use the poster attribute for extraContentUrl.
*/
export default class Mastodon extends ScrapeEngineBase {
name = "mastodon";
features: ScrapeEngineFeature[] = ["content", "source"];
notes = [
"Rating is assumed to be safe."
features: ScrapeEngineFeature[] = ["content"];
notes = [];
supportedHosts = [
"aethy.com",
"baraag.net",
"mastodon.online",
"mastodon.social",
"mastodon.world",
"mstdn.jp",
"mstdn.social",
"pawoo.net"
];
supportedHosts = ["baraag.net"];
/**
* Enabled for any other instances matching the pattern: .../@user/<postID>
*/
canImport(url: Location): boolean {
if (this.supportedHosts.indexOf(url.host.replace("www.", "")) !== -1) {
return true;
}
// Generic Mastodon status URL shape: /@user/<numeric id>
return /^\/@[^/]+\/\d+\/?$/.test(url.pathname);
return this.supportedHosts.indexOf(url.host.replace("www.", "")) !== -1
|| /^\/@[^/]+\/\d+\/?$/.test(url.pathname);
}
scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name);
const result = new ScrapeResult(this.name);
const attachments = this.findAttachments(document);
this.log(`Found ${attachments.length} attachment(s).`);
for (const att of attachments) {
const post = new ScrapedPost();
post.pageUrl = document.location.href;
post.contentUrl = att.url;
post.contentType = att.contentType;
post.referrer = document.location.origin;
result.tryAddPost(post);
}
if (attachments.length === 0) {
this.log("No media attachments found on this page.");
}
return result;
}
@ -85,7 +53,13 @@ export default class Mastodon extends ScrapeEngineBase {
out.push(att);
};
// 1. Images: anchor href points at the original.
/**
* Images -> a[href]
*
* <a class="media-gallery__item-thumbnail" href=".../original/....png">
* <img src=".../small/....png">
* </a>
*/
let anchors = Array.from(
document.querySelectorAll<HTMLAnchorElement>("a.media-gallery__item-thumbnail[href]")
);
@ -100,7 +74,7 @@ export default class Mastodon extends ScrapeEngineBase {
const img = a.querySelector("img");
const url = this.normalizeUrl(rawHref) || this.largestFromSrcset(img);
const url = this.resolveUrl(rawHref) || this.largestFromSrcset(img);
if (!url) continue;
add({
@ -109,13 +83,23 @@ export default class Mastodon extends ScrapeEngineBase {
});
}
// 2 & 3. Videos and GIFVs: <video> elements carry the original .mp4 in src.
/**
* GIFs and Videos -> video[src]
*
* <div class="media-gallery__gifv autoplay">
* <video class="media-gallery__item-gifv-thumbnail" src=".../original/....mp4" autoplay loop></video>
* </div>
*
* <div role="menuitem" class="video-player">
* <video src=".../original/....mp4" poster=".../small/....png"></video>
* </div>
*/
const videos = Array.from(document.querySelectorAll<HTMLVideoElement>("video[src]"));
for (const v of videos) {
const rawSrc = v.getAttribute("src");
const rawSrc = v.getAttribute("src") || v.currentSrc;
if (!rawSrc) continue;
const url = this.normalizeUrl(rawSrc);
const url = this.resolveUrl(rawSrc);
if (!url) continue;
add({
@ -128,15 +112,11 @@ export default class Mastodon extends ScrapeEngineBase {
}
/**
* Strip Mastodon's ?sensitive=true marker. The marker is metadata, not
* part of the media URL, and leaving it in can confuse booru deduplication.
* Returns an empty string if the URL is unusable.
* Resolve possible relative media URLs against current page.
*/
private normalizeUrl(href: string): string {
private resolveUrl(href: string): string {
try {
const u = new URL(href, document.location.href);
u.searchParams.delete("sensitive");
return u.toString();
return new URL(href, document.location.href).toString();
} catch {
return href;
}