Bugfixes

Fixed safety detection on chan.sankakucomplex.com, and improved source parsing on Gelbooru v0.2.0 sites.

.eslintrc.cjs

@@ -0,0 +1,17 @@
+module.exports = {
+  env: {
+    browser: true,
+    commonjs: true,
+    es2021: true,
+    node: true,
+  },
+  extends: ["eslint:recommended", "plugin:@typescript-eslint/recommended"],
+  parser: "@typescript-eslint/parser",
+  parserOptions: {
+    ecmaVersion: "latest",
+  },
+  plugins: ["@typescript-eslint"],
+  rules: {
+    semi: ["error", "always"],
+  },
+};

.gitignore

@@ -2,3 +2,5 @@
 node_modules/
 dist/
 coverage/
+package-lock.json
+package.json

README.md

@@ -1,3 +1,42 @@
 # neo-scraper
 
 Scrape a booru-like post object from the DOM.
+
+## Supported sites
+
+| Engine | Sites | Features | Notes |
+|---|---|---|---|
+| animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
+| bluesky | bsky.app | content |  |
+| danbooru | danbooru.donmai.us, safebooru.donmai.us, sonohara.donmai.us, hijiribe.donmai.us | content, rating, resolution, tags, tag_category, source, notes |  |
+| e621 | e621.net, e926.net, e6ai.net | content, rating, resolution, tags, tag_category, source |  |
+| E-hentai | e-hentai.org, exhentai.org | content, resolution, extra_content, cookies | Rating is assumed to be unsafe. |
+| furaffinity | furaffinity.net | content, rating, resolution, tags, tag_category |  |
+| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com, mspabooru.com, hypnohub.net | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. |
+| inkbunny | inkbunny.net | content, rating, resolution, tags |  |
+| moebooru | yande.re, konachan.com, konachan.net, sakugabooru.com, img.genshiken-itb.org | content, rating, resolution, tags, tag_category, source, notes |  |
+| nozomi | nozomi.la | content, tags, tag_category | Image preview might not work due to hotlink protection. |
+| Philomena | derpibooru.org, trixiebooru.org, ponybooru.org, furbooru.org, ponerpics.org, manebooru.art, twibooru.org | content, rating, resolution, tags, tag_category, source |  |
+| pixiv | pixiv.net | content |  |
+| reddit | reddit.com, new.reddit.com, old.reddit.com | content |  |
+| rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. |
+| sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes |  |
+| shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. |
+| shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. |
+| twitter | twitter.com, mobile.twitter.com, x.com | content |  |
+| zerochan | zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
+| fallback |  | content | Tries to find the largest image or video on the current page. |
+
+## Development
+
+Build SzuruChrome with your local version of neo-scraper.
+
+```sh
+cd neo-scraper
+pnpm install
+pnpm build # or pnpm watch in another terminal
+pnpm link --global
+cd ../SzuruChrome
+pnpm link --global neo-scraper
+pnpm dev
+```

docs/TESTING.md

@@ -0,0 +1,24 @@
+# Testing notes
+
+## Run all tests in testfiles matching some pattern 
+
+```sh
+pnpm test pixiv
+```
+
+## Run one specific test ({{describe name}} {{it name}})
+
+```sh
+pnpm test pixiv -t "get pixiv post 96268450 \(multipage, expanded\) should find exactly four posts"
+```
+
+For some reason you need to escape the parentheses.
+
+## Debug puppeteer tests
+
+```sh
+pnpm test:debug # you can add more arguments if you want
+```
+
+This will disable headless mode.
+Use `await jestPuppeteer.debug();` in the test code to pause code execution.

jest-puppeteer.config.cjs

@@ -1,6 +1,6 @@
 module.exports = {
   launch: {
-    headless: process.env.HEADLESS !== "false",
+    headless: process.env.HEADLESS !== "false" ? "new" : false,
     executablePath: process.env.CHROME_BIN || null,
     args: process.env.CI ? ["--disable-gpu", "--disable-dev-shm-usage", "--no-sandbox"] : [],
   },

package.json

@@ -1,52 +1,77 @@
 {
   "name": "neo-scraper",
-  "version": "0.5.2",
+  "version": "0.11.1",
   "description": "Scrape a booru-like post object from the DOM.",
+  "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.js",
   "scripts": {
-    "test": "jest",
+    "test": "cross-env JEST_PUPPETEER_CONFIG=jest-puppeteer.config.cjs jest",
-    "test:ci": "jest --colors --silent",
+    "test:ci": "cross-env JEST_PUPPETEER_CONFIG=jest-puppeteer.config.cjs jest --colors --silent",
+    "test:debug": "cross-env JEST_PUPPETEER_CONFIG=jest-puppeteer.config.cjs HEADLESS=false jest --testTimeout=100000000",
     "build": "tsc",
     "watch": "tsc -w",
-    "prepare": "npm run build"
+    "lint": "eslint src scripts",
+    "summary-table": "npm run build && node --experimental-specifier-resolution=node --loader ts-node/esm scripts/summary-table.ts"
   },
   "author": "neobooru",
   "license": "MIT",
+  "dependencies": {
+    "js-cookie": "^3.0.5",
+    "turndown": "^7.1.2"
+  },
   "devDependencies": {
-    "@types/expect-puppeteer": "^4.4.7",
+    "@types/expect-puppeteer": "^5.0.5",
-    "@types/jest": "^26.0.14",
+    "@types/jest": "^29.5.7",
-    "@types/jest-environment-puppeteer": "^4.4.1",
+    "@types/jest-environment-puppeteer": "^5.0.5",
+    "@types/js-cookie": "^3.0.5",
     "@types/jsdom": "^12.2.4",
-    "@types/puppeteer": "^5.4.4",
+    "@types/turndown": "^5.0.3",
-    "@types/turndown": "^5.0.1",
+    "@typescript-eslint/eslint-plugin": "^5.62.0",
-    "jest": "^26.6.3",
+    "@typescript-eslint/parser": "^5.62.0",
-    "jest-puppeteer": "^6.0.0",
+    "cross-env": "^7.0.3",
+    "eslint": "8.7.0",
+    "eslint-plugin-import": "^2.29.0",
+    "eslint-plugin-n": "^15.7.0",
+    "eslint-plugin-promise": "^6.1.1",
+    "jest": "^29.7.0",
+    "jest-puppeteer": "^10.1.4",
+    "jest-ts-webcompat-resolver": "^1.0.0",
     "jsdom": "^15.2.1",
-    "puppeteer": "^13.1.2",
+    "prettier": "2.5.1",
-    "ts-jest": "^26.5.6",
+    "puppeteer": "^23.10.1",
-    "typescript": "^4.5.2"
+    "ts-jest": "^29.2.5",
+    "ts-node": "^10.9.2",
+    "typescript": "^4.9.5"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
   },
   "jest": {
-    "preset": "./jest.preset.js",
+    "preset": "./jest.preset.cjs",
+    "resolver": "jest-ts-webcompat-resolver",
     "transform": {
-      "^.+\\.tsx?$": "ts-jest"
+      "^.+\\.tsx?$": [
+        "ts-jest",
+        {
+          "tsconfig": "tsconfig.test.json"
+        }
+      ]
     },
     "moduleFileExtensions": [
       "ts",
       "js"
     ],
+    "testTimeout": 30000,
     "testRegex": "^.+\\.test\\.ts$",
     "setupFilesAfterEnv": [
       "./src/test/jest.setup.ts"
-    ],
+    ]
-    "globals": {
-      "ts-jest": {
-        "tsconfig": "tsconfig.test.json"
-      }
-    }
   },
-  "dependencies": {
+  "overrides": {
-    "turndown": "^7.1.1"
+	  "@types/node": "18.19.115"
   }
 }

scripts/summary-table.ts

@@ -0,0 +1,15 @@
+import NeoScraper from "../src/NeoScraper.js";
+import FallbackEngine from "../src/engines/Fallback.js";
+
+const engines = [...new NeoScraper().engines, new FallbackEngine()];
+const txt = ["| Engine | Sites | Features | Notes |", "|---|---|---|---|"];
+
+for (const engine of engines) {
+  txt.push(
+    `| ${engine.name} | ${engine.supportedHosts.join(", ")} | ${engine.features.join(", ")} | ${engine.notes.join(
+      " "
+    )} |`
+  );
+}
+
+console.log(txt.join("\n"));

src/NeoScraper.test.ts

@@ -1,12 +1,12 @@
-import NeoScraper from "./NeoScraper";
+import NeoScraper from "./NeoScraper.js";
 
 test("create scraper with default engines", async () => {
-  const scraper = new NeoScraper();
+  new NeoScraper();
 });
 
 test("all default scrapers have unique names", () => {
   const scraper = new NeoScraper();
-  let alreadySeen: string[] = [];
+  const alreadySeen: string[] = [];
 
   for (const engine of scraper.engines) {
     if (alreadySeen.indexOf(engine.name) != -1) {

src/NeoScraper.ts

@@ -1,20 +1,25 @@
-import { ScrapeEngine, ScrapeResults } from "./ScrapeEngine";
+import { ScrapeEngine, ScrapeResults } from "./ScrapeEngine.js";
-import * as engines from "./engines";
+import * as engines from "./engines/index.js";
 
 export default class NeoScraper {
   engines: ScrapeEngine[] = [];
   fallbackEngine: ScrapeEngine | null = null;
 
-  constructor(useDefaultEngines: boolean = true) {
+  constructor(useDefaultEngines = true) {
     if (useDefaultEngines) {
       this.engines.push(
         new engines.AnimePictures(),
+        new engines.Bluesky(),
         new engines.Danbooru(),
         new engines.e621(),
+        new engines.Ehentai(),
         new engines.FurAffinity(),
         new engines.Gelbooru(),
         new engines.Inkbunny(),
+		new engines.Mastodon(),
         new engines.Moebooru(),
+        new engines.Nozomi(),
+        new engines.Philomena(),
         new engines.Pixiv(),
         new engines.Reddit(),
         new engines.Rule34us(),
@@ -28,12 +33,7 @@ export default class NeoScraper {
     }
   }
 
-  scrapeDocument(
+  scrapeDocument(document: Document, allowFallbackEngine = false, ignoreCanImport = false): ScrapeResults {
-    document: Document,
-    engines: string[] | null = null,
-    allowFallbackEngine: boolean = false,
-    ignoreCanImport: boolean = false
-  ): ScrapeResults {
     const res = new ScrapeResults();
 
     for (const engine of this.engines) {

src/ScrapeEngine.ts

@@ -1,17 +1,24 @@
-import { TagCategory, SafetyRating } from "./BooruTypes";
+import { TagCategory, SafetyRating } from "./BooruTypes.js";
 
+export type Cookies = { [key: string]: string };
 export type ContentType = "image" | "video";
+export type UploadMode = "content" | "url";
 
 export class ScrapedPost {
-  contentUrl: string = "";
+  name: string | undefined = undefined;
-  pageUrl: string = "";
+  contentUrl = "";
+  pageUrl = "";
   contentType: ContentType = "image";
+  contentSize: number | undefined;
   resolution: [number, number] | undefined;
   rating: SafetyRating = "safe";
   tags: ScrapedTag[] = [];
   notes: ScrapedNote[] = [];
-  source: string | undefined;
+  sources: string[] = [];
   referrer: string | undefined;
+  extraContentUrl: string | undefined;
+  cookies: Cookies | undefined;
+  uploadMode: UploadMode = "url";
 }
 
 export class ScrapedTag {
@@ -30,7 +37,7 @@ export class ScrapedNote {
 
 export class ScrapeResult {
   engine: string;
-  description: string = "";
+  description = "";
   posts: ScrapedPost[] = [];
 
   constructor(engine: string) {
@@ -50,9 +57,9 @@ export class ScrapeResults {
   results: ScrapeResult[] = [];
 
   get posts(): ScrapedPost[] {
-    var posts: ScrapedPost[] = [];
+    let posts: ScrapedPost[] = [];
 
-    for (var res of this.results) {
+    for (const res of this.results) {
       posts = posts.concat(res.posts);
     }
 
@@ -60,15 +67,37 @@ export class ScrapeResults {
   }
 }
 
+export type ScrapeEngineFeature =
+  | "content"
+  | "rating"
+  | "resolution"
+  | "tags"
+  | "tag_category"
+  | "source"
+  | "notes"
+  | "extra_content"
+  | "cookies";
+
 export interface ScrapeEngine {
   name: string;
+  features: ScrapeEngineFeature[];
+  notes: string[];
+  supportedHosts: string[];
+
   canImport(url: Location): boolean;
   scrapeDocument(document: Document): ScrapeResult;
 }
 
 export abstract class ScrapeEngineBase implements ScrapeEngine {
   abstract name: string;
-  abstract canImport(url: Location): boolean;
+  abstract features: ScrapeEngineFeature[];
+  abstract notes: string[];
+  abstract supportedHosts: string[];
+
+  canImport(url: Location): boolean {
+    return this.supportedHosts.indexOf(url.host.replace("www.", "")) != -1;
+  }
+
   abstract scrapeDocument(document: Document): ScrapeResult;
 
   protected log(str: string) {

src/Utility.test.ts

@@ -1,4 +1,13 @@
-import { guessContentType } from "./Utility";
+import { getUrlExtension, guessContentType } from "./Utility.js";
+
+test("getUrlExtension", () => {
+  expect(getUrlExtension("image.png")).toBe("png");
+  expect(getUrlExtension("image.jpeg?34")).toBe("jpeg");
+  expect(getUrlExtension("image.php?q=test&p=123%204")).toBe("php");
+
+  // I would probably rather have this return an empty string, but maybe some code relies on this functionality?
+  expect(getUrlExtension("http://site/index")).toBe("http://site/index");
+});
 
 test("guessContentType", () => {
   expect(guessContentType("video.mp4")).toBe("video");

src/Utility.ts

@@ -1,5 +1,5 @@
 import TurndownService from "turndown";
-import { ContentType, ScrapedNote, ScrapedPost } from "./ScrapeEngine";
+import { ContentType, ScrapedNote, ScrapedPost } from "./ScrapeEngine.js";
 
 // https://github.com/sindresorhus/video-extensions/blob/main/video-extensions.json
 export const videoExtensions = [
@@ -39,10 +39,18 @@ export const videoExtensions = [
 ];
 
 export function getUrlExtension(url: string) {
+  // https://stackoverflow.com/a/47767860
   return url.split(/[#?]/)[0].split(".").pop()!.trim();
 }
 
-export function guessContentType(url: string): ContentType {
+/**
+ * Returns "image" when the guessed type is NOT a video.
+ * This means that it will also return "image" when the content type
+ * could not be guessed!
+ * @param url
+ * @returns "image" | "video"
+ */
+export function guessContentType(url: string | undefined): ContentType {
   if (url != undefined) {
     const ext = getUrlExtension(url);
     if (videoExtensions.indexOf(ext) != -1) return "video";
@@ -101,7 +109,7 @@ export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement
       return undefined;
     }
     const num = parseInt(value);
-    if (num == NaN) {
+    if (isNaN(num)) {
       logFail(`parseInt on '${value}' returned NaN.`);
     }
     return num;
@@ -112,6 +120,7 @@ export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement
     return undefined;
   }
 
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
   const data = (<any>el).dataset as DOMStringMap;
 
   if (!data) {
@@ -124,10 +133,10 @@ export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement
     text = htmlNoteToMarkdownNote(text);
   }
 
-  let x = getInt(data, "x");
+  const x = getInt(data, "x");
-  let y = getInt(data, "y");
+  const y = getInt(data, "y");
-  let w = getInt(data, "width");
+  const w = getInt(data, "width");
-  let h = getInt(data, "height");
+  const h = getInt(data, "height");
 
   if (!text || !x || !y || !w || !h) {
     logFail("A required data field could not be parsed.");
@@ -151,7 +160,7 @@ export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement
  * @returns
  */
 export function createNotesFromMoebooruBoxes(document: Document, boxSize: [number, number]) {
-  let notes: ScrapedNote[] = [];
+  const notes: ScrapedNote[] = [];
   const noteBoxes = Array.from(document.querySelectorAll(".note-box")).map((x) => x as HTMLLIElement);
   const noteBodies = Array.from(document.querySelectorAll(".note-body")).map((x) => x as HTMLLIElement);
 
@@ -186,8 +195,8 @@ export function parseResolutionString(str: string | undefined | null): [number,
   // Example string: 1600x2200
   const res = str
     .split("x") // Split on 'x' character
-    .map((v, _) => v.replace(/\D/g, "")) // Remove all non-digits
+    .map((v) => v.replace(/\D/g, "")) // Remove all non-digits
-    .map((v, _) => parseInt(v)) // Parse ints
+    .map((v) => parseInt(v)) // Parse ints
     .filter(Number); // Filter NaN. Also removes the number 0, but that's fine because the resolution can't be 0.
 
   if (res.length == 2) {

src/engines/AnimePictures.test.ts

@@ -1,8 +1,8 @@
-import { scrapeUrl } from "../test/engine";
+import { scrapeEvalUrl } from "../test/engine.js";
 
 test("get anime-pictures.net post 729833", async () => {
-  const pageUrl = "https://anime-pictures.net/pictures/view_post/729833?lang=en";
+  const pageUrl = "https://anime-pictures.net/posts/729833?lang=en";
-  const res = await scrapeUrl(pageUrl);
+  const res = await scrapeEvalUrl(pageUrl, "body");
   expect(res.posts.length).toBe(1);
   expect(res.posts[0]).toHaveTag("arknights", "copyright");
   expect(res.posts[0]).toHaveTag("ch'en_(arknights)", "character");
@@ -14,6 +14,6 @@ test("get anime-pictures.net post 729833", async () => {
   expect(res.posts[0]).toHaveResolution([4000, 6865]);
   expect(res.posts[0].pageUrl).toBe(pageUrl);
   expect(res.posts[0].contentUrl).toBe(
-    "https://anime-pictures.net/pictures/download_image/729833-4000x6865-arknights-ch%26%2339%3Ben+%28arknights%29-ttk+%28kirinottk%29-single-long+hair-tall+image.jpeg"
+    "https://api.anime-pictures.net/pictures/download_image/729833-4000x6865-arknights-ch'en+(arknights)-ttk+(kirinottk)-single-long+hair-tall+image.jpeg"
   );
 });

src/engines/AnimePictures.ts

@@ -1,22 +1,20 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
-import { TagCategory } from "../BooruTypes";
+import { TagCategory } from "../BooruTypes.js";
-import { guessContentType, parseResolutionString } from "../Utility";
+import { guessContentType, parseResolutionString } from "../Utility.js";
 
-export default class AnimePictures implements ScrapeEngine {
+export default class AnimePictures extends ScrapeEngineBase {
   name = "animepictures";
-
+  features: ScrapeEngineFeature[] = ["content", "resolution", "tags", "tag_category"];
-  canImport(url: Location): boolean {
+  notes = ["Rating is assumed to be safe."];
-    return url.host == "anime-pictures.net";
+  supportedHosts = ["anime-pictures.net"];
-  }
 
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
-    let post = new ScrapedPost();
+    const post = new ScrapedPost();
     post.pageUrl = document.location.href;
-    post.source = document.location.href;
 
     // Set image url
-    const originalImageElements = Array.from(document.getElementsByClassName("download_icon"));
+    const originalImageElements = Array.from(document.getElementsByClassName("icon-download"));
 
     if (originalImageElements.length > 0) {
       post.contentUrl = (originalImageElements[0] as HTMLAnchorElement).href;
@@ -29,9 +27,7 @@ export default class AnimePictures implements ScrapeEngine {
     post.rating = "safe"; // Probably
 
     // Set resolution
-    const postContentLinkEls = Array.from(
+    const postContentLinkEls = Array.from(document.querySelectorAll<HTMLAnchorElement>(".post_content > .body a"));
-      document.querySelectorAll<HTMLAnchorElement>("#content > div > .post_content > a")
-    );
     const resEl = postContentLinkEls.find((x) => x.href.indexOf("res_x") != -1);
     if (resEl) {
       post.resolution = parseResolutionString(resEl.innerText);
@@ -40,7 +36,7 @@ export default class AnimePictures implements ScrapeEngine {
     // Set tags
     const tagElements = Array.from(document.querySelectorAll<HTMLAnchorElement>("ul.tags > li > a"));
     for (const tagElement of tagElements) {
-      let tagName = tagElement.innerText;
+      const tagName = tagElement.innerText;
 
       let category: TagCategory | undefined;
       for (const className of tagElement.classList) {
@@ -58,7 +54,7 @@ export default class AnimePictures implements ScrapeEngine {
       }
 
       if (tagName) {
-        let tag = new ScrapedTag(tagName, category);
+        const tag = new ScrapedTag(tagName, category);
         post.tags.push(tag);
       }
     }

src/engines/Bluesky.ts

@@ -0,0 +1,126 @@
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapeEngineFeature } from "../ScrapeEngine.js";
+
+function getFullSizeImg(link: string) {
+  return link.replace("/feed_thumbnail/", "/feed_fullsize/");
+}
+
+/**
+ * Bluesky serves video via HLS in the player, but the underlying blob is a
+ * single .mp4 retrievable from the AT Protocol sync endpoint:
+ *
+ *   https://bsky.social/xrpc/com.atproto.sync.getBlob?did=<DID>&cid=<CID>
+ *
+ * The DID and CID are URL-encoded in the <video> poster's path:
+ *
+ *   https://video.bsky.app/watch/<DID>/<CID>/thumbnail.jpg
+ *
+ * We reconstruct the blob URL and mark the post as uploadMode="content"
+ * so SzuruChrome fetches it in the browser instead of asking Szurubooru's
+ * server to download it (which would fail on the blob endpoint).
+ */
+export default class Bluesky extends ScrapeEngineBase {
+  name = "bluesky";
+  features: ScrapeEngineFeature[] = ["content"];
+  notes = [];
+  supportedHosts = ["bsky.app"];
+
+  scrapeDocument(document: Document): ScrapeResult {
+    const result = new ScrapeResult(this.name);
+	
+	type Candidate = { el: Element; post: ScrapedPost };
+	const candidates: Candidate[] = [];
+	
+	// Images
+    const imgEls = Array.from<HTMLImageElement>(document.querySelectorAll("div > img"));
+    let els = imgEls.filter((x) => x.src.indexOf("feed_fullsize") != -1);
+    // Only add the 'normal' feed elements if no post is in fullscreen.
+    if (els.length == 0) {
+      this.log("Post is not fullscreen.");
+      els = imgEls.filter((x) => x.src.indexOf("feed") != -1);
+    }
+    for (const el of els) {
+	  if(!isVisible(el)) continue;
+	  
+      const post = new ScrapedPost();
+      post.pageUrl = document.location.href;
+      post.contentUrl = getFullSizeImg(el.src);
+      candidates.push({ el, post });
+    }
+
+    // Videos
+    const videoEls = Array.from<HTMLVideoElement>(
+      document.querySelectorAll('video[poster^="https://video.bsky.app/watch"]')
+    );
+    for (const v of videoEls) {
+	  if(!isVisible(v)) continue;
+	  
+      const poster = v.getAttribute("poster");
+      if (!poster) continue;
+
+      const blobUrl = this.buildBlobUrl(poster);
+      if (!blobUrl) {
+        this.log(`Could not parse poster URL: ${poster}`);
+        continue;
+      }
+      const post = new ScrapedPost();
+      post.pageUrl = document.location.href;
+      post.contentUrl = blobUrl;
+      post.contentType = "video";
+      candidates.push({ el: v, post });
+    }
+	
+	candidates.sort((a, b) => {
+      const pos = a.el.compareDocumentPosition(b.el);
+      if (pos & Node.DOCUMENT_POSITION_FOLLOWING) return -1;
+      if (pos & Node.DOCUMENT_POSITION_PRECEDING) return 1;
+      return 0;
+    });
+	
+	for(const c of candidates) result.tryAddPost(c.post);
+
+    return result;
+  }
+
+  /**
+   * Poster format:
+   *   https://video.bsky.app/watch/<DID-urlencoded>/<CID>/thumbnail.jpg
+   * Returns the AT Protocol blob URL, or undefined if the poster doesn't
+   * match the expected shape.
+   */
+  private buildBlobUrl(poster: string): string | undefined {
+    try {
+      const u = new URL(poster);
+      const parts = u.pathname.split("/").filter((p) => p.length > 0);
+      // Expected: ["watch", "<DID>", "<CID>", "thumbnail.jpg"]
+      if (parts.length < 3 || parts[0] !== "watch") return undefined;
+
+      const did = decodeURIComponent(parts[1]);   // did:plc:...
+      const cid = parts[2];                        // bafkrei...
+
+      return (
+        "https://bsky.social/xrpc/com.atproto.sync.getBlob" +
+        `?did=${encodeURIComponent(did)}&cid=${encodeURIComponent(cid)}`
+      );
+    } catch {
+      return undefined;
+    }
+  }
+}
+
+/**
+ * A lightweight visibility check. We use this rather than IntersectionObserver
+ * because scrapeDocument is synchronous and must return immediately.
+ */
+function isVisible(el: Element): boolean {
+  const rect = el.getBoundingClientRect();
+  if (rect.width === 0 || rect.height === 0) return false;
+  if (rect.bottom <= 0 || rect.top >= window.innerHeight) return false;
+  if (rect.right <= 0 || rect.left >= window.innerWidth) return false;
+
+  const style = window.getComputedStyle(el);
+  if (style.display === "none") return false;
+  if (style.visibility === "hidden") return false;
+  if (style.opacity === "0") return false;
+
+  return true;
+}

src/engines/Common.ts

@@ -1,3 +1,3 @@
-import { TagCategory } from "../BooruTypes";
+import { TagCategory } from "../BooruTypes.js";
 
 export type CategoryMap = Record<string, TagCategory>;

src/engines/Danbooru.test.ts

@@ -1,27 +1,27 @@
-import { scrapeUrl } from "../test/engine";
+import { scrapeEvalUrl } from "../test/engine.js";
 
 test("get danbooru.donmai.us post 3555872", async () => {
   const pageUrl = "https://danbooru.donmai.us/posts/3555872";
-  const res = await scrapeUrl(pageUrl);
+  const res = await scrapeEvalUrl(pageUrl, "#tag-list");
   expect(res.posts.length).toBe(1);
   expect(res.posts[0]).toHaveTag("omachi_(slabco)", "artist");
-  expect(res.posts[0]).toHaveTag("dr_pepper", "copyright");
+  expect(res.posts[0]).toHaveTag("dr_pepper");
   expect(res.posts[0]).toHaveTag("flint_(girls_und_panzer)", "character");
-  expect(res.posts[0]).toHaveTag("bangs");
   expect(res.posts[0]).toHaveTag("shirt");
   expect(res.posts[0]).toHaveTag("skirt");
   expect(res.posts[0]).toHaveTag("t-shirt");
   expect(res.posts[0]).toHaveTag("long_hair");
   expect(res.posts[0]).toHaveResolution([1142, 1600]);
+  expect(res.posts[0]).toHaveSource("https://twitter.com/omatiosake/status/1124306601584672768");
   expect(res.posts[0].pageUrl).toBe(pageUrl);
   expect(res.posts[0].contentUrl).toBe(
-    "https://cdn.donmai.us/original/90/33/__flint_girls_und_panzer_and_1_more_drawn_by_omachi_slabco__90332299216db81b8793084e4cb7c15d.jpg"
+    "https://cdn.donmai.us/original/90/33/__flint_girls_und_panzer_drawn_by_omachi_slabco__90332299216db81b8793084e4cb7c15d.jpg"
   );
 });
 
 test("get danbooru.donmai.us post 4924490", async () => {
   const pageUrl = "https://danbooru.donmai.us/posts/4924490";
-  const res = await scrapeUrl(pageUrl);
+  const res = await scrapeEvalUrl(pageUrl, "#tag-list");
   expect(res.posts.length).toBe(1);
   expect(res.posts[0]).toHaveTag("yuureidoushi_(yuurei6214)", "artist");
   expect(res.posts[0]).toHaveTag("comic");
@@ -33,3 +33,16 @@ test("get danbooru.donmai.us post 4924490", async () => {
   expect(res.posts[0].notes).toHaveLength(17);
   expect(res.posts[0]).toHaveNote("Nya~nya ♪");
 });
+
+test("get danbooru.donmai.us post 4530532", async () => {
+  const pageUrl = "https://danbooru.donmai.us/posts/4530532";
+  const res = await scrapeEvalUrl(pageUrl, "#tag-list");
+  expect(res.posts.length).toBe(1);
+  expect(res.posts[0].rating).toBe("sketchy");
+  expect(res.posts[0]).toHaveTag("chainsaw_man", "copyright");
+  expect(res.posts[0]).toHaveResolution([1105, 1803]);
+  expect(res.posts[0].pageUrl).toBe(pageUrl);
+  expect(res.posts[0].contentUrl).toBe(
+    "https://cdn.donmai.us/original/fc/e3/__makima_power_and_reze_chainsaw_man_drawn_by_hews__fce35c7a8515a00c566b288a591fd5f9.png"
+  );
+});

src/engines/Danbooru.ts

@@ -1,17 +1,23 @@
-import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapedNote } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
-import { TagCategory } from "../BooruTypes";
+import { TagCategory } from "../BooruTypes.js";
-import { createNoteFromDanbooruArticle, guessContentType } from "../Utility";
+import { createNoteFromDanbooruArticle, guessContentType } from "../Utility.js";
 
 export default class Danbooru extends ScrapeEngineBase {
   name = "danbooru";
-
+  features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
-  canImport(url: Location): boolean {
+  notes = [];
-    return url.host == "danbooru.donmai.us" || url.host == "safebooru.donmai.us";
+  supportedHosts = [
-  }
+    "danbooru.donmai.us", 
+	"safebooru.donmai.us", 
+	"sonohara.donmai.us", 
+	"hijiribe.donmai.us",
+	"booru.allthefallen.moe"
+  ];
 
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
-    let post = new ScrapedPost();
+    const post = new ScrapedPost();
+    post.uploadMode = "content";
     post.pageUrl = document.location.href;
 
     // Set image url
@@ -56,6 +62,7 @@ export default class Danbooru extends ScrapeEngineBase {
           case "safe":
             post.rating = "safe";
             break;
+          case "sensitive":
           case "questionable":
             post.rating = "sketchy";
             break;
@@ -76,7 +83,8 @@ export default class Danbooru extends ScrapeEngineBase {
       }
 
       let category: TagCategory | undefined;
-      switch (tagElement.className) {
+      for (const className of tagElement.classList) {
+        switch (className) {
           case "tag-type-3":
             category = "copyright";
             break;
@@ -90,9 +98,10 @@ export default class Danbooru extends ScrapeEngineBase {
             category = "meta";
             break;
         }
+      }
 
       if (tagName) {
-        let tag = new ScrapedTag(tagName, category);
+        const tag = new ScrapedTag(tagName, category);
         post.tags.push(tag);
       }
     }
@@ -102,7 +111,7 @@ export default class Danbooru extends ScrapeEngineBase {
     if (sourceEl && sourceEl.children.length > 0) {
       const sourceLink = sourceEl.children[0] as HTMLAnchorElement;
       if (sourceLink) {
-        post.source = sourceLink.href;
+        post.sources = [sourceLink.href];
       }
     }
 

src/engines/Ehentai.ts

@@ -0,0 +1,49 @@
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapeEngineFeature } from "../ScrapeEngine.js";
+import { parseResolutionString } from "../Utility.js";
+import Cookies from "js-cookie";
+
+export default class Ehentai extends ScrapeEngineBase {
+  name = "E-hentai";
+  features: ScrapeEngineFeature[] = ["content", "resolution", "extra_content", "cookies"];
+  notes = ["Rating is assumed to be unsafe."];
+  supportedHosts = ["e-hentai.org", "exhentai.org"];
+
+  scrapeDocument(document: Document): ScrapeResult {
+    const result = new ScrapeResult(this.name);
+    const post = new ScrapedPost();
+    post.pageUrl = document.location.href;
+
+    // Set image url
+    const imgUrl = document.getElementById("img") as HTMLImageElement;
+    if (imgUrl) {
+      post.contentUrl = imgUrl.src;
+    }
+
+    const originalSourceUrl = document.querySelector("a[href*='fullimg']") as HTMLAnchorElement;
+    if (originalSourceUrl) {
+      // Set "extra" image url. This URL only gives a good response if the cookies are included.
+      // This might also only work on certain accounts.
+      post.extraContentUrl = originalSourceUrl.href;
+
+      // Set resolution
+      const m = originalSourceUrl.innerText.match(/\d+ x \d+/);
+      if (m?.length == 1) {
+        post.resolution = parseResolutionString(m[0]);
+      }
+    }
+
+    // Set rating hardcoded unsafe
+    post.rating = "unsafe";
+
+    // No tags available on this page.
+    // On the parent page you could probably get some tags, though these don't say anything about the individual image.
+    // The tags only say something about the gallery as a whole.
+
+    // Set cookies
+    post.cookies = Cookies.get();
+
+    result.tryAddPost(post);
+
+    return result;
+  }
+}

src/engines/Fallback.ts

@@ -1,4 +1,5 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ContentType } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ContentType, ScrapeEngineFeature } from "../ScrapeEngine.js";
+import { guessContentType } from "../Utility.js";
 
 class MediaElement {
   constructor(
@@ -8,14 +9,28 @@ class MediaElement {
   ) { }
 }
 
-export default class Fallback implements ScrapeEngine {
+export default class Fallback extends ScrapeEngineBase {
   name = "fallback";
-
+  features: ScrapeEngineFeature[] = ["content"];
-  canImport(location: Location): boolean {
+  notes = ["Tries to find the largest image or video on the current page."];
-    return false;
+  supportedHosts = [];
-  }
 
   scrapeDocument(document: Document): ScrapeResult {
+    const result = new ScrapeResult(this.name);
+
+    // Check if the current page is a link to a video file.
+    // If it is then use the document.location.href as media source and immediately return.
+    // This is needed because firefox does not add a <source> to a <video> element
+    // when going to a direct link. It looks like chromium-based browsers do not have this issue.
+    if (guessContentType(document.location.href) == "video") {
+      const post = new ScrapedPost();
+      post.pageUrl = document.location.href;
+      post.contentUrl = document.location.href;
+      post.contentType = "video";
+      result.tryAddPost(post);
+      return result;
+    }
+
     // Get all img elements.
     const largestImgElement = Array.from(document.getElementsByTagName("img")).map(
       (x) => new MediaElement("image", x.src, x.width * x.height)
@@ -33,14 +48,18 @@ export default class Fallback implements ScrapeEngine {
       a?.size > b?.size ? a : b
     );
 
-    let result = new ScrapeResult(this.name);
-
     if (largestMediaElement) {
-      let post = new ScrapedPost();
+      const post = new ScrapedPost();
+      post.name = "Upload as URL";
       post.pageUrl = document.location.href;
       post.contentUrl = largestMediaElement.contentUrl;
       post.contentType = largestMediaElement.contentType;
-      result.posts.push(post);
+      result.tryAddPost(post);
+
+      const clone = Object.assign({}, post);
+      clone.name = "Upload as content";
+      clone.uploadMode = "content";
+      result.tryAddPost(clone);
     }
 
     return result;

src/engines/FurAffinity.test.ts

@@ -0,0 +1,16 @@
+import { scrapeUrl } from "../test/engine.js";
+
+test("get furaffinity.net post 23281942", async () => {
+  const pageUrl = "https://www.furaffinity.net/view/23281942/";
+  const res = await scrapeUrl(pageUrl);
+  expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("zac");
+  expect(res.posts[0]).toHaveTag("riven");
+  expect(res.posts[0]).toHaveTag("pool");
+  expect(res.posts[0]).toHaveTag("nielsdejong", "artist");
+  expect(res.posts[0]).toHaveResolution([1280, 1279]);
+  expect(res.posts[0].pageUrl).toBe(pageUrl);
+  expect(res.posts[0].contentUrl).toBe(
+    "https://d.furaffinity.net/art/nielsdejong/1492781239/1492781239.nielsdejong_pool_party_fun_with_riven_and_zac_by_polkinart.jpg"
+  );
+});

src/engines/FurAffinity.ts

@@ -1,17 +1,15 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
-import { guessContentType } from "../Utility";
+import { guessContentType, parseResolutionString } from "../Utility.js";
 
-export default class e621 implements ScrapeEngine {
+export default class FurAffinity extends ScrapeEngineBase {
   name = "furaffinity";
-
+  features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category"];
-  canImport(url: Location): boolean {
+  notes = [];
-    return url.host == "www.furaffinity.net";
+  supportedHosts = ["furaffinity.net"];
-  }
 
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
-
+    const post = new ScrapedPost();
-    let post = new ScrapedPost();
     post.pageUrl = document.location.href;
 
     // Set image url
@@ -22,6 +20,14 @@ export default class e621 implements ScrapeEngine {
     // Set content type
     post.contentType = guessContentType(post.contentUrl);
 
+    // Set resolution
+    const infoTexts = Array.from(document.querySelector("section.info.text")?.children ?? []);
+    const sizeDiv = infoTexts.find((x) => x.querySelector("strong")?.innerText == "Size");
+    if (sizeDiv) {
+      const resolution = parseResolutionString(sizeDiv.querySelector("span")?.innerText);
+      post.resolution = resolution;
+    }
+
     // Set rating
     const ratingEls = Array.from(document.querySelectorAll("div.rating > span.rating-box")).map(
       (x) => x as HTMLSpanElement
@@ -41,12 +47,15 @@ export default class e621 implements ScrapeEngine {
       }
 
     // Set tags
-    post.tags = Array.from(document.querySelectorAll("section.tags-row > span.tags"))
+    post.tags = Array.from(document.querySelectorAll("section.tags-row > span.tags a"))
       .map((x) => (x as HTMLSpanElement).innerText)
       .map((x) => new ScrapedTag(x));
 
-    // Set source
+    // Set user who submitted the art as the artist
-    post.source = document.location.href;
+    const submissionUser = document.querySelector<HTMLElement>(".submission-id-sub-container a")?.innerText;
+    if (submissionUser) {
+      post.tags.push(new ScrapedTag(submissionUser, "artist"));
+    }
 
     result.tryAddPost(post);
 

src/engines/Gelbooru.test.ts

@@ -1,4 +1,4 @@
-import { scrapeUrl } from "../test/engine";
+import { scrapeUrl } from "../test/engine.js";
 
 test("get safebooru.org post 2871532", async () => {
   const pageUrl = "https://safebooru.org/index.php?page=post&s=view&id=2871532";
@@ -13,7 +13,8 @@ test("get safebooru.org post 2871532", async () => {
   expect(res.posts[0]).toHaveTag("long_hair");
   expect(res.posts[0]).toHaveResolution([1142, 1600]);
   expect(res.posts[0].pageUrl).toBe(pageUrl);
-  expect(res.posts[0].contentUrl).toBe(
+  expect(res.posts[0].rating).toBe("safe");
+  expect(res.posts[0].contentUrl).toContain(
     "https://safebooru.org//images/2757/5f19c7744db918c3d4c4c6035bc65dfa9e18866f.jpg"
   );
 });
@@ -25,15 +26,16 @@ test("get gelbooru.com post 4925055", async () => {
   expect(res.posts[0]).toHaveTag("maosen", "artist");
   expect(res.posts[0]).toHaveTag("butterfly");
   expect(res.posts[0]).toHaveResolution([720, 1103]);
+  expect(res.posts[0]).toHaveSource("https://www.pixiv.net/artworks/76853958");
   expect(res.posts[0].pageUrl).toBe(pageUrl);
-  expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
+  expect(res.posts[0].rating).toBe("sketchy");
+  expect(res.posts[0].contentUrl).toContain("gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
 });
 
 test("get gelbooru.com post 6225495 with notes", async () => {
   const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=6225495";
   const res = await scrapeUrl(pageUrl);
   expect(res.posts.length).toBe(1);
-  expect(res.posts[0]).toHaveTag("bangs");
   expect(res.posts[0]).toHaveTag("hair_up");
   expect(res.posts[0]).toHaveTag("hair_up");
   expect(res.posts[0]).toHaveTag("teeth");
@@ -45,8 +47,10 @@ test("get gelbooru.com post 6225495 with notes", async () => {
   expect(res.posts[0]).toHaveNote("But no need to worry!");
   expect(res.posts[0]).toHaveNote("You're always the best just by being yourself-!!!");
   expect(res.posts[0]).toHaveResolution([1600, 2200]);
+  expect(res.posts[0]).toHaveSource("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=88721816");
   expect(res.posts[0].pageUrl).toBe(pageUrl);
-  expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/af/1d/af1dd661fab6714dffc29cb38b6b2269.png");
+  expect(res.posts[0].rating).toBe("sketchy");
+  expect(res.posts[0].contentUrl).toContain("gelbooru.com/images/af/1d/af1dd661fab6714dffc29cb38b6b2269.png");
 });
 
 test("get rule34.xxx post 2740994", async () => {
@@ -54,12 +58,13 @@ test("get rule34.xxx post 2740994", async () => {
   const res = await scrapeUrl(pageUrl);
   expect(res.posts[0]).toHaveTag("fire_emblem", "copyright");
   expect(res.posts[0]).toHaveTag("nintendo", "copyright");
-  expect(res.posts[0]).toHaveTag("lucina", "character");
+  expect(res.posts[0]).toHaveTag("lucina_(fire_emblem)", "character");
   expect(res.posts[0]).toHaveTag("blue_hair");
   expect(res.posts[0]).toHaveTag("solo");
   expect(res.posts[0]).toHaveResolution([600, 800]);
   expect(res.posts.length).toBe(1);
   expect(res.posts[0].pageUrl).toBe(pageUrl);
+  expect(res.posts[0].rating).toBe("safe");
   expect(res.posts[0].contentUrl).toContain(".rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg");
 });
 
@@ -74,5 +79,6 @@ test("get xbooru.com post 834083", async () => {
   expect(res.posts[0]).toHaveResolution([1024, 1878]);
   expect(res.posts.length).toBe(1);
   expect(res.posts[0].pageUrl).toBe(pageUrl);
-  expect(res.posts[0].contentUrl).toBe("https://img.xbooru.com//images/584/040dc1625c094a7d5d7b0918aa0d8324.png");
+  expect(res.posts[0].rating).toBe("safe");
+  expect(res.posts[0].contentUrl).toContain("https://img.xbooru.com//images/584/040dc1625c094a7d5d7b0918aa0d8324.png");
 });

src/engines/Gelbooru.ts

@@ -1,7 +1,7 @@
-import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
-import { TagCategory } from "../BooruTypes";
+import { TagCategory } from "../BooruTypes.js";
-import { CategoryMap } from "./Common";
+import { CategoryMap } from "./Common.js";
-import { createNoteFromDanbooruArticle, guessContentType, parseResolutionString } from "../Utility";
+import { createNoteFromDanbooruArticle, guessContentType, parseResolutionString } from "../Utility.js";
 
 // Small hack to avoid screwing with semver
 enum Version {
@@ -11,6 +11,17 @@ enum Version {
 
 export default class Gelbooru extends ScrapeEngineBase {
   name = "gelbooru";
+  features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
+  notes = ["Supported features might vary between hosts."];
+  supportedHosts = [
+    "safebooru.org",
+    "gelbooru.com",
+    "rule34.xxx",
+    "tbib.org",
+    "xbooru.com",
+    "mspabooru.com",
+    "hypnohub.net",
+  ];
 
   private readonly classNameToCategoryMap: CategoryMap = {
     "tag-type-copyright": "copyright",
@@ -19,24 +30,16 @@ export default class Gelbooru extends ScrapeEngineBase {
     "tag-type-metadata": "meta",
   };
 
-  canImport(url: Location): boolean {
-    return (
-      url.host == "safebooru.org" ||
-      url.host == "gelbooru.com" ||
-      url.host == "rule34.xxx" ||
-      url.host == "tbib.org" ||
-      url.host == "xbooru.com"
-    );
-  }
-
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
     let version: Version | null = null;
     switch (document.location.host) {
       case "safebooru.org":
       case "rule34.xxx":
       case "tbib.org":
       case "xbooru.com":
+      case "mspabooru.com":
+      case "hypnohub.net":
         version = Version.v020;
         break;
       case "gelbooru.com":
@@ -49,7 +52,7 @@ export default class Gelbooru extends ScrapeEngineBase {
 
     this.log("Guessed version: " + version);
 
-    let post = new ScrapedPost();
+    const post = new ScrapedPost();
     post.pageUrl = document.location.href;
 
     // Set image url
@@ -86,10 +89,12 @@ export default class Gelbooru extends ScrapeEngineBase {
         if (matches && matches.length == 3) {
           // If Source element
           if (matches[1] == "Source") {
-            if (version == Version.v020) {
+            const anchor = el.querySelector("a");
-              post.source = matches[2];
+            if (anchor && anchor.href) {
+              post.sources = [anchor.href];
             } else {
-              post.source = (el.children[0] as HTMLAnchorElement).href;
+              // Fallback to plain text just in case
+              post.sources = [matches[2]];
             }
           }
           // If Rating element
@@ -98,6 +103,7 @@ export default class Gelbooru extends ScrapeEngineBase {
               case "safe":
                 post.rating = "safe";
                 break;
+              case "sensitive":
               case "questionable":
                 post.rating = "sketchy";
                 break;
@@ -134,7 +140,9 @@ export default class Gelbooru extends ScrapeEngineBase {
           // Some <li/> don't actually contain a name.
           // E.g. the sub-headers ("Copyright", "Character", "General", "Meta") on rule34.xxx
           if (nameElements.length > 0) {
-            tagName = el.getElementsByTagName("a")[0].innerText;
+            // Get the last element, because rule34.xxx also adds a "?" link in front.
+            const linkEls = Array.from(el.getElementsByTagName("a"));
+            tagName = linkEls[linkEls.length - 1].innerText;
           } else {
             continue;
           }
@@ -154,7 +162,7 @@ export default class Gelbooru extends ScrapeEngineBase {
           }
         }
 
-        let tag = new ScrapedTag(tagName, category);
+        const tag = new ScrapedTag(tagName, category);
         post.tags.push(tag);
       }
     }
@@ -173,6 +181,7 @@ export default class Gelbooru extends ScrapeEngineBase {
     }
     // End of code duplication with Danbooru engine.
 
+    post.referrer = document.location.origin;
     result.tryAddPost(post);
 
     return result;

src/engines/Inkbunny.test.ts

@@ -1,4 +1,4 @@
-import { scrapeUrl } from "../test/engine";
+import { scrapeUrl } from "../test/engine.js";
 
 test("get inkbunny.net post 2430530", async () => {
   const pageUrl = "https://inkbunny.net/s/2430530";
@@ -7,6 +7,7 @@ test("get inkbunny.net post 2430530", async () => {
   expect(res.posts[0]).toHaveTag("comic");
   expect(res.posts[0]).toHaveTag("flower");
   expect(res.posts[0]).toHaveTag("medusa");
+  expect(res.posts[0]).toHaveResolution([1062, 1521]);
   expect(res.posts[0].pageUrl).toBe(pageUrl);
-  expect(res.posts[0].contentUrl).toBe("https://nl.ib.metapix.net/files/full/3562/3562592_vavacung_medusa.png");
+  expect(res.posts[0].contentUrl).toContain("metapix.net/files/full/3562/3562592_vavacung_medusa.png");
 });

src/engines/Inkbunny.ts

@@ -1,16 +1,14 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
 
-export default class Inkbunny implements ScrapeEngine {
+export default class Inkbunny extends ScrapeEngineBase {
   name = "inkbunny";
-
+  features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags"];
-  canImport(url: Location): boolean {
+  notes = [];
-    return url.host == "inkbunny.net";
+  supportedHosts = ["inkbunny.net"];
-  }
 
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
-
+    const post = new ScrapedPost();
-    let post = new ScrapedPost();
     post.pageUrl = document.location.href;
 
     // Set image url
@@ -33,13 +31,20 @@ export default class Inkbunny implements ScrapeEngine {
       return result;
     }
 
+    // Set post resolution
+    const width = parseInt((<HTMLInputElement>document.getElementById("submission-origwidth"))?.value);
+    const height = parseInt((<HTMLInputElement>document.getElementById("submission-origheight"))?.value);
+    if (!isNaN(width) && !isNaN(height)) {
+      post.resolution = [width, height];
+    }
+
     // Set rating
     const ratingExp = new RegExp("Rating: (.*)");
     const spanEls = Array.from(document.querySelectorAll("div > span"));
     const ratingEls = spanEls.map((x) => x as HTMLDivElement).filter((x) => x.innerText.indexOf("Rating:") != -1);
 
     if (ratingEls.length > 0) {
-      const matches = ratingEls[0].parentElement!.innerText.match(ratingExp);
+      const matches = ratingEls[0].parentElement?.innerText.match(ratingExp);
       if (matches && matches.length > 0) {
         switch (matches[1].toLowerCase()) {
           case "general":
@@ -61,7 +66,7 @@ export default class Inkbunny implements ScrapeEngine {
       const keywords = keywordsContent.split(",").map((x) => x.trim());
 
       for (const keyword of keywords) {
-        let tag = new ScrapedTag(keyword);
+        const tag = new ScrapedTag(keyword);
         post.tags.push(tag);
       }
     }

src/engines/Mastodon.ts

@@ -0,0 +1,145 @@
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapeEngineFeature, ContentType} from "../ScrapeEngine.js";
+
+interface Attachment {
+  url: string;          // original media URL
+  contentType: ContentType;
+}
+
+export default class Mastodon extends ScrapeEngineBase {
+  name = "mastodon";
+  features: ScrapeEngineFeature[] = ["content"];
+  notes = [];
+  supportedHosts = [
+	"aethy.com",
+    "baraag.net",
+    "mastodon.online",
+    "mastodon.social",
+	"mastodon.world",
+	"mstdn.jp",
+	"mstdn.social",
+    "pawoo.net"
+  ];
+  /**
+   * Enabled for any other instances matching the pattern: .../@user/<postID>
+   */
+  canImport(url: Location): boolean {
+	return this.supportedHosts.indexOf(url.host.replace("www.", "")) !== -1
+		|| /^\/@[^/]+\/\d+\/?$/.test(url.pathname);
+  }
+
+  scrapeDocument(document: Document): ScrapeResult {
+	const result = new ScrapeResult(this.name);
+    const attachments = this.findAttachments(document);
+	
+    for (const att of attachments) {
+      const post = new ScrapedPost();
+      post.pageUrl = document.location.href;
+      post.contentUrl = att.url;
+      post.contentType = att.contentType;
+      post.referrer = document.location.origin;
+      result.tryAddPost(post);
+    }
+
+    return result;
+  }
+
+  private findAttachments(document: Document): Attachment[] {
+    const out: Attachment[] = [];
+    const seen = new Set<string>();
+
+    const add = (att: Attachment) => {
+      if (seen.has(att.url)) return;
+      seen.add(att.url);
+      out.push(att);
+    };
+
+    /**
+     *  Images -> a[href]
+	 *
+	 *  <a class="media-gallery__item-thumbnail" href=".../original/....png">
+     *    <img src=".../small/....png">
+     *  </a>
+	 */
+    let anchors = Array.from(
+      document.querySelectorAll<HTMLAnchorElement>("a.media-gallery__item-thumbnail[href]")
+    );
+    if (anchors.length === 0) {
+      anchors = Array.from(
+        document.querySelectorAll<HTMLAnchorElement>("a[href*='/media_attachments/']")
+      );
+    }
+    for (const a of anchors) {
+      const rawHref = a.getAttribute("href");
+      if (!rawHref) continue;
+
+      const img = a.querySelector("img");
+
+      const url = this.resolveUrl(rawHref) || this.largestFromSrcset(img);
+      if (!url) continue;
+
+      add({
+        url,
+        contentType: this.contentTypeFor(url),
+      });
+    }
+
+    /**
+	 *  GIFs and Videos -> video[src]
+	 *
+     *  <div class="media-gallery__gifv autoplay">
+     *    <video class="media-gallery__item-gifv-thumbnail" src=".../original/....mp4" autoplay loop></video>
+     *  </div>
+	 *
+     *  <div role="menuitem" class="video-player">
+     *    <video src=".../original/....mp4" poster=".../small/....png"></video>
+     *  </div>
+	 */
+    const videos = Array.from(document.querySelectorAll<HTMLVideoElement>("video[src]"));
+    for (const v of videos) {
+      const rawSrc = v.getAttribute("src") || v.currentSrc;
+      if (!rawSrc) continue;
+
+      const url = this.resolveUrl(rawSrc);
+      if (!url) continue;
+
+      add({
+        url,
+        contentType: "video",
+      });
+    }
+
+    return out;
+  }
+
+  /**
+   * Resolve possible relative media URLs against current page.
+   */
+  private resolveUrl(href: string): string {
+    try {
+	  return new URL(href, document.location.href).toString();
+    } catch {
+      return href;
+    }
+  }
+
+  private largestFromSrcset(img: HTMLImageElement | null): string | undefined {
+    if (!img) return undefined;
+    const srcset = img.getAttribute("srcset");
+    if (!srcset) return img.src || undefined;
+
+    let best: { url: string; w: number } | undefined;
+    for (const part of srcset.split(",")) {
+      const [urlPart, descriptor] = part.trim().split(/\s+/);
+      if (!urlPart) continue;
+      const w = descriptor?.endsWith("w") ? parseInt(descriptor, 10) : 0;
+      if (!best || w > best.w) best = { url: urlPart, w };
+    }
+    return best?.url;
+  }
+
+  private contentTypeFor(url: string): ContentType {
+    const path = url.split("?")[0].toLowerCase();
+    if (/\.(mp4|webm|mov|m4v)$/.test(path)) return "video";
+    return "image";
+  }
+}

src/engines/Moebooru.test.ts

@@ -1,4 +1,4 @@
-import { scrapeUrl, scrapeEvalUrl } from "../test/engine";
+import { scrapeUrl, scrapeEvalUrl } from "../test/engine.js";
 
 test("get yande.re post 438510", async () => {
   const pageUrl = "https://yande.re/post/show/438510";
@@ -21,6 +21,7 @@ test("get yande.re post 684478 with notes", async () => {
   expect(res.posts[0]).toHaveTag("gym_uniform");
   expect(res.posts[0]).toHaveNote("post #671879");
   expect(res.posts[0]).toHaveResolution([2480, 3508]);
+  expect(res.posts[0]).toHaveSource("https://www.pixiv.net/artworks/84228323");
   expect(res.posts[0].pageUrl).toBe(pageUrl);
   expect(res.posts[0].contentUrl).toBe(
     "https://files.yande.re/image/8bdf7c54b0a459acee080f0aeb2a259c/yande.re%20684478%20gym_uniform%20pal_lett.jpg"
@@ -33,6 +34,7 @@ test("get konachan.com post 289968", async () => {
   expect(res.posts.length).toBe(1);
   expect(res.posts[0]).toHaveTag("league_of_legends", "copyright");
   expect(res.posts[0]).toHaveResolution([1200, 783]);
+  expect(res.posts[0]).toHaveSource("https://www.pixiv.net/member_illust.php?mode=medium&illust_id=72418059");
   expect(res.posts[0].pageUrl).toBe(pageUrl);
   expect(res.posts[0].contentUrl).toBe(
     "https://konachan.com/image/e40b38fb704434e4da3eae41e26bca39/Konachan.com%20-%20289968%20akali%20blue_eyes%20breasts%20choker%20cleavage%20evelynn%20foxgirl%20gloves%20group%20hat%20kai%27sa%20long_hair%20microphone%20ponytail%20saphirya%20short_hair%20signed%20watermark.jpg"
@@ -49,6 +51,7 @@ test("get konachan.com post 327454 with notes", async () => {
   expect(res.posts[0]).toHaveTag("water");
   expect(res.posts[0]).toHaveNote("Under construction.  \nBe careful of your safety");
   expect(res.posts[0]).toHaveResolution([5629, 2505]);
+  expect(res.posts[0]).toHaveSource("https://www.pixiv.net/en/artworks/86644963");
   expect(res.posts[0].pageUrl).toBe(pageUrl);
   expect(res.posts[0].contentUrl).toBe(
     "https://konachan.com/image/94a13c8a1fcb9c0f6c9a6c6c5c316259/Konachan.com%20-%20327454%20apron%20brown_hair%20building%20car%20gloves%20kexue%20long_hair%20original%20rain%20see_through%20short_hair%20twintails%20uniform%20water.jpg"
@@ -71,4 +74,23 @@ test("get konachan.com post 285782 with notes", async () => {
   expect(res.posts[0].contentUrl).toBe(
     "https://konachan.com/image/24f5974ad8ed725292ccb3f682181bc1/Konachan.com%20-%20285782%20anthropomorphism%20bed%20black_eyes%20blonde_hair%20cropped%20fang%20kantai_collection%20long_hair%20momoniku%20pajamas%20white%20yuudachi_%28kancolle%29.jpg"
   );
+  expect(res.posts[0]).toHaveSource("https://www.pixiv.net/member_illust.php?mode=medium&illust_id=75747354");
+});
+
+test("get konachan.com post 358247 jpeg and png", async () => {
+  const pageUrl = "https://konachan.com/post/show/358247/3d-animal_ears-arknights-black_hair-blonde_hair-br";
+  const res = await scrapeUrl(pageUrl);
+  expect(res.posts.length).toBe(2);
+  expect(res.posts[0]).toHaveTag("texas_(arknights)", "character");
+  expect(res.posts[0]).toHaveResolution([6144, 3456]);
+  expect(res.posts[0].pageUrl).toBe(pageUrl);
+  expect(res.posts[0].contentUrl).toBe(
+    "https://konachan.com/image/56983a2166a13f4af963774ccd424f0e/Konachan.com%20-%20358247%203d%20arknights%20computer%20couch%20dress%20drink%20gloves%20gun%20halo%20hood%20necklace%20pantyhose%20red_eyes%20red_hair%20shirt%20shorts%20skirt%20tie%20twintails%20weapon%20wolfgirl.png"
+  );
+  expect(res.posts[0]).toHaveSource("https://www.pixiv.net/en/artworks/108925945");
+
+  // JPEG variant
+  expect(res.posts[1].contentUrl).toBe(
+    "https://konachan.com/jpeg/56983a2166a13f4af963774ccd424f0e/Konachan.com%20-%20358247%203d%20arknights%20computer%20couch%20dress%20drink%20gloves%20gun%20halo%20hood%20necklace%20pantyhose%20red_eyes%20red_hair%20shirt%20shorts%20skirt%20tie%20twintails%20weapon%20wolfgirl.jpg"
+  );
 });

src/engines/Moebooru.ts

@@ -1,19 +1,22 @@
-import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapedNote } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
-import { TagCategory } from "../BooruTypes";
+import { TagCategory } from "../BooruTypes.js";
-import { createNotesFromMoebooruBoxes, guessContentType, parseResolutionString } from "../Utility";
+import { createNotesFromMoebooruBoxes, guessContentType, parseResolutionString } from "../Utility.js";
 
 export default class Moebooru extends ScrapeEngineBase {
   name = "moebooru";
-
+  features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
-  canImport(url: Location): boolean {
+  notes = [];
-    return url.host == "yande.re" || url.host == "konachan.com";
+  supportedHosts = ["yande.re", "konachan.com", "konachan.net", "sakugabooru.com", "img.genshiken-itb.org"];
-  }
 
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
-    let post = new ScrapedPost();
+    const post = new ScrapedPost();
     post.pageUrl = document.location.href;
 
+    if (document.location.host == "yande.re") {
+      post.uploadMode = "content";
+    }
+
     // Set image url
     const originalImageElement = document.querySelector("#highres") as HTMLAnchorElement;
     if (originalImageElement) {
@@ -35,8 +38,8 @@ export default class Moebooru extends ScrapeEngineBase {
       const matches = el.innerText.match(regex);
       if (matches && matches.length == 3) {
         // If Source element
-        if (matches[1] == "Source") {
+        if (matches[1] == "Source" && el.children.length > 0) {
-          post.source = (el.children[0] as HTMLAnchorElement).href;
+          post.sources = [(el.children[0] as HTMLAnchorElement).href];
         }
         // If Rating element
         else if (matches[1] == "Rating") {
@@ -64,12 +67,14 @@ export default class Moebooru extends ScrapeEngineBase {
 
     for (const el of tagElements) {
       let tagName: string;
-      let tagNameElements = el.getElementsByTagName("a");
+      const tagNameElements = el.getElementsByTagName("a");
 
       if (tagNameElements.length > 1) {
         // First <a> is "?"
-        // Second <a> is "actual_tag_name"
+        // (opt) Second <a> is "+" (when logged in)
-        tagName = tagNameElements[1].innerText;
+        // (opt) Third <a> is "-" (when logged in)
+        // Last <a> is the actual tag name
+        tagName = tagNameElements[tagNameElements.length - 1].innerText;
       } else {
         // Skip if no tag name is found
         continue;
@@ -96,7 +101,7 @@ export default class Moebooru extends ScrapeEngineBase {
         }
       }
 
-      let tag = new ScrapedTag(tagName, category);
+      const tag = new ScrapedTag(tagName, category);
       post.tags.push(tag);
     }
 
@@ -109,6 +114,18 @@ export default class Moebooru extends ScrapeEngineBase {
       post.notes = createNotesFromMoebooruBoxes(document, boxSize);
     }
 
+    // Sometimes there is both a JPEG and a PNG version of a post.
+    // If this is the case, copy all the information from the JPEG
+    // version and only replace the contentUrl with the link to the PNG version.
+    // We want to add this PNG version before the other version, so that it appears on top.
+    const unchangedImgEl = document.querySelector<HTMLAnchorElement>("a.original-file-unchanged");
+    if (unchangedImgEl && unchangedImgEl.href != post.contentUrl) {
+      const copy = Object.assign(new ScrapedPost(), post);
+      copy.contentUrl = unchangedImgEl.href;
+      result.tryAddPost(copy);
+    }
+
+    // Add normal/non-png post.
     result.tryAddPost(post);
 
     return result;

src/engines/Nozomi.ts

@@ -0,0 +1,49 @@
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
+import { TagCategory } from "../BooruTypes.js";
+
+export default class e621 extends ScrapeEngineBase {
+  name = "nozomi";
+  features: ScrapeEngineFeature[] = ["content", "tags", "tag_category"];
+  notes = ["Image preview might not work due to hotlink protection."];
+  supportedHosts = ["nozomi.la"];
+
+  scrapeDocument(document: Document): ScrapeResult {
+    const result = new ScrapeResult(this.name);
+    const post = new ScrapedPost();
+    post.pageUrl = document.location.href;
+
+    // Set image url
+    const downloadEl = document.querySelector(".post a") as HTMLAnchorElement;
+    post.contentUrl = downloadEl?.href;
+
+    // Set tags
+    const tagEls = Array.from(document.querySelectorAll(".sidebar li")).map((x) => x as HTMLLIElement);
+
+    for (const el of tagEls) {
+      const tagName = el.innerText;
+      let category: TagCategory | undefined;
+
+      switch (el.querySelector("a")?.className) {
+        // https://e621.net/help/api#tags_listing
+        case "copyright":
+          category = "copyright";
+          break;
+        case "character":
+          category = "character";
+          break;
+        case "artist":
+          category = "artist";
+          break;
+      }
+
+      if (tagName) {
+        const tag = new ScrapedTag(tagName, category);
+        post.tags.push(tag);
+      }
+    }
+
+    result.tryAddPost(post);
+
+    return result;
+  }
+}

src/engines/Philomena.test.ts

@@ -0,0 +1,43 @@
+import { scrapeUrl } from "../test/engine.js";
+
+test("get derpibooru.org post 3153524", async () => {
+  const pageUrl = "https://derpibooru.org/images/3153524";
+  const res = await scrapeUrl(pageUrl);
+  expect(res.posts.length).toBe(1);
+  expect(res.posts[0].rating).toBe("safe");
+  expect(res.posts[0]).not.toHaveTag("safe");
+  expect(res.posts[0]).toHaveTag("yay");
+  expect(res.posts[0]).toHaveTag("flutteryay");
+  expect(res.posts[0]).toHaveSource("https://twitter.com/Confetti_Cakez/status/1674145455666544640");
+  expect(res.posts[0]).toHaveResolution([4000, 4000]);
+  expect(res.posts[0].pageUrl).toBe(pageUrl);
+  expect(res.posts[0].contentUrl).toBe("https://derpicdn.net/img/download/2023/6/29/3153524.jpg");
+});
+
+test("get derpibooru.org post 3157356", async () => {
+  const pageUrl = "https://derpibooru.org/images/3157356?q=oc";
+  const res = await scrapeUrl(pageUrl);
+  expect(res.posts.length).toBe(1);
+  expect(res.posts[0].rating).toBe("safe");
+  expect(res.posts[0]).not.toHaveTag("safe");
+  expect(res.posts[0]).toHaveTag("oc");
+  expect(res.posts[0]).toHaveTag("serra20", "artist");
+  expect(res.posts[0]).toHaveTag("pony", "species");
+  expect(res.posts[0]).toHaveTag("eris_(oc)", "character");
+  expect(res.posts[0]).toHaveResolution([960, 960]);
+  expect(res.posts[0].pageUrl).toBe(pageUrl);
+  expect(res.posts[0].contentUrl).toBe("https://derpicdn.net/img/download/2023/7/5/3157356.png");
+});
+
+test("get derpibooru.org video post 3161674", async () => {
+  const pageUrl = "https://derpibooru.org/images/3161674?q=video";
+  const res = await scrapeUrl(pageUrl);
+  expect(res.posts.length).toBe(1);
+  expect(res.posts[0].rating).toBe("safe");
+  expect(res.posts[0]).not.toHaveTag("safe");
+  expect(res.posts[0]).toHaveTag("oc");
+  expect(res.posts[0]).toHaveTag("rainbow_dash", "character");
+  expect(res.posts[0]).toHaveResolution([1920, 1080]);
+  expect(res.posts[0].pageUrl).toBe(pageUrl);
+  expect(res.posts[0].contentUrl).toBe("https://derpicdn.net/img/download/2023/7/12/3161674.webm");
+});

src/engines/Philomena.ts

@@ -0,0 +1,127 @@
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
+import { SafetyRating, TagCategory } from "../BooruTypes.js";
+import { parseResolutionString } from "../Utility.js";
+
+export default class Philomena extends ScrapeEngineBase {
+  name = "Philomena";
+  features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source"];
+  notes = [];
+  supportedHosts = [
+    "derpibooru.org",
+    "trixiebooru.org",
+    "ponybooru.org",
+    "furbooru.org",
+    "ponerpics.org",
+    "manebooru.art",
+    "twibooru.org",
+  ];
+  removePrefixes = ["artist", "editor", "character", "species", "oc", "ship"];
+  removePrefixesSuffix = new Map([["oc", "_(oc)"]]);
+  artistPrefixes = ["artist", "editor"];
+  safetyMap = new Map<string, SafetyRating>([
+    ["safe", "safe"],
+    ["suggestive", "sketchy"],
+    ["questionable", "sketchy"],
+    ["explicit", "unsafe"],
+  ]);
+  categoryMap = new Map<string, TagCategory>([
+    ["character", "character"],
+    ["species", "species"],
+    ["oc", "character"],
+    ["content-official", "copyright"],
+  ]);
+
+  scrapeDocument(document: Document): ScrapeResult {
+    const result = new ScrapeResult(this.name);
+    const post = new ScrapedPost();
+    post.pageUrl = document.location.href;
+
+    // Set image url
+    const downloadEl = document.querySelector("a[title='Download (no tags in filename)']") as HTMLAnchorElement;
+    post.contentUrl = downloadEl?.href;
+
+    // Set content type
+    if (document.querySelector("video#image-display") != null) {
+      post.contentType = "video";
+    }
+
+    // Set default rating (Will be determined later)
+    post.rating = "unsafe";
+
+    // Set resolution
+    const resStr = (<HTMLElement>document.querySelector("span[class='image-size']"))?.innerText?.match(
+      /(\d+x\d+)/
+    )?.[0];
+    if (resStr) {
+      const res = parseResolutionString(resStr);
+      if (res) {
+        post.resolution = res;
+      }
+    }
+
+    // Set tags
+    const tagEls = Array.from(document.querySelectorAll(".tag.dropdown")).map((x) => x as HTMLSpanElement);
+
+    for (const el of tagEls) {
+      let tagName = el.getAttribute("data-tag-name");
+      if (!tagName) continue;
+
+      // Set rating (philomena uses tags for this)
+      // These tags also have `data-tag-category="rating"` which we could use.
+      const safety = this.safetyMap.get(tagName);
+      if (safety != undefined) {
+        post.rating = safety;
+        // Don't add this as a tag.
+        continue;
+      }
+
+      let category: TagCategory | undefined;
+
+      // Categorize tags using data-tag-category
+      const dataTagCategory = el.getAttribute("data-tag-category");
+      if (dataTagCategory) {
+        const mappedCategory = this.categoryMap.get(dataTagCategory);
+        if (mappedCategory) {
+          category = mappedCategory;
+        }
+      }
+
+      // Artist tag categorize using prefixes
+      for (const prefix of this.artistPrefixes) {
+        if (tagName.includes(prefix + ":")) {
+          category = "artist";
+          break;
+        }
+      }
+
+      // Remove listed prefixes (removePrefixes) from tagName
+      for (const prefix of this.removePrefixes) {
+        if (tagName.includes(prefix + ":")) {
+          tagName = tagName.replace(prefix + ":", "").trim();
+
+          // Add suffix if configured
+          const suffix = this.removePrefixesSuffix.get(prefix);
+          if (suffix) {
+            tagName = tagName + suffix;
+          }
+
+          break;
+        }
+      }
+
+      // Push tag
+      if (tagName) {
+        const tag = new ScrapedTag(tagName, category);
+        post.tags.push(tag);
+      }
+    }
+
+    // Set source
+    const sourceEls = Array.from(document.querySelectorAll("a.js-source-link")).map((x) => x as HTMLAnchorElement);
+    post.sources = sourceEls.map((x) => x.href);
+
+    result.tryAddPost(post);
+
+    return result;
+  }
+}

src/engines/Pixiv.test.ts

@@ -1,5 +1,18 @@
-import { ScrapeResults } from "../ScrapeEngine";
+import { ScrapeResults } from "../ScrapeEngine.js";
-import { scrapeEvalUrl } from "../test/engine";
+import { MobileUserAgent, scrapeEvalUrl, scrapePage } from "../test/engine.js";
+import { upgradeUrlToOriginalQuality } from "./Pixiv.js";
+
+test("upgradeUrlToOriginalQuality", async () => {
+  expect(
+    upgradeUrlToOriginalQuality("https://i.pximg.net/img-master/img/2022/02/15/06/50/28/96268450_p2_master1200.jpg")
+  ).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p2.jpg");
+
+  expect(
+    upgradeUrlToOriginalQuality(
+      "https://i.pximg.net/c/600x1200_90_webp/img-master/img/2022/02/15/06/50/28/96268450_p2_master1200.jpg"
+    )
+  ).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p2.jpg");
+});
 
 describe("get pixiv post 93964654", () => {
   const pageUrl = "https://www.pixiv.net/en/artworks/93964654";
@@ -9,10 +22,72 @@ describe("get pixiv post 93964654", () => {
     res = await scrapeEvalUrl(pageUrl, ".gtm-expand-full-size-illust");
   });
 
-  it("should grab original image url", async () => {
+  it("should find exactly one post", () => {
-    pending("Test does not work.");
+    expect(res.posts.length).toBe(1);
-    // expect(res.posts.length).toBe(1);
+  });
-    // expect(res.posts[0].pageUrl).toBe(pageUrl);
+
-    // expect(res.posts[0].contentUrl).toBe("https://i.pximg.net/img-original/img/2021/11/07/00/01/02/93964654_p0.jpg");
+  it("should grab original image url", () => {
+    expect(res.posts[0].contentUrl).toBe("https://i.pximg.net/img-original/img/2021/11/07/00/01/02/93964654_p0.jpg");
+  });
+});
+
+describe("get pixiv post 93964654 (mobile)", () => {
+  const pageUrl = "https://www.pixiv.net/en/artworks/93964654";
+  let res: ScrapeResults;
+
+  beforeAll(async () => {
+    res = await scrapeEvalUrl(pageUrl, ".work-thumb", MobileUserAgent);
+  });
+
+  it("should find exactly one post", () => {
+    expect(res.posts.length).toBe(1);
+  });
+
+  it("should grab original image url", () => {
+    expect(res.posts[0].contentUrl).toBe("https://i.pximg.net/img-original/img/2021/11/07/00/01/02/93964654_p0.jpg");
+  });
+});
+
+describe("get pixiv post 96268450 (multipage, not expanded)", () => {
+  const pageUrl = "https://www.pixiv.net/en/artworks/96268450";
+  let res: ScrapeResults;
+
+  beforeAll(async () => {
+    res = await scrapeEvalUrl(pageUrl, "div[role='presentation'] img");
+  });
+
+  it("should find exactly one post", () => {
+    expect(res.posts.length).toBe(1);
+  });
+
+  it("should grab original image url", () => {
+    expect(res.posts[0].contentUrl).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p0.jpg");
+  });
+});
+
+describe("get pixiv post 96268450 (multipage, expanded)", () => {
+  const pageUrl = "https://www.pixiv.net/en/artworks/96268450";
+  let res: ScrapeResults;
+
+  beforeAll(async () => {
+    await jestPuppeteer.resetPage();
+    await page.goto(pageUrl);
+    await page.waitForSelector("div[role='presentation'] img", { visible: true });
+    await expect(page).toClick("button", { text: "Show all" });
+    await page.waitForSelector(".gtm-expand-full-size-illust", { visible: true });
+    // await jestPuppeteer.debug();
+    await page.waitForTimeout(2000);
+    res = await scrapePage(pageUrl, page);
+  });
+
+  it("should find exactly four posts", () => {
+    expect(res.posts.length).toBe(4);
+  });
+
+  it("should grab original image url", () => {
+    expect(res.posts[0].contentUrl).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p0.jpg");
+    expect(res.posts[1].contentUrl).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p1.jpg");
+    expect(res.posts[2].contentUrl).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p2.jpg");
+    expect(res.posts[3].contentUrl).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p3.jpg");
   });
 });

src/engines/Pixiv.ts

@@ -1,28 +1,78 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapeEngineFeature } from "../ScrapeEngine.js";
-import { TagCategory } from "../BooruTypes";
+import { guessContentType } from "../Utility.js";
-import { guessContentType } from "../Utility";
 
-export default class Pixiv implements ScrapeEngine {
+export function upgradeUrlToOriginalQuality(imgLink: string) {
-  name = "pixiv";
+  if (imgLink.indexOf("img-master") != -1) {
+    // Example:
+    // Input:  https://i.pximg.net/img-master/img/2022/02/15/06/50/28/96268450_p2_master1200.jpg
+    // Input:  https://i.pximg.net/c/600x1200_90_webp/img-master/img/2022/02/15/06/50/28/96268450_p2_master1200.jpg
+    // Output: https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p2.jpg
+    //
+    // Warning:
+    // This will NOT work for a srcset like this:
+    // https://i.pximg.net/c/540x540_70/img-master/img/2022/02/15/06/50/28/96268450_p0_master1200.jpg 540w,https://i.pximg.net/img-master/img/2022/02/15/06/50/28/96268450_p0_master1200.jpg
+    //
+    // See regexr.com/6rcag for an example.
 
-  canImport(url: Location): boolean {
+    // TODO: Add a check to verify the newly constructed/guessed url?
-    return url.host == "pixiv.net" || url.host == "www.pixiv.net";
+    return imgLink.replace(/(https?:\/\/i\.pximg\.net).*\/img-master\/(.*)_master\d+(.*)/, "$1/img-original/$2$3");
   }
 
-  scrapeDocument(document: Document): ScrapeResult {
+  return imgLink;
-    let result = new ScrapeResult(this.name);
+}
 
-    const imgLinks = Array.from(document.querySelectorAll("div[role='presentation'] > a")).map(
+export default class Pixiv extends ScrapeEngineBase {
-      (x) => x as HTMLAnchorElement
+  name = "pixiv";
+  features: ScrapeEngineFeature[] = ["content"];
+  notes = [];
+  supportedHosts = ["pixiv.net"];
+
+  scrapeDocument(document: Document): ScrapeResult {
+    const result = new ScrapeResult(this.name);
+    const imgLinks = [] as string[];
+
+    // Query for when the user is on the desktop site, is logged in, and has optionally clicked on "Show all".
+    // If the user has not clicked on "Show all" then this will only grab the first post.
+    imgLinks.push(
+      ...Array.from(document.querySelectorAll<HTMLAnchorElement>("div[role='presentation'] > a")).map((x) => x.href)
     );
 
+    if (imgLinks.length == 0) {
+      // Query for when the user is on the desktop site and not logged in.
+      imgLinks.push(
+        ...Array.from(document.querySelectorAll<HTMLImageElement>("div[role='presentation'] img")).map((x) => x.src)
+      );
+    }
+
+    if (imgLinks.length == 0) {
+      // Query for when the user is on the mobile site, and has clicked on "Show all".
+      // This only works on pages with multiple posts. The user can optionally be logged in.
+      imgLinks.push(
+        ...Array.from(document.querySelectorAll<HTMLImageElement>(".manga-page > a > img[data-big]")).map(
+          // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+          (x) => x.getAttribute("data-big")!
+        )
+      );
+    }
+
+    if (imgLinks.length == 0) {
+      // Query for when the user is on the mobile site, and has not clicked on "See all" -or- there is only one post.
+      const imgLink = document.querySelector<HTMLImageElement>(".work-main-image img")?.src;
+      if (imgLink) {
+        imgLinks.push(imgLink);
+      }
+    }
+
     for (const imgLink of imgLinks) {
-      let post = new ScrapedPost();
+      const post = new ScrapedPost();
       post.pageUrl = document.location.href;
-      post.contentUrl = imgLink.href;
+      post.contentUrl = upgradeUrlToOriginalQuality(imgLink);
       post.contentType = guessContentType(post.contentUrl);
       post.referrer = document.location.origin;
       result.tryAddPost(post);
+      if (document.querySelector('a[href*="/tags/R-18/artworks"]')) {
+        post.rating = "unsafe";
+      }
     }
 
     return result;

src/engines/Reddit.test.ts

@@ -1,4 +1,4 @@
-import { scrapeUrl } from "../test/engine";
+import { scrapeUrl } from "../test/engine.js";
 
 test("get old.reddit.com post", async () => {
   const pageUrl = "https://old.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/";

src/engines/Reddit.ts

@@ -1,16 +1,15 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapeEngineFeature } from "../ScrapeEngine.js";
-import { guessContentType } from "../Utility";
+import { guessContentType } from "../Utility.js";
 
-export default class Reddit implements ScrapeEngine {
+export default class Reddit extends ScrapeEngineBase {
   name = "reddit";
-
+  features: ScrapeEngineFeature[] = ["content"];
-  canImport(url: Location): boolean {
+  notes = [];
-    return url.host.endsWith("reddit.com");
+  supportedHosts = ["reddit.com", "new.reddit.com", "old.reddit.com"];
-  }
 
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
-    let post = new ScrapedPost();
+    const post = new ScrapedPost();
     post.pageUrl = document.location.href;
 
     // Set image url

src/engines/Rule34us.test.ts

@@ -1,4 +1,4 @@
-import { scrapeUrl } from "../test/engine";
+import { scrapeUrl } from "../test/engine.js";
 
 test("get rule34.us post 4646579", async () => {
   // Kinda sketchy.

src/engines/Rule34us.ts

@@ -1,13 +1,16 @@
 // Technically speaking rule34.us also uses gelbooru, albeit a (heavily?) modified one.
 // Because pretty much no scraper code can be shared with other gelbooru-likes it makes more sense to create a new engine for rule34.us.
 
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
-import { TagCategory } from "../BooruTypes";
+import { TagCategory } from "../BooruTypes.js";
-import { CategoryMap } from "./Common";
+import { CategoryMap } from "./Common.js";
-import { guessContentType, parseResolutionString } from "../Utility";
+import { guessContentType, parseResolutionString } from "../Utility.js";
 
-export default class Gelbooru implements ScrapeEngine {
+export default class Gelbooru extends ScrapeEngineBase {
   name = "rule34us";
+  features: ScrapeEngineFeature[] = ["content", "resolution", "tags", "tag_category"];
+  notes = ["Rating is assumed to be unsafe."];
+  supportedHosts = ["rule34.us"];
 
   private readonly classNameToCategoryMap: CategoryMap = {
     "character-tag": "character",
@@ -16,14 +19,9 @@ export default class Gelbooru implements ScrapeEngine {
     "metadata-tag": "meta",
   };
 
-  canImport(url: Location): boolean {
-    return url.host == "rule34.us";
-  }
-
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
-
+    const post = new ScrapedPost();
-    let post = new ScrapedPost();
     post.pageUrl = document.location.href;
 
     // Set image url

src/engines/SankakuComplex.test.ts

@@ -1,37 +1,15 @@
-import { scrapeUrl } from "../test/engine";
+import { scrapeUrl } from "../test/engine.js";
 
 test("get chan.sankakucomplex.com post 19614481 with notes", async () => {
-  const pageUrl = "https://chan.sankakucomplex.com/post/show/19614481";
+  const pageUrl = "https://chan.sankakucomplex.com/en/posts/porqpNvZBa7";
   const res = await scrapeUrl(pageUrl);
   expect(res.posts.length).toBe(1);
   expect(res.posts[0]).toHaveTag("bangs");
   expect(res.posts[0]).toHaveTag("bare_shoulders");
-  expect(res.posts[0]).toHaveTag("solo");
-  expect(res.posts[0]).toHaveTag("translated");
-  expect(res.posts[0]).toHaveTag("text");
   expect(res.posts[0]).toHaveTag("idolmaster");
   expect(res.posts[0]).toHaveTag("mayuzumi_fuyuko");
-  expect(res.posts[0]).toHaveNote("What are you looking at...");
+  // expect(res.posts[0]).toHaveNote("What are you looking at...");
-  expect(res.posts[0]).toHaveNote("What...?");
+  // expect(res.posts[0]).toHaveNote("What...?");
   expect(res.posts[0].pageUrl).toBe(pageUrl);
-  expect(res.posts[0].contentUrl).toContain(
+  expect(res.posts[0].contentUrl).toContain("sankakucomplex.com/data/50/7e/507e986a62cc5b866d92ad491dcfb1a4.jpg");
-    "https://s.sankakucomplex.com/data/50/7e/507e986a62cc5b866d92ad491dcfb1a4.jpg"
-  );
-});
-
-test("get chan.sankakucomplex.com post 29354061 with notes", async () => {
-  // Kinda sketchy
-  const pageUrl = "https://chan.sankakucomplex.com/post/show/29354061";
-  const res = await scrapeUrl(pageUrl);
-  expect(res.posts.length).toBe(1);
-  expect(res.posts[0]).toHaveTag("bangs");
-  expect(res.posts[0]).toHaveTag("braid");
-  expect(res.posts[0]).toHaveTag("sankuro_(agoitei)");
-  expect(res.posts[0]).toHaveTag("translated");
-  expect(res.posts[0]).toHaveTag("note_translated");
-  expect(res.posts[0]).toHaveNote("No.  \nI already told you...");
-  expect(res.posts[0].pageUrl).toBe(pageUrl);
-  expect(res.posts[0].contentUrl).toContain(
-    "https://s.sankakucomplex.com/data/c8/37/c8376f02608548365d394a658e85cb3f.jpg"
-  );
 });

src/engines/SankakuComplex.ts

@@ -1,17 +1,16 @@
-import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapedNote } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
-import { TagCategory } from "../BooruTypes";
+import { TagCategory } from "../BooruTypes.js";
-import { createNotesFromMoebooruBoxes, guessContentType } from "../Utility";
+import { createNotesFromMoebooruBoxes, guessContentType } from "../Utility.js";
 
 export default class SankakuComplex extends ScrapeEngineBase {
   name = "sankakucomplex";
-
+  features: ScrapeEngineFeature[] = ["content", "rating", "tags", "tag_category", "notes"];
-  canImport(url: Location): boolean {
+  notes = [];
-    return url.host == "chan.sankakucomplex.com";
+  supportedHosts = ["chan.sankakucomplex.com"];
-  }
 
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
-    let post = new ScrapedPost();
+    const post = new ScrapedPost();
     post.pageUrl = document.location.href;
 
     // Set image url
@@ -28,25 +27,14 @@ export default class SankakuComplex extends ScrapeEngineBase {
     post.contentType = guessContentType(post.contentUrl);
 
     // Set rating
-    const safetyExp = new RegExp("Rating: (.*)");
+    const ratingElement = document.querySelector("#stats .rating-s, #stats .rating-q, #stats .rating-e");
-    const safetyElements = Array.from(document.querySelectorAll("#stats > ul > li"))
+    if (ratingElement) {
-      .map((x) => x as HTMLLIElement)
+      if (ratingElement.classList.contains("rating-s")) {
-      .filter((x) => x.innerText.startsWith("Rating"));
-
-    if (safetyElements.length > 0) {
-      const matches = safetyElements[0].innerText.match(safetyExp);
-      if (matches && matches.length > 0) {
-        switch (matches[1].toLowerCase()) {
-          case "safe":
         post.rating = "safe";
-            break;
+      } else if (ratingElement.classList.contains("rating-q")) {
-          case "questionable":
         post.rating = "sketchy";
-            break;
+      } else if (ratingElement.classList.contains("rating-e")) {
-          case "explicit":
         post.rating = "unsafe";
-            break;
-        }
       }
     }
 
@@ -76,7 +64,7 @@ export default class SankakuComplex extends ScrapeEngineBase {
       }
 
       if (tagName) {
-        let tag = new ScrapedTag(tagName, category);
+        const tag = new ScrapedTag(tagName, category);
         post.tags.push(tag);
       }
     }

src/engines/Shimmie2.test.ts

@@ -1,11 +1,13 @@
-import { scrapeUrl } from "../test/engine";
+import { scrapeUrl } from "../test/engine.js";
 
 test("get rule34.paheal.net post 2505982", async () => {
   const pageUrl = "https://rule34.paheal.net/post/view/2505982"; // Somewhat sketchy
   const res = await scrapeUrl(pageUrl);
   expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("kai'sa");
   expect(res.posts[0].pageUrl).toBe(pageUrl);
-  expect(res.posts[0].contentUrl).toContain(
+  expect(res.posts[0]).toHaveSource(
-    ".paheal.net/_images/15427ab5ed2cddbeef947e30f5c9cab6/2505982%20-%20Kai%27Sa%20League_of_Legends%20lawzilla.png"
+    "http://law-zilla.tumblr.com/post/171249285285/finished-subdraw-37-kaisa-from-league-of-legends"
   );
+  expect(res.posts[0].contentUrl).toContain("https://r34i.paheal-cdn.net");
 });

src/engines/Shimmie2.ts

@@ -1,31 +1,33 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
-import { guessContentType } from "../Utility";
+import { guessContentType } from "../Utility.js";
 
-export default class Shimmie2 implements ScrapeEngine {
+export default class Shimmie2 extends ScrapeEngineBase {
   name = "shimmie2";
-
+  features: ScrapeEngineFeature[] = ["content", "tags", "source"];
-  canImport(url: Location): boolean {
+  notes = ["Rating is assumed to be unsafe."];
-    return url.host == "rule34.paheal.net" || url.host == "rule34hentai.net";
+  supportedHosts = ["rule34.paheal.net", "rule34hentai.net"];
-  }
 
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
-    let post = new ScrapedPost();
+    const post = new ScrapedPost();
     post.pageUrl = document.location.href;
 
     // Set image url
-    const originalImageElements = Array.from(document.querySelectorAll(".image_info > tbody > tr > td > a"))
+    // const originalImageElements = Array.from(document.querySelectorAll(".image_info > tbody > tr > td > a"))
-      .map((x) => x as HTMLAnchorElement)
+    //   .map((x) => x as HTMLAnchorElement)
-      .filter((x) => x.innerText == "Image Only" || x.innerText == "File Only");
+    //   .filter((x) => x.innerText == "Image Only" || x.innerText == "File Only");
 
-    if (originalImageElements.length > 0) {
+    // if (originalImageElements.length > 0) {
-      post.contentUrl = originalImageElements[0].href;
+    //   post.contentUrl = originalImageElements[0].href;
-    } else {
+    // } else {
-      const dowloadBtnEl = document.querySelector("a[download]") as HTMLAnchorElement;
+    //   const dowloadBtnEl = document.querySelector("a[download]") as HTMLAnchorElement;
-      if (dowloadBtnEl) {
+    //   if (dowloadBtnEl) {
-        post.contentUrl = dowloadBtnEl.href;
+    //     post.contentUrl = dowloadBtnEl.href;
-      }
+    //   }
-    }
+    // }
+
+    // @ts-expect-error
+    post.contentUrl = document.getElementById("main_image").src;
 
     // Set content type
     post.contentType = guessContentType(post.contentUrl);
@@ -45,7 +47,7 @@ export default class Shimmie2 implements ScrapeEngine {
     // Set source
     const sourceEl = document.querySelector(".image_info > tbody > tr > td > div > a") as HTMLAnchorElement;
     if (sourceEl) {
-      post.source = sourceEl.href;
+      post.sources = [sourceEl.href];
     }
 
     result.tryAddPost(post);

src/engines/Shuushuu.test.ts

@@ -1,4 +1,4 @@
-import { scrapeUrl } from "../test/engine";
+import { scrapeUrl } from "../test/engine.js";
 
 test("get e-shuushuu.net post 1063586", async () => {
   const pageUrl = "https://e-shuushuu.net/image/1063586/";

src/engines/Shuushuu.ts

@@ -1,17 +1,16 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
-import { TagCategory } from "../BooruTypes";
+import { TagCategory } from "../BooruTypes.js";
-import { guessContentType, parseResolutionString } from "../Utility";
+import { guessContentType, parseResolutionString } from "../Utility.js";
 
-export default class Shuushuu implements ScrapeEngine {
+export default class Shuushuu extends ScrapeEngineBase {
   name = "shuushuu";
-
+  features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category"];
-  canImport(url: Location): boolean {
+  notes = ["Rating is assumed to be safe."];
-    return url.host == "e-shuushuu.net";
+  supportedHosts = ["e-shuushuu.net"];
-  }
 
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
-    let post = new ScrapedPost();
+    const post = new ScrapedPost();
     post.pageUrl = document.location.href;
 
     // Set image url

src/engines/Twitter.test.ts

@@ -1,5 +1,5 @@
-import { ScrapeResults } from "../ScrapeEngine";
+import { ScrapeResults } from "../ScrapeEngine.js";
-import { scrapeEvalUrl } from "../test/engine";
+import { scrapeEvalUrl } from "../test/engine.js";
 
 describe("get twitter.com post", () => {
   const pageUrl = "https://twitter.com/Dishwasher1910/status/1171162651461746688/photo/1";

src/engines/Twitter.ts

@@ -1,30 +1,29 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapeEngineFeature } from "../ScrapeEngine.js";
 
 export function getOriginalImageUrl(image: HTMLImageElement) {
   // if the link includes a resolution, change it to orig:
   // https://pbs.twimg.com/media/stuff?format=jpg&name=900x900
   // -> https://pbs.twimg.com/media/stuff?format=jpg&name=orig
-  let img_url = new URL((image as HTMLImageElement).src);
+  const img_url = new URL((image as HTMLImageElement).src);
   if (img_url.searchParams.has("name") && img_url.searchParams.get("name") != "orig") {
     img_url.searchParams.set("name", "orig");
   }
   return img_url;
 }
 
-export default class Twitter implements ScrapeEngine {
+export default class Twitter extends ScrapeEngineBase {
   name = "twitter";
-
+  features: ScrapeEngineFeature[] = ["content"];
-  canImport(url: Location): boolean {
+  notes = [];
-    return url.host == "twitter.com" || url.host == "mobile.twitter.com";
+  supportedHosts = ["twitter.com", "mobile.twitter.com", "x.com"];
-  }
 
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
 
     // first look for a focused image on desktop:
     const center_img = document.querySelector("div[role=dialog] div[role=dialog] div[aria-label=Image] img");
     if (center_img) {
-      let post = new ScrapedPost();
+      const post = new ScrapedPost();
       post.pageUrl = document.location.href;
       post.contentUrl = getOriginalImageUrl(center_img as HTMLImageElement).href;
       result.tryAddPost(post);
@@ -34,7 +33,7 @@ export default class Twitter implements ScrapeEngine {
     const imgs = [...document.querySelectorAll("article a[role='link'] div[data-testid='tweetPhoto'] img")];
 
     for (const img of imgs) {
-      let post = new ScrapedPost();
+      const post = new ScrapedPost();
       post.pageUrl = document.location.href;
       post.contentUrl = getOriginalImageUrl(img as HTMLImageElement).href;
       result.tryAddPost(post);

src/engines/Zerochan.test.ts

@@ -1,4 +1,4 @@
-import { scrapeUrl } from "../test/engine";
+import { scrapeUrl } from "../test/engine.js";
 
 test("get zerochan.net post 2934998", async () => {
   const pageUrl = "https://www.zerochan.net/2934998";
@@ -13,11 +13,11 @@ test("get zerochan.net post 2934998", async () => {
 });
 
 test("get zerochan.net gif post 3511809", async () => {
-  const pageUrl = "https://www.zerochan.net/3511809";
+  const pageUrl = "https://www.zerochan.net/4356451";
   const res = await scrapeUrl(pageUrl);
   expect(res.posts.length).toBe(1);
-  expect(res.posts[0]).toHaveTag("rwby");
+  expect(res.posts[0]).toHaveTag("honkai_star_rail");
-  expect(res.posts[0]).toHaveResolution([500, 150]);
+  expect(res.posts[0]).toHaveResolution([2192, 1644]);
   expect(res.posts[0].pageUrl).toBe(pageUrl);
-  expect(res.posts[0].contentUrl).toBe("https://static.zerochan.net/Ruby.Rose.full.3511809.gif");
+  expect(res.posts[0].contentUrl).toBe("https://static.zerochan.net/Honkai.Star.Rail.full.4356451.jpg");
 });

src/engines/Zerochan.ts

@@ -1,22 +1,21 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
-import { TagCategory } from "../BooruTypes";
+import { TagCategory } from "../BooruTypes.js";
-import { guessContentType, parseResolutionString } from "../Utility";
+import { guessContentType, parseResolutionString } from "../Utility.js";
 
-export default class Zerochan implements ScrapeEngine {
+export default class Zerochan extends ScrapeEngineBase {
   name = "zerochan";
-
+  features: ScrapeEngineFeature[] = ["content", "resolution", "tags", "tag_category"];
-  canImport(url: Location): boolean {
+  notes = ["Rating is assumed to be safe."];
-    return url.host.endsWith("zerochan.net");
+  supportedHosts = ["zerochan.net"];
-  }
 
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
-    let post = new ScrapedPost();
+    const post = new ScrapedPost();
     post.pageUrl = document.location.href;
 
     // Set image url
     const previewElements = document.getElementsByClassName("preview");
-    if (previewElements.length > 0) {
+    if (previewElements.length > 0 && (previewElements[0] as HTMLAnchorElement).href) {
       // The preview element only exists when there is a larger size available. Preview's img element contains a downscaled version.
       post.contentUrl = (previewElements[0] as HTMLAnchorElement).href;
     } else {
@@ -31,9 +30,9 @@ export default class Zerochan implements ScrapeEngine {
     post.contentType = guessContentType(post.contentUrl);
 
     // Set resolution
-    const resSizeEl = document.querySelector("#large > p > br")?.parentNode as HTMLParagraphElement;
+    const resSizeEl = document.querySelector("#image-info")?.firstElementChild;
     if (resSizeEl) {
-      post.resolution = parseResolutionString(resSizeEl.childNodes[0].textContent);
+      post.resolution = parseResolutionString(resSizeEl.textContent?.replace("×", "x"));
     }
 
     // Set rating
@@ -53,26 +52,26 @@ export default class Zerochan implements ScrapeEngine {
       }
 
       let category: TagCategory | undefined;
-      switch (tagElement.className) {
+      const className = tagElement.classList[0];
+      switch (className) {
         case "game":
         case "series":
         case "studio":
           category = "copyright";
           break;
         case "character":
-        case "character group":
           category = "character";
           break;
         case "mangaka":
           category = "artist";
           break;
         case "source":
-          // Skip these type of tags, because they are not 'booru-like'
+          // Skip these type of tags, because they are not 'booru-like'.
           continue;
       }
 
       if (tagName) {
-        let tag = new ScrapedTag(tagName, category);
+        const tag = new ScrapedTag(tagName, category);
         post.tags.push(tag);
       }
     }

src/engines/e621.test.ts

@@ -0,0 +1,15 @@
+import { scrapeUrl } from "../test/engine.js";
+
+test("get e621.net post 380315", async () => {
+  const pageUrl = "https://e621.net/posts/380315?q=league_of_legends+rating%3Asafe+order%3Ascore";
+  const res = await scrapeUrl(pageUrl);
+  expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("inkinesss", "artist");
+  expect(res.posts[0]).toHaveTag("nasus_(lol)", "character");
+  expect(res.posts[0]).toHaveTag("ambiguous_gender");
+  expect(res.posts[0]).toHaveTag("low_res", "meta");
+  expect(res.posts[0]).toHaveResolution([500, 500]);
+  expect(res.posts[0]).toHaveSource("https://inkinesss.deviantart.com/art/Nasus-388347996");
+  expect(res.posts[0].pageUrl).toBe(pageUrl);
+  expect(res.posts[0].contentUrl).toBe("https://static1.e621.net/data/92/e7/92e71bd73ab219d80516c4d9adad3db5.jpg");
+});

src/engines/e621.ts

@@ -1,21 +1,19 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
-import { TagCategory } from "../BooruTypes";
+import { TagCategory } from "../BooruTypes.js";
 
-export default class e621 implements ScrapeEngine {
+export default class e621 extends ScrapeEngineBase {
   name = "e621";
-
+  features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source"];
-  canImport(url: Location): boolean {
+  notes = [];
-    return url.host == "e621.net" || url.host == "e926.net";
+  supportedHosts = ["e621.net", "e926.net", "e6ai.net"];
-  }
 
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
-
+    const post = new ScrapedPost();
-    let post = new ScrapedPost();
     post.pageUrl = document.location.href;
 
     // Set image url
-    const downloadEl = document.querySelector("#image-download-link > a") as HTMLAnchorElement;
+    const downloadEl = document.querySelector(".ptbr-fullscreen-button") as HTMLAnchorElement;
     post.contentUrl = downloadEl?.href;
 
     // Set content type
@@ -37,50 +35,54 @@ export default class e621 implements ScrapeEngine {
         break;
     }
 
+    // Set resolution
+    const width = parseInt((<HTMLSpanElement>document.querySelector("span[itemprop='width']"))?.innerText);
+    const height = parseInt((<HTMLSpanElement>document.querySelector("span[itemprop='height']"))?.innerText);
+    if (!isNaN(width) && !isNaN(height)) {
+      post.resolution = [width, height];
+    }
+
     // Set tags
     const tagEls = Array.from(document.querySelectorAll("#tag-list > ul > li")).map((x) => x as HTMLLIElement);
 
     for (const el of tagEls) {
-      const tagName = (el.querySelector(".search-tag") as HTMLAnchorElement)?.innerText;
+      const tagName = decodeURIComponent(el.getAttribute("data-name") as string);
       let category: TagCategory | undefined;
 
-      switch (el.className) {
+      switch (el.getAttribute("data-category")) {
         // https://e621.net/help/api#tags_listing
-        case "category-3":
+        case "copyright":
           category = "copyright";
           break;
-        case "category-4":
+        case "character":
           category = "character";
           break;
-        case "category-1":
+        case "artist":
           category = "artist";
           break;
-        case "category-7":
+        case "meta":
           category = "meta";
           break;
-        case "category-5":
+        case "species":
           category = "species";
           break;
-        // case "category-6":
+        // case "invalid":
         //   category = "invalid";
         //   break;
-        case "category-8":
+        case "lore":
           category = "lore";
           break;
       }
 
       if (tagName) {
-        let tag = new ScrapedTag(tagName, category);
+        const tag = new ScrapedTag(tagName, category);
         post.tags.push(tag);
       }
     }
 
     // Set source
     const sourceEls = Array.from(document.querySelectorAll(".source-link > a")).map((x) => x as HTMLAnchorElement);
-    // We currently only support one source.
+    post.sources = sourceEls.map((x) => x.href);
-    if (sourceEls.length > 0) {
-      post.source = sourceEls[0].href;
-    }
 
     result.tryAddPost(post);
 

src/engines/index.ts

@@ -1,16 +1,21 @@
-export { default as AnimePictures } from "./AnimePictures";
+export { default as AnimePictures } from "./AnimePictures.js";
-export { default as Danbooru } from "./Danbooru";
+export { default as Bluesky } from "./Bluesky.js";
-export { default as e621 } from "./e621";
+export { default as Danbooru } from "./Danbooru.js";
-export { default as Fallback } from "./Fallback";
+export { default as e621 } from "./e621.js";
-export { default as FurAffinity } from "./FurAffinity";
+export { default as Ehentai } from "./Ehentai.js";
-export { default as Gelbooru } from "./Gelbooru";
+export { default as Fallback } from "./Fallback.js";
-export { default as Inkbunny } from "./Inkbunny";
+export { default as FurAffinity } from "./FurAffinity.js";
-export { default as Moebooru } from "./Moebooru";
+export { default as Gelbooru } from "./Gelbooru.js";
-export { default as Pixiv } from "./Pixiv";
+export { default as Inkbunny } from "./Inkbunny.js";
-export { default as Reddit } from "./Reddit";
+export { default as Mastodon } from "./Mastodon.js";
-export { default as Rule34us } from "./Rule34us";
+export { default as Moebooru } from "./Moebooru.js";
-export { default as SankakuComplex } from "./SankakuComplex";
+export { default as Nozomi } from "./Nozomi.js";
-export { default as Shimmie2 } from "./Shimmie2";
+export { default as Philomena } from "./Philomena.js";
-export { default as Shuushuu } from "./Shuushuu";
+export { default as Pixiv } from "./Pixiv.js";
-export { default as Twitter } from "./Twitter";
+export { default as Reddit } from "./Reddit.js";
-export { default as Zerochan } from "./Zerochan";
+export { default as Rule34us } from "./Rule34us.js";
+export { default as SankakuComplex } from "./SankakuComplex.js";
+export { default as Shimmie2 } from "./Shimmie2.js";
+export { default as Shuushuu } from "./Shuushuu.js";
+export { default as Twitter } from "./Twitter.js";
+export { default as Zerochan } from "./Zerochan.js";

src/index.ts

@@ -1,8 +1,8 @@
-export { default as NeoScraper } from "./NeoScraper";
+export { default as NeoScraper } from "./NeoScraper.js";
-export * from "./ScrapeEngine";
+export * from "./ScrapeEngine.js";
 
-import * as Engines from "./engines";
+import * as Engines from "./ScrapeEngine.js";
 export { Engines };
 
-import * as BooruTypes from "./BooruTypes";
+import * as BooruTypes from "./BooruTypes.js";
 export { BooruTypes };

src/test/engine.ts

@@ -1,5 +1,7 @@
 import { JSDOM, ConstructorOptions } from "jsdom";
-import NeoScraper from "../NeoScraper";
+import NeoScraper from "../NeoScraper.js";
+
+export const MobileUserAgent = "Mozilla/5.0 (Android 12; Mobile; rv:68.0) Gecko/68.0 Firefox/103.0";
 
 /**
  * Go to an URL and parse it's raw HTML without executing any client-side code.
@@ -7,7 +9,9 @@ import NeoScraper from "../NeoScraper";
  * @returns
  */
 export async function scrapeUrl(url: string) {
-  const dom = await JSDOM.fromURL(url);
+  const dom = await JSDOM.fromURL(url, {
+    userAgent: "PostmanRuntime/7.37.3",
+  });
   return scrapeDom(dom.window.document);
 }
 
@@ -17,9 +21,26 @@ export async function scrapeUrl(url: string) {
  * @param waitForSelector
  * @returns
  */
-export async function scrapeEvalUrl(url: string, waitForSelector: string) {
+export async function scrapeEvalUrl(
+  url: string,
+  waitForSelector: string,
+  userAgent: string | undefined = "PostmanRuntime/7.37.3"
+) {
+  await jestPuppeteer.resetPage();
+
+  if (userAgent) {
+    page.setUserAgent(userAgent);
+  }
+
   await page.goto(url);
-  await page.waitForSelector(waitForSelector, { visible: true });
+
+  const res = await page.waitForSelector(waitForSelector, { visible: true, timeout: 10_000 });
+  if (res == null) throw new Error("waitForSelector timed out.");
+
+  return scrapePage(url, page);
+}
+
+export async function scrapePage(url: string, page: any) {
   const document = await page.content();
   return scrapeHtml(url, document);
 }

src/test/jest.d.ts

@@ -1,13 +1,15 @@
 import "jest-extended";
-import { TagCategory } from "../BooruTypes";
+import { TagCategory } from "../BooruTypes.js";
 
 declare global {
   namespace jest {
     interface Matchers<R> {
-      // The `= undefined` part is incorrect according to VSCode, but it is actually required.
+      toHaveTag(tagName: string, tagCategory?: TagCategory): CustomMatcherResult;
-      toHaveTag(tagName: string, tagCategory: TagCategory | undefined = undefined): CustomMatcherResult;
       toHaveNote(noteText: string): CustomMatcherResult;
       toHaveResolution(resolution: [number, number]): CustomMatcherResult;
+      toHaveSource(source: string): CustomMatcherResult;
     }
   }
 }
+
+export {};

src/test/jest.setup.ts

@@ -1,6 +1,6 @@
 import { JSDOM } from "jsdom";
-import { TagCategory } from "../BooruTypes";
+import { TagCategory } from "../BooruTypes.js";
-import { ScrapedPost } from "../ScrapeEngine";
+import { ScrapedPost } from "../ScrapeEngine.js";
 
 // Expose JSDOM Element constructor
 global.Element = new JSDOM().window.Element;
@@ -19,7 +19,7 @@ expect.extend({
     tagName: string,
     tagCategory: TagCategory | undefined = undefined
   ): jest.CustomMatcherResult {
-    let pass: boolean = false;
+    let pass = false;
     const message = () => (pass ? "" : `Post does not have the '${tagName}' tag, or the category is incorrect.`);
 
     if (received instanceof ScrapedPost) {
@@ -35,7 +35,7 @@ expect.extend({
     return { pass, message };
   },
   toHaveNote(received: any, noteText: string): jest.CustomMatcherResult {
-    let pass: boolean = false;
+    let pass = false;
     const message = () => (pass ? "" : `Post does not have a note with '${noteText}'.`);
 
     if (received instanceof ScrapedPost) {
@@ -44,7 +44,7 @@ expect.extend({
     return { pass, message };
   },
   toHaveResolution(received: any, resolution: [number, number]): jest.CustomMatcherResult {
-    let pass: boolean = false;
+    let pass = false;
     const message = () => (pass ? "" : `Expected: ${resolution}\nReceived: ${received?.resolution}`);
 
     if (received instanceof ScrapedPost && received.resolution?.length == 2) {
@@ -52,4 +52,13 @@ expect.extend({
     }
     return { pass, message };
   },
+  toHaveSource(received: any, source: string): jest.CustomMatcherResult {
+    let pass = false;
+    const message = () => (pass ? "" : `Post does not have source '${source}'.`);
+
+    if (received instanceof ScrapedPost) {
+      pass = received.sources.find((x) => x == source) != undefined;
+    }
+    return { pass, message };
+  },
 });

tsconfig.json

@@ -1,16 +1,17 @@
 {
   "compilerOptions": {
     "lib": [
-      "es2019",
+      "ES2019",
-      "dom",
+      "DOM",
-      "dom.iterable"
+      "DOM.Iterable"
     ],
+    "rootDir": "./src/",
     "outDir": "./dist/",
     "sourceMap": true,
     "strict": true,
-    "module": "es2015",
+    "module": "node16",
-    "moduleResolution": "node",
+    "moduleResolution": "node16",
-    "target": "es2015",
+    "target": "es2020",
     "skipLibCheck": true,
     "esModuleInterop": true,
     "removeComments": true,

tsconfig.test.json

@@ -1,16 +1,16 @@
 {
   "compilerOptions": {
     "lib": [
-      "es2019",
+      "ES2019",
-      "dom",
+      "DOM",
-      "dom.iterable"
+      "DOM.Iterable"
     ],
     "outDir": "./dist/",
     "sourceMap": true,
     "strict": true,
-    "module": "es2015",
+    "module": "ESNext",
-    "moduleResolution": "node",
+    "moduleResolution": "NodeNext",
-    "target": "es2015",
+    "target": "ESNext",
     "skipLibCheck": true,
     "esModuleInterop": true,
     "removeComments": true,