Bugfixes

Fixed safety detection on chan.sankakucomplex.com, and improved source parsing on Gelbooru v0.2.0 sites.
Update Bluesky.ts
2026-10-17 19:29:36 -07:00 · 2026-10-13 23:46:36 -07:00 · 2026-10-13 23:14:14 -07:00 · 2026-10-12 20:50:39 -07:00 · 2026-10-11 16:46:03 -07:00 · 2026-10-11 16:41:41 -07:00
60 changed files with 7771 additions and 5137 deletions
--- a/.eslintrc.cjs
+++ b/.eslintrc.cjs
@ -0,0 +1,17 @@
+module.exports = {
+  env: {
+    browser: true,
+    commonjs: true,
+    es2021: true,
+    node: true,
+  },
+  extends: ["eslint:recommended", "plugin:@typescript-eslint/recommended"],
+  parser: "@typescript-eslint/parser",
+  parserOptions: {
+    ecmaVersion: "latest",
+  },
+  plugins: ["@typescript-eslint"],
+  rules: {
+    semi: ["error", "always"],
+  },
+};
--- a/.gitignore
+++ b/.gitignore
@ -2,3 +2,5 @@
 node_modules/
 dist/
 coverage/
+package-lock.json
+package.json
--- a/README.md
+++ b/README.md
@ -1,3 +1,42 @@
 # neo-scraper

 Scrape a booru-like post object from the DOM.
+
+## Supported sites
+
+| Engine | Sites | Features | Notes |
+|---|---|---|---|
+| animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
+| bluesky | bsky.app | content |  |
+| danbooru | danbooru.donmai.us, safebooru.donmai.us, sonohara.donmai.us, hijiribe.donmai.us | content, rating, resolution, tags, tag_category, source, notes |  |
+| e621 | e621.net, e926.net, e6ai.net | content, rating, resolution, tags, tag_category, source |  |
+| E-hentai | e-hentai.org, exhentai.org | content, resolution, extra_content, cookies | Rating is assumed to be unsafe. |
+| furaffinity | furaffinity.net | content, rating, resolution, tags, tag_category |  |
+| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com, mspabooru.com, hypnohub.net | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. |
+| inkbunny | inkbunny.net | content, rating, resolution, tags |  |
+| moebooru | yande.re, konachan.com, konachan.net, sakugabooru.com, img.genshiken-itb.org | content, rating, resolution, tags, tag_category, source, notes |  |
+| nozomi | nozomi.la | content, tags, tag_category | Image preview might not work due to hotlink protection. |
+| Philomena | derpibooru.org, trixiebooru.org, ponybooru.org, furbooru.org, ponerpics.org, manebooru.art, twibooru.org | content, rating, resolution, tags, tag_category, source |  |
+| pixiv | pixiv.net | content |  |
+| reddit | reddit.com, new.reddit.com, old.reddit.com | content |  |
+| rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. |
+| sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes |  |
+| shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. |
+| shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. |
+| twitter | twitter.com, mobile.twitter.com, x.com | content |  |
+| zerochan | zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
+| fallback |  | content | Tries to find the largest image or video on the current page. |
+
+## Development
+
+Build SzuruChrome with your local version of neo-scraper.
+
+```sh
+cd neo-scraper
+pnpm install
+pnpm build # or pnpm watch in another terminal
+pnpm link --global
+cd ../SzuruChrome
+pnpm link --global neo-scraper
+pnpm dev
+```
--- a/docs/TESTING.md
+++ b/docs/TESTING.md
@ -0,0 +1,24 @@
+# Testing notes
+
+## Run all tests in testfiles matching some pattern 
+
+```sh
+pnpm test pixiv
+```
+
+## Run one specific test ({{describe name}} {{it name}})
+
+```sh
+pnpm test pixiv -t "get pixiv post 96268450 \(multipage, expanded\) should find exactly four posts"
+```
+
+For some reason you need to escape the parentheses.
+
+## Debug puppeteer tests
+
+```sh
+pnpm test:debug # you can add more arguments if you want
+```
+
+This will disable headless mode.
+Use `await jestPuppeteer.debug();` in the test code to pause code execution.
--- a/jest-puppeteer.config.cjs
+++ b/jest-puppeteer.config.cjs
@ -1,6 +1,6 @@
 module.exports = {
  launch: {
-    headless: process.env.HEADLESS !== "false",
+    headless: process.env.HEADLESS !== "false" ? "new" : false,
    executablePath: process.env.CHROME_BIN || null,
    args: process.env.CI ? ["--disable-gpu", "--disable-dev-shm-usage", "--no-sandbox"] : [],
  },
--- a/jest.preset.cjs
+++ b/jest.preset.cjs
--- a/package.json
+++ b/package.json
@ -1,52 +1,77 @@
 {
  "name": "neo-scraper",
-  "version": "0.5.1",
+  "version": "0.11.1",
  "description": "Scrape a booru-like post object from the DOM.",
+  "type": "module",
  "main": "dist/index.js",
  "types": "dist/index.d.js",
  "scripts": {
-    "test": "jest",
-    "test:ci": "jest --colors --silent",
+    "test": "cross-env JEST_PUPPETEER_CONFIG=jest-puppeteer.config.cjs jest",
+    "test:ci": "cross-env JEST_PUPPETEER_CONFIG=jest-puppeteer.config.cjs jest --colors --silent",
+    "test:debug": "cross-env JEST_PUPPETEER_CONFIG=jest-puppeteer.config.cjs HEADLESS=false jest --testTimeout=100000000",
    "build": "tsc",
    "watch": "tsc -w",
-    "prepare": "npm run build"
+    "lint": "eslint src scripts",
+    "summary-table": "npm run build && node --experimental-specifier-resolution=node --loader ts-node/esm scripts/summary-table.ts"
  },
  "author": "neobooru",
  "license": "MIT",
+  "dependencies": {
+    "js-cookie": "^3.0.5",
+    "turndown": "^7.1.2"
+  },
  "devDependencies": {
-    "@types/expect-puppeteer": "^4.4.7",
-    "@types/jest": "^26.0.14",
-    "@types/jest-environment-puppeteer": "^4.4.1",
+    "@types/expect-puppeteer": "^5.0.5",
+    "@types/jest": "^29.5.7",
+    "@types/jest-environment-puppeteer": "^5.0.5",
+    "@types/js-cookie": "^3.0.5",
    "@types/jsdom": "^12.2.4",
-    "@types/puppeteer": "^5.4.4",
-    "@types/turndown": "^5.0.1",
-    "jest": "^26.6.3",
-    "jest-puppeteer": "^6.0.0",
+    "@types/turndown": "^5.0.3",
+    "@typescript-eslint/eslint-plugin": "^5.62.0",
+    "@typescript-eslint/parser": "^5.62.0",
+    "cross-env": "^7.0.3",
+    "eslint": "8.7.0",
+    "eslint-plugin-import": "^2.29.0",
+    "eslint-plugin-n": "^15.7.0",
+    "eslint-plugin-promise": "^6.1.1",
+    "jest": "^29.7.0",
+    "jest-puppeteer": "^10.1.4",
+    "jest-ts-webcompat-resolver": "^1.0.0",
    "jsdom": "^15.2.1",
-    "puppeteer": "^11.0.0",
-    "ts-jest": "^26.5.6",
-    "typescript": "^4.5.2"
+    "prettier": "2.5.1",
+    "puppeteer": "^23.10.1",
+    "ts-jest": "^29.2.5",
+    "ts-node": "^10.9.2",
+    "typescript": "^4.9.5"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
  },
  "jest": {
-    "preset": "./jest.preset.js",
+    "preset": "./jest.preset.cjs",
+    "resolver": "jest-ts-webcompat-resolver",
    "transform": {
-      "^.+\\.tsx?$": "ts-jest"
+      "^.+\\.tsx?$": [
+        "ts-jest",
+        {
+          "tsconfig": "tsconfig.test.json"
+        }
+      ]
    },
    "moduleFileExtensions": [
      "ts",
      "js"
    ],
+    "testTimeout": 30000,
    "testRegex": "^.+\\.test\\.ts$",
    "setupFilesAfterEnv": [
      "./src/test/jest.setup.ts"
-    ],
-    "globals": {
-      "ts-jest": {
-        "tsconfig": "tsconfig.test.json"
-      }
-    }
+    ]
  },
-  "dependencies": {
-    "turndown": "^7.1.1"
+  "overrides": {
+	  "@types/node": "18.19.115"
  }
 }
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
--- a/scripts/summary-table.ts
+++ b/scripts/summary-table.ts
@ -0,0 +1,15 @@
+import NeoScraper from "../src/NeoScraper.js";
+import FallbackEngine from "../src/engines/Fallback.js";
+
+const engines = [...new NeoScraper().engines, new FallbackEngine()];
+const txt = ["| Engine | Sites | Features | Notes |", "|---|---|---|---|"];
+
+for (const engine of engines) {
+  txt.push(
+    `| ${engine.name} | ${engine.supportedHosts.join(", ")} | ${engine.features.join(", ")} | ${engine.notes.join(
+      " "
+    )} |`
+  );
+}
+
+console.log(txt.join("\n"));
--- a/src/NeoScraper.test.ts
+++ b/src/NeoScraper.test.ts
@ -1,12 +1,12 @@
-import NeoScraper from "./NeoScraper";
+import NeoScraper from "./NeoScraper.js";

 test("create scraper with default engines", async () => {
-  const scraper = new NeoScraper();
+  new NeoScraper();
 });

 test("all default scrapers have unique names", () => {
  const scraper = new NeoScraper();
-  let alreadySeen: string[] = [];
+  const alreadySeen: string[] = [];

  for (const engine of scraper.engines) {
    if (alreadySeen.indexOf(engine.name) != -1) {
--- a/src/NeoScraper.ts
+++ b/src/NeoScraper.ts
@ -1,20 +1,25 @@
-import { ScrapeEngine, ScrapeResults } from "./ScrapeEngine";
-import * as engines from "./engines";
+import { ScrapeEngine, ScrapeResults } from "./ScrapeEngine.js";
+import * as engines from "./engines/index.js";

 export default class NeoScraper {
  engines: ScrapeEngine[] = [];
  fallbackEngine: ScrapeEngine | null = null;

-  constructor(useDefaultEngines: boolean = true) {
+  constructor(useDefaultEngines = true) {
    if (useDefaultEngines) {
      this.engines.push(
        new engines.AnimePictures(),
+        new engines.Bluesky(),
        new engines.Danbooru(),
        new engines.e621(),
+        new engines.Ehentai(),
        new engines.FurAffinity(),
        new engines.Gelbooru(),
        new engines.Inkbunny(),
+		new engines.Mastodon(),
        new engines.Moebooru(),
+        new engines.Nozomi(),
+        new engines.Philomena(),
        new engines.Pixiv(),
        new engines.Reddit(),
        new engines.Rule34us(),
@ -28,12 +33,7 @@ export default class NeoScraper {
    }
  }

-  scrapeDocument(
-    document: Document,
-    engines: string[] | null = null,
-    allowFallbackEngine: boolean = false,
-    ignoreCanImport: boolean = false
-  ): ScrapeResults {
+  scrapeDocument(document: Document, allowFallbackEngine = false, ignoreCanImport = false): ScrapeResults {
    const res = new ScrapeResults();

    for (const engine of this.engines) {
--- a/src/ScrapeEngine.ts
+++ b/src/ScrapeEngine.ts
@ -1,17 +1,24 @@
-import { TagCategory, SafetyRating } from "./BooruTypes";
+import { TagCategory, SafetyRating } from "./BooruTypes.js";

+export type Cookies = { [key: string]: string };
 export type ContentType = "image" | "video";
+export type UploadMode = "content" | "url";

 export class ScrapedPost {
-  contentUrl: string = "";
-  pageUrl: string = "";
+  name: string | undefined = undefined;
+  contentUrl = "";
+  pageUrl = "";
  contentType: ContentType = "image";
+  contentSize: number | undefined;
  resolution: [number, number] | undefined;
  rating: SafetyRating = "safe";
  tags: ScrapedTag[] = [];
  notes: ScrapedNote[] = [];
-  source: string | undefined;
+  sources: string[] = [];
  referrer: string | undefined;
+  extraContentUrl: string | undefined;
+  cookies: Cookies | undefined;
+  uploadMode: UploadMode = "url";
 }

 export class ScrapedTag {
@ -30,7 +37,7 @@ export class ScrapedNote {

 export class ScrapeResult {
  engine: string;
-  description: string = "";
+  description = "";
  posts: ScrapedPost[] = [];

  constructor(engine: string) {
@ -50,9 +57,9 @@ export class ScrapeResults {
  results: ScrapeResult[] = [];

  get posts(): ScrapedPost[] {
-    var posts: ScrapedPost[] = [];
+    let posts: ScrapedPost[] = [];

-    for (var res of this.results) {
+    for (const res of this.results) {
      posts = posts.concat(res.posts);
    }

@ -60,15 +67,37 @@ export class ScrapeResults {
  }
 }

+export type ScrapeEngineFeature =
+  | "content"
+  | "rating"
+  | "resolution"
+  | "tags"
+  | "tag_category"
+  | "source"
+  | "notes"
+  | "extra_content"
+  | "cookies";
+
 export interface ScrapeEngine {
  name: string;
+  features: ScrapeEngineFeature[];
+  notes: string[];
+  supportedHosts: string[];
+
  canImport(url: Location): boolean;
  scrapeDocument(document: Document): ScrapeResult;
 }

 export abstract class ScrapeEngineBase implements ScrapeEngine {
  abstract name: string;
-  abstract canImport(url: Location): boolean;
+  abstract features: ScrapeEngineFeature[];
+  abstract notes: string[];
+  abstract supportedHosts: string[];
+
+  canImport(url: Location): boolean {
+    return this.supportedHosts.indexOf(url.host.replace("www.", "")) != -1;
+  }
+
  abstract scrapeDocument(document: Document): ScrapeResult;

  protected log(str: string) {
--- a/src/Utility.test.ts
+++ b/src/Utility.test.ts
@ -1,4 +1,13 @@
-import { guessContentType } from "./Utility";
+import { getUrlExtension, guessContentType } from "./Utility.js";
+
+test("getUrlExtension", () => {
+  expect(getUrlExtension("image.png")).toBe("png");
+  expect(getUrlExtension("image.jpeg?34")).toBe("jpeg");
+  expect(getUrlExtension("image.php?q=test&p=123%204")).toBe("php");
+
+  // I would probably rather have this return an empty string, but maybe some code relies on this functionality?
+  expect(getUrlExtension("http://site/index")).toBe("http://site/index");
+});

 test("guessContentType", () => {
  expect(guessContentType("video.mp4")).toBe("video");
--- a/src/Utility.ts
+++ b/src/Utility.ts
@ -1,5 +1,5 @@
 import TurndownService from "turndown";
-import { ContentType, ScrapedNote, ScrapedPost } from "./ScrapeEngine";
+import { ContentType, ScrapedNote, ScrapedPost } from "./ScrapeEngine.js";

 // https://github.com/sindresorhus/video-extensions/blob/main/video-extensions.json
 export const videoExtensions = [
@ -39,10 +39,18 @@ export const videoExtensions = [
 ];

 export function getUrlExtension(url: string) {
+  // https://stackoverflow.com/a/47767860
  return url.split(/[#?]/)[0].split(".").pop()!.trim();
 }

-export function guessContentType(url: string): ContentType {
+/**
+ * Returns "image" when the guessed type is NOT a video.
+ * This means that it will also return "image" when the content type
+ * could not be guessed!
+ * @param url
+ * @returns "image" | "video"
+ */
+export function guessContentType(url: string | undefined): ContentType {
  if (url != undefined) {
    const ext = getUrlExtension(url);
    if (videoExtensions.indexOf(ext) != -1) return "video";
@ -101,7 +109,7 @@ export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement
      return undefined;
    }
    const num = parseInt(value);
-    if (num == NaN) {
+    if (isNaN(num)) {
      logFail(`parseInt on '${value}' returned NaN.`);
    }
    return num;
@ -112,6 +120,7 @@ export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement
    return undefined;
  }

+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
  const data = (<any>el).dataset as DOMStringMap;

  if (!data) {
@ -124,10 +133,10 @@ export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement
    text = htmlNoteToMarkdownNote(text);
  }

-  let x = getInt(data, "x");
-  let y = getInt(data, "y");
-  let w = getInt(data, "width");
-  let h = getInt(data, "height");
+  const x = getInt(data, "x");
+  const y = getInt(data, "y");
+  const w = getInt(data, "width");
+  const h = getInt(data, "height");

  if (!text || !x || !y || !w || !h) {
    logFail("A required data field could not be parsed.");
@ -151,7 +160,7 @@ export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement
 * @returns
 */
 export function createNotesFromMoebooruBoxes(document: Document, boxSize: [number, number]) {
-  let notes: ScrapedNote[] = [];
+  const notes: ScrapedNote[] = [];
  const noteBoxes = Array.from(document.querySelectorAll(".note-box")).map((x) => x as HTMLLIElement);
  const noteBodies = Array.from(document.querySelectorAll(".note-body")).map((x) => x as HTMLLIElement);

@ -179,3 +188,21 @@ export function createNotesFromMoebooruBoxes(document: Document, boxSize: [numbe
  }
  return notes;
 }
+
+export function parseResolutionString(str: string | undefined | null): [number, number] | undefined {
+  if (!str) return undefined;
+
+  // Example string: 1600x2200
+  const res = str
+    .split("x") // Split on 'x' character
+    .map((v) => v.replace(/\D/g, "")) // Remove all non-digits
+    .map((v) => parseInt(v)) // Parse ints
+    .filter(Number); // Filter NaN. Also removes the number 0, but that's fine because the resolution can't be 0.
+
+  if (res.length == 2) {
+    // Kinda hacky, but this is currently the correct way to convert an array to a tuple.
+    return [res[0], res[1]];
+  } else {
+    console.log(`[parseResolutionString] Couldn't parse resolution from '${str}'.`);
+  }
+}
--- a/src/engines/AnimePictures.test.ts
+++ b/src/engines/AnimePictures.test.ts
@ -1,11 +1,19 @@
-import { scrapeUrl } from "../test/engine";
+import { scrapeEvalUrl } from "../test/engine.js";

 test("get anime-pictures.net post 729833", async () => {
-  const pageUrl = "https://anime-pictures.net/pictures/view_post/729833?lang=en";
-  const res = await scrapeUrl(pageUrl);
+  const pageUrl = "https://anime-pictures.net/posts/729833?lang=en";
+  const res = await scrapeEvalUrl(pageUrl, "body");
  expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("arknights", "copyright");
+  expect(res.posts[0]).toHaveTag("ch'en_(arknights)", "character");
+  expect(res.posts[0]).toHaveTag("ttk_(kirinottk)", "artist");
+  expect(res.posts[0]).toHaveTag("single");
+  expect(res.posts[0]).toHaveTag("holding");
+  expect(res.posts[0]).toHaveTag("gloves");
+  expect(res.posts[0]).toHaveTag("jacket");
+  expect(res.posts[0]).toHaveResolution([4000, 6865]);
  expect(res.posts[0].pageUrl).toBe(pageUrl);
  expect(res.posts[0].contentUrl).toBe(
-    "https://anime-pictures.net/pictures/download_image/729833-4000x6865-arknights-ch%26%2339%3Ben+%28arknights%29-ttk+%28kirinottk%29-single-long+hair-tall+image.jpeg"
+    "https://api.anime-pictures.net/pictures/download_image/729833-4000x6865-arknights-ch'en+(arknights)-ttk+(kirinottk)-single-long+hair-tall+image.jpeg"
  );
 });
--- a/src/engines/AnimePictures.ts
+++ b/src/engines/AnimePictures.ts
@ -1,28 +1,23 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
-import { TagCategory } from "../BooruTypes";
-import { guessContentType } from "../Utility";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
+import { TagCategory } from "../BooruTypes.js";
+import { guessContentType, parseResolutionString } from "../Utility.js";

-export default class AnimePictures implements ScrapeEngine {
+export default class AnimePictures extends ScrapeEngineBase {
  name = "animepictures";
-
-  canImport(url: Location): boolean {
-    return url.host == "anime-pictures.net";
-  }
+  features: ScrapeEngineFeature[] = ["content", "resolution", "tags", "tag_category"];
+  notes = ["Rating is assumed to be safe."];
+  supportedHosts = ["anime-pictures.net"];

  scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
-    let post = new ScrapedPost();
+    const result = new ScrapeResult(this.name);
+    const post = new ScrapedPost();
    post.pageUrl = document.location.href;
-    post.source = document.location.href;

    // Set image url
-    const originalImageElements = Array.from(document.getElementsByClassName("download_icon"));
+    const originalImageElements = Array.from(document.getElementsByClassName("icon-download"));

    if (originalImageElements.length > 0) {
      post.contentUrl = (originalImageElements[0] as HTMLAnchorElement).href;
-    } else {
-      // No point in continuing when we don't have an image.
-      return result;
    }

    // Set content type
@ -31,10 +26,17 @@ export default class AnimePictures implements ScrapeEngine {
    // Set rating
    post.rating = "safe"; // Probably

+    // Set resolution
+    const postContentLinkEls = Array.from(document.querySelectorAll<HTMLAnchorElement>(".post_content > .body a"));
+    const resEl = postContentLinkEls.find((x) => x.href.indexOf("res_x") != -1);
+    if (resEl) {
+      post.resolution = parseResolutionString(resEl.innerText);
+    }
+
    // Set tags
-    const tagElements = Array.from(document.querySelectorAll("ul.tags > li > a")).map((x) => x as HTMLAnchorElement);
+    const tagElements = Array.from(document.querySelectorAll<HTMLAnchorElement>("ul.tags > li > a"));
    for (const tagElement of tagElements) {
-      let tagName = tagElement.innerText;
+      const tagName = tagElement.innerText;

      let category: TagCategory | undefined;
      for (const className of tagElement.classList) {
@ -52,7 +54,7 @@ export default class AnimePictures implements ScrapeEngine {
      }

      if (tagName) {
-        let tag = new ScrapedTag(tagName, category);
+        const tag = new ScrapedTag(tagName, category);
        post.tags.push(tag);
      }
    }
--- a/src/engines/Bluesky.ts
+++ b/src/engines/Bluesky.ts
@ -0,0 +1,126 @@
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapeEngineFeature } from "../ScrapeEngine.js";
+
+function getFullSizeImg(link: string) {
+  return link.replace("/feed_thumbnail/", "/feed_fullsize/");
+}
+
+/**
+ * Bluesky serves video via HLS in the player, but the underlying blob is a
+ * single .mp4 retrievable from the AT Protocol sync endpoint:
+ *
+ *   https://bsky.social/xrpc/com.atproto.sync.getBlob?did=<DID>&cid=<CID>
+ *
+ * The DID and CID are URL-encoded in the <video> poster's path:
+ *
+ *   https://video.bsky.app/watch/<DID>/<CID>/thumbnail.jpg
+ *
+ * We reconstruct the blob URL and mark the post as uploadMode="content"
+ * so SzuruChrome fetches it in the browser instead of asking Szurubooru's
+ * server to download it (which would fail on the blob endpoint).
+ */
+export default class Bluesky extends ScrapeEngineBase {
+  name = "bluesky";
+  features: ScrapeEngineFeature[] = ["content"];
+  notes = [];
+  supportedHosts = ["bsky.app"];
+
+  scrapeDocument(document: Document): ScrapeResult {
+    const result = new ScrapeResult(this.name);
+	
+	type Candidate = { el: Element; post: ScrapedPost };
+	const candidates: Candidate[] = [];
+	
+	// Images
+    const imgEls = Array.from<HTMLImageElement>(document.querySelectorAll("div > img"));
+    let els = imgEls.filter((x) => x.src.indexOf("feed_fullsize") != -1);
+    // Only add the 'normal' feed elements if no post is in fullscreen.
+    if (els.length == 0) {
+      this.log("Post is not fullscreen.");
+      els = imgEls.filter((x) => x.src.indexOf("feed") != -1);
+    }
+    for (const el of els) {
+	  if(!isVisible(el)) continue;
+	  
+      const post = new ScrapedPost();
+      post.pageUrl = document.location.href;
+      post.contentUrl = getFullSizeImg(el.src);
+      candidates.push({ el, post });
+    }
+
+    // Videos
+    const videoEls = Array.from<HTMLVideoElement>(
+      document.querySelectorAll('video[poster^="https://video.bsky.app/watch"]')
+    );
+    for (const v of videoEls) {
+	  if(!isVisible(v)) continue;
+	  
+      const poster = v.getAttribute("poster");
+      if (!poster) continue;
+
+      const blobUrl = this.buildBlobUrl(poster);
+      if (!blobUrl) {
+        this.log(`Could not parse poster URL: ${poster}`);
+        continue;
+      }
+      const post = new ScrapedPost();
+      post.pageUrl = document.location.href;
+      post.contentUrl = blobUrl;
+      post.contentType = "video";
+      candidates.push({ el: v, post });
+    }
+	
+	candidates.sort((a, b) => {
+      const pos = a.el.compareDocumentPosition(b.el);
+      if (pos & Node.DOCUMENT_POSITION_FOLLOWING) return -1;
+      if (pos & Node.DOCUMENT_POSITION_PRECEDING) return 1;
+      return 0;
+    });
+	
+	for(const c of candidates) result.tryAddPost(c.post);
+
+    return result;
+  }
+
+  /**
+   * Poster format:
+   *   https://video.bsky.app/watch/<DID-urlencoded>/<CID>/thumbnail.jpg
+   * Returns the AT Protocol blob URL, or undefined if the poster doesn't
+   * match the expected shape.
+   */
+  private buildBlobUrl(poster: string): string | undefined {
+    try {
+      const u = new URL(poster);
+      const parts = u.pathname.split("/").filter((p) => p.length > 0);
+      // Expected: ["watch", "<DID>", "<CID>", "thumbnail.jpg"]
+      if (parts.length < 3 || parts[0] !== "watch") return undefined;
+
+      const did = decodeURIComponent(parts[1]);   // did:plc:...
+      const cid = parts[2];                        // bafkrei...
+
+      return (
+        "https://bsky.social/xrpc/com.atproto.sync.getBlob" +
+        `?did=${encodeURIComponent(did)}&cid=${encodeURIComponent(cid)}`
+      );
+    } catch {
+      return undefined;
+    }
+  }
+}
+
+/**
+ * A lightweight visibility check. We use this rather than IntersectionObserver
+ * because scrapeDocument is synchronous and must return immediately.
+ */
+function isVisible(el: Element): boolean {
+  const rect = el.getBoundingClientRect();
+  if (rect.width === 0 || rect.height === 0) return false;
+  if (rect.bottom <= 0 || rect.top >= window.innerHeight) return false;
+  if (rect.right <= 0 || rect.left >= window.innerWidth) return false;
+
+  const style = window.getComputedStyle(el);
+  if (style.display === "none") return false;
+  if (style.visibility === "hidden") return false;
+  if (style.opacity === "0") return false;
+
+  return true;
+}
--- a/src/engines/Common.ts
+++ b/src/engines/Common.ts
@ -1,3 +1,3 @@
-import { TagCategory } from "../BooruTypes";
+import { TagCategory } from "../BooruTypes.js";

 export type CategoryMap = Record<string, TagCategory>;
--- a/src/engines/Danbooru.test.ts
+++ b/src/engines/Danbooru.test.ts
@ -1,24 +1,31 @@
-import { scrapeUrl } from "../test/engine";
+import { scrapeEvalUrl } from "../test/engine.js";

 test("get danbooru.donmai.us post 3555872", async () => {
  const pageUrl = "https://danbooru.donmai.us/posts/3555872";
-  const res = await scrapeUrl(pageUrl);
+  const res = await scrapeEvalUrl(pageUrl, "#tag-list");
  expect(res.posts.length).toBe(1);
-  expect(res.posts[0]).toHaveTag("bangs");
+  expect(res.posts[0]).toHaveTag("omachi_(slabco)", "artist");
+  expect(res.posts[0]).toHaveTag("dr_pepper");
+  expect(res.posts[0]).toHaveTag("flint_(girls_und_panzer)", "character");
  expect(res.posts[0]).toHaveTag("shirt");
  expect(res.posts[0]).toHaveTag("skirt");
  expect(res.posts[0]).toHaveTag("t-shirt");
  expect(res.posts[0]).toHaveTag("long_hair");
+  expect(res.posts[0]).toHaveResolution([1142, 1600]);
+  expect(res.posts[0]).toHaveSource("https://twitter.com/omatiosake/status/1124306601584672768");
  expect(res.posts[0].pageUrl).toBe(pageUrl);
  expect(res.posts[0].contentUrl).toBe(
-    "https://cdn.donmai.us/original/90/33/__flint_girls_und_panzer_and_1_more_drawn_by_omachi_slabco__90332299216db81b8793084e4cb7c15d.jpg"
+    "https://cdn.donmai.us/original/90/33/__flint_girls_und_panzer_drawn_by_omachi_slabco__90332299216db81b8793084e4cb7c15d.jpg"
  );
 });

 test("get danbooru.donmai.us post 4924490", async () => {
  const pageUrl = "https://danbooru.donmai.us/posts/4924490";
-  const res = await scrapeUrl(pageUrl);
+  const res = await scrapeEvalUrl(pageUrl, "#tag-list");
  expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("yuureidoushi_(yuurei6214)", "artist");
+  expect(res.posts[0]).toHaveTag("comic");
+  expect(res.posts[0]).toHaveResolution([1000, 2062]);
  expect(res.posts[0].pageUrl).toBe(pageUrl);
  expect(res.posts[0].contentUrl).toBe(
    "https://cdn.donmai.us/original/c5/fb/__yuuki_and_tamaki_princess_connect_drawn_by_yuureidoushi_yuurei6214__c5fbc62758d0f1d8c07aac524282434e.png"
@ -26,3 +33,16 @@ test("get danbooru.donmai.us post 4924490", async () => {
  expect(res.posts[0].notes).toHaveLength(17);
  expect(res.posts[0]).toHaveNote("Nya~nya ♪");
 });
+
+test("get danbooru.donmai.us post 4530532", async () => {
+  const pageUrl = "https://danbooru.donmai.us/posts/4530532";
+  const res = await scrapeEvalUrl(pageUrl, "#tag-list");
+  expect(res.posts.length).toBe(1);
+  expect(res.posts[0].rating).toBe("sketchy");
+  expect(res.posts[0]).toHaveTag("chainsaw_man", "copyright");
+  expect(res.posts[0]).toHaveResolution([1105, 1803]);
+  expect(res.posts[0].pageUrl).toBe(pageUrl);
+  expect(res.posts[0].contentUrl).toBe(
+    "https://cdn.donmai.us/original/fc/e3/__makima_power_and_reze_chainsaw_man_drawn_by_hews__fce35c7a8515a00c566b288a591fd5f9.png"
+  );
+});
--- a/src/engines/Danbooru.ts
+++ b/src/engines/Danbooru.ts
@ -1,17 +1,23 @@
-import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapedNote } from "../ScrapeEngine";
-import { TagCategory } from "../BooruTypes";
-import { createNoteFromDanbooruArticle, guessContentType } from "../Utility";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
+import { TagCategory } from "../BooruTypes.js";
+import { createNoteFromDanbooruArticle, guessContentType } from "../Utility.js";

 export default class Danbooru extends ScrapeEngineBase {
  name = "danbooru";
-
-  canImport(url: Location): boolean {
-    return url.host == "danbooru.donmai.us" || url.host == "safebooru.donmai.us";
-  }
+  features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
+  notes = [];
+  supportedHosts = [
+    "danbooru.donmai.us", 
+	"safebooru.donmai.us", 
+	"sonohara.donmai.us", 
+	"hijiribe.donmai.us",
+	"booru.allthefallen.moe"
+  ];

  scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
-    let post = new ScrapedPost();
+    const result = new ScrapeResult(this.name);
+    const post = new ScrapedPost();
+    post.uploadMode = "content";
    post.pageUrl = document.location.href;

    // Set image url
@ -56,6 +62,7 @@ export default class Danbooru extends ScrapeEngineBase {
          case "safe":
            post.rating = "safe";
            break;
+          case "sensitive":
          case "questionable":
            post.rating = "sketchy";
            break;
@ -76,23 +83,25 @@ export default class Danbooru extends ScrapeEngineBase {
      }

      let category: TagCategory | undefined;
-      switch (tagElement.className) {
-        case "tag-type-3":
-          category = "copyright";
-          break;
-        case "tag-type-4":
-          category = "character";
-          break;
-        case "tag-type-1":
-          category = "artist";
-          break;
-        case "tag-type-5":
-          category = "meta";
-          break;
+      for (const className of tagElement.classList) {
+        switch (className) {
+          case "tag-type-3":
+            category = "copyright";
+            break;
+          case "tag-type-4":
+            category = "character";
+            break;
+          case "tag-type-1":
+            category = "artist";
+            break;
+          case "tag-type-5":
+            category = "meta";
+            break;
+        }
      }

      if (tagName) {
-        let tag = new ScrapedTag(tagName, category);
+        const tag = new ScrapedTag(tagName, category);
        post.tags.push(tag);
      }
    }
@ -102,7 +111,7 @@ export default class Danbooru extends ScrapeEngineBase {
    if (sourceEl && sourceEl.children.length > 0) {
      const sourceLink = sourceEl.children[0] as HTMLAnchorElement;
      if (sourceLink) {
-        post.source = sourceLink.href;
+        post.sources = [sourceLink.href];
      }
    }

--- a/src/engines/Ehentai.ts
+++ b/src/engines/Ehentai.ts
@ -0,0 +1,49 @@
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapeEngineFeature } from "../ScrapeEngine.js";
+import { parseResolutionString } from "../Utility.js";
+import Cookies from "js-cookie";
+
+export default class Ehentai extends ScrapeEngineBase {
+  name = "E-hentai";
+  features: ScrapeEngineFeature[] = ["content", "resolution", "extra_content", "cookies"];
+  notes = ["Rating is assumed to be unsafe."];
+  supportedHosts = ["e-hentai.org", "exhentai.org"];
+
+  scrapeDocument(document: Document): ScrapeResult {
+    const result = new ScrapeResult(this.name);
+    const post = new ScrapedPost();
+    post.pageUrl = document.location.href;
+
+    // Set image url
+    const imgUrl = document.getElementById("img") as HTMLImageElement;
+    if (imgUrl) {
+      post.contentUrl = imgUrl.src;
+    }
+
+    const originalSourceUrl = document.querySelector("a[href*='fullimg']") as HTMLAnchorElement;
+    if (originalSourceUrl) {
+      // Set "extra" image url. This URL only gives a good response if the cookies are included.
+      // This might also only work on certain accounts.
+      post.extraContentUrl = originalSourceUrl.href;
+
+      // Set resolution
+      const m = originalSourceUrl.innerText.match(/\d+ x \d+/);
+      if (m?.length == 1) {
+        post.resolution = parseResolutionString(m[0]);
+      }
+    }
+
+    // Set rating hardcoded unsafe
+    post.rating = "unsafe";
+
+    // No tags available on this page.
+    // On the parent page you could probably get some tags, though these don't say anything about the individual image.
+    // The tags only say something about the gallery as a whole.
+
+    // Set cookies
+    post.cookies = Cookies.get();
+
+    result.tryAddPost(post);
+
+    return result;
+  }
+}
--- a/src/engines/Fallback.ts
+++ b/src/engines/Fallback.ts
@ -1,21 +1,36 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ContentType } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ContentType, ScrapeEngineFeature } from "../ScrapeEngine.js";
+import { guessContentType } from "../Utility.js";

 class MediaElement {
  constructor(
    public readonly contentType: ContentType,
    public readonly contentUrl: string,
    public readonly size: number
-  ) {}
+  ) { }
 }

-export default class Fallback implements ScrapeEngine {
+export default class Fallback extends ScrapeEngineBase {
  name = "fallback";
-
-  canImport(location: Location): boolean {
-    return false;
-  }
+  features: ScrapeEngineFeature[] = ["content"];
+  notes = ["Tries to find the largest image or video on the current page."];
+  supportedHosts = [];

  scrapeDocument(document: Document): ScrapeResult {
+    const result = new ScrapeResult(this.name);
+
+    // Check if the current page is a link to a video file.
+    // If it is then use the document.location.href as media source and immediately return.
+    // This is needed because firefox does not add a <source> to a <video> element
+    // when going to a direct link. It looks like chromium-based browsers do not have this issue.
+    if (guessContentType(document.location.href) == "video") {
+      const post = new ScrapedPost();
+      post.pageUrl = document.location.href;
+      post.contentUrl = document.location.href;
+      post.contentType = "video";
+      result.tryAddPost(post);
+      return result;
+    }
+
    // Get all img elements.
    const largestImgElement = Array.from(document.getElementsByTagName("img")).map(
      (x) => new MediaElement("image", x.src, x.width * x.height)
@ -33,14 +48,18 @@ export default class Fallback implements ScrapeEngine {
      a?.size > b?.size ? a : b
    );

-    let result = new ScrapeResult(this.name);
-
    if (largestMediaElement) {
-      let post = new ScrapedPost();
+      const post = new ScrapedPost();
+      post.name = "Upload as URL";
      post.pageUrl = document.location.href;
      post.contentUrl = largestMediaElement.contentUrl;
      post.contentType = largestMediaElement.contentType;
-      result.posts.push(post);
+      result.tryAddPost(post);
+
+      const clone = Object.assign({}, post);
+      clone.name = "Upload as content";
+      clone.uploadMode = "content";
+      result.tryAddPost(clone);
    }

    return result;
--- a/src/engines/FurAffinity.test.ts
+++ b/src/engines/FurAffinity.test.ts
@ -0,0 +1,16 @@
+import { scrapeUrl } from "../test/engine.js";
+
+test("get furaffinity.net post 23281942", async () => {
+  const pageUrl = "https://www.furaffinity.net/view/23281942/";
+  const res = await scrapeUrl(pageUrl);
+  expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("zac");
+  expect(res.posts[0]).toHaveTag("riven");
+  expect(res.posts[0]).toHaveTag("pool");
+  expect(res.posts[0]).toHaveTag("nielsdejong", "artist");
+  expect(res.posts[0]).toHaveResolution([1280, 1279]);
+  expect(res.posts[0].pageUrl).toBe(pageUrl);
+  expect(res.posts[0].contentUrl).toBe(
+    "https://d.furaffinity.net/art/nielsdejong/1492781239/1492781239.nielsdejong_pool_party_fun_with_riven_and_zac_by_polkinart.jpg"
+  );
+});
--- a/src/engines/FurAffinity.ts
+++ b/src/engines/FurAffinity.ts
@ -1,17 +1,15 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
-import { guessContentType } from "../Utility";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
+import { guessContentType, parseResolutionString } from "../Utility.js";

-export default class e621 implements ScrapeEngine {
+export default class FurAffinity extends ScrapeEngineBase {
  name = "furaffinity";
-
-  canImport(url: Location): boolean {
-    return url.host == "www.furaffinity.net";
-  }
+  features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category"];
+  notes = [];
+  supportedHosts = ["furaffinity.net"];

  scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
-
-    let post = new ScrapedPost();
+    const result = new ScrapeResult(this.name);
+    const post = new ScrapedPost();
    post.pageUrl = document.location.href;

    // Set image url
@ -22,6 +20,14 @@ export default class e621 implements ScrapeEngine {
    // Set content type
    post.contentType = guessContentType(post.contentUrl);

+    // Set resolution
+    const infoTexts = Array.from(document.querySelector("section.info.text")?.children ?? []);
+    const sizeDiv = infoTexts.find((x) => x.querySelector("strong")?.innerText == "Size");
+    if (sizeDiv) {
+      const resolution = parseResolutionString(sizeDiv.querySelector("span")?.innerText);
+      post.resolution = resolution;
+    }
+
    // Set rating
    const ratingEls = Array.from(document.querySelectorAll("div.rating > span.rating-box")).map(
      (x) => x as HTMLSpanElement
@ -41,12 +47,15 @@ export default class e621 implements ScrapeEngine {
      }

    // Set tags
-    post.tags = Array.from(document.querySelectorAll("section.tags-row > span.tags"))
+    post.tags = Array.from(document.querySelectorAll("section.tags-row > span.tags a"))
      .map((x) => (x as HTMLSpanElement).innerText)
      .map((x) => new ScrapedTag(x));

-    // Set source
-    post.source = document.location.href;
+    // Set user who submitted the art as the artist
+    const submissionUser = document.querySelector<HTMLElement>(".submission-id-sub-container a")?.innerText;
+    if (submissionUser) {
+      post.tags.push(new ScrapedTag(submissionUser, "artist"));
+    }

    result.tryAddPost(post);

--- a/src/engines/Gelbooru.test.ts
+++ b/src/engines/Gelbooru.test.ts
@ -1,11 +1,20 @@
-import { scrapeUrl } from "../test/engine";
+import { scrapeUrl } from "../test/engine.js";

 test("get safebooru.org post 2871532", async () => {
  const pageUrl = "https://safebooru.org/index.php?page=post&s=view&id=2871532";
  const res = await scrapeUrl(pageUrl);
  expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("omachi_(slabco)", "artist");
+  expect(res.posts[0]).toHaveTag("flint_(girls_und_panzer)", "character");
+  expect(res.posts[0]).toHaveTag("bangs");
+  expect(res.posts[0]).toHaveTag("shirt");
+  expect(res.posts[0]).toHaveTag("skirt");
+  expect(res.posts[0]).toHaveTag("t-shirt");
+  expect(res.posts[0]).toHaveTag("long_hair");
+  expect(res.posts[0]).toHaveResolution([1142, 1600]);
  expect(res.posts[0].pageUrl).toBe(pageUrl);
-  expect(res.posts[0].contentUrl).toBe(
+  expect(res.posts[0].rating).toBe("safe");
+  expect(res.posts[0].contentUrl).toContain(
    "https://safebooru.org//images/2757/5f19c7744db918c3d4c4c6035bc65dfa9e18866f.jpg"
  );
 });
@ -14,15 +23,19 @@ test("get gelbooru.com post 4925055", async () => {
  const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=4925055&tags=kimetsu_no_yaiba";
  const res = await scrapeUrl(pageUrl);
  expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("maosen", "artist");
+  expect(res.posts[0]).toHaveTag("butterfly");
+  expect(res.posts[0]).toHaveResolution([720, 1103]);
+  expect(res.posts[0]).toHaveSource("https://www.pixiv.net/artworks/76853958");
  expect(res.posts[0].pageUrl).toBe(pageUrl);
-  expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
+  expect(res.posts[0].rating).toBe("sketchy");
+  expect(res.posts[0].contentUrl).toContain("gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
 });

 test("get gelbooru.com post 6225495 with notes", async () => {
  const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=6225495";
  const res = await scrapeUrl(pageUrl);
  expect(res.posts.length).toBe(1);
-  expect(res.posts[0]).toHaveTag("bangs");
  expect(res.posts[0]).toHaveTag("hair_up");
  expect(res.posts[0]).toHaveTag("hair_up");
  expect(res.posts[0]).toHaveTag("teeth");
@ -33,14 +46,39 @@ test("get gelbooru.com post 6225495 with notes", async () => {
  expect(res.posts[0]).toHaveNote("I'm fine, more or less.");
  expect(res.posts[0]).toHaveNote("But no need to worry!");
  expect(res.posts[0]).toHaveNote("You're always the best just by being yourself-!!!");
+  expect(res.posts[0]).toHaveResolution([1600, 2200]);
+  expect(res.posts[0]).toHaveSource("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=88721816");
  expect(res.posts[0].pageUrl).toBe(pageUrl);
-  expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/af/1d/af1dd661fab6714dffc29cb38b6b2269.png");
+  expect(res.posts[0].rating).toBe("sketchy");
+  expect(res.posts[0].contentUrl).toContain("gelbooru.com/images/af/1d/af1dd661fab6714dffc29cb38b6b2269.png");
 });

 test("get rule34.xxx post 2740994", async () => {
  const pageUrl = "https://rule34.xxx/index.php?page=post&s=view&id=2740994";
  const res = await scrapeUrl(pageUrl);
+  expect(res.posts[0]).toHaveTag("fire_emblem", "copyright");
+  expect(res.posts[0]).toHaveTag("nintendo", "copyright");
+  expect(res.posts[0]).toHaveTag("lucina_(fire_emblem)", "character");
+  expect(res.posts[0]).toHaveTag("blue_hair");
+  expect(res.posts[0]).toHaveTag("solo");
+  expect(res.posts[0]).toHaveResolution([600, 800]);
  expect(res.posts.length).toBe(1);
  expect(res.posts[0].pageUrl).toBe(pageUrl);
+  expect(res.posts[0].rating).toBe("safe");
  expect(res.posts[0].contentUrl).toContain(".rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg");
 });
+
+test("get xbooru.com post 834083", async () => {
+  // Don't worry this isn't a cursed image.
+  const pageUrl = "https://xbooru.com/index.php?page=post&s=view&id=834083";
+  const res = await scrapeUrl(pageUrl);
+  expect(res.posts[0]).toHaveTag("friendship_is_magic", "copyright");
+  expect(res.posts[0]).toHaveTag("nurse_redheart", "character");
+  expect(res.posts[0]).toHaveTag("nurse");
+  expect(res.posts[0]).toHaveTag("standing");
+  expect(res.posts[0]).toHaveResolution([1024, 1878]);
+  expect(res.posts.length).toBe(1);
+  expect(res.posts[0].pageUrl).toBe(pageUrl);
+  expect(res.posts[0].rating).toBe("safe");
+  expect(res.posts[0].contentUrl).toContain("https://img.xbooru.com//images/584/040dc1625c094a7d5d7b0918aa0d8324.png");
+});
--- a/src/engines/Gelbooru.ts
+++ b/src/engines/Gelbooru.ts
@ -1,7 +1,7 @@
-import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
-import { TagCategory } from "../BooruTypes";
-import { CategoryMap } from "./Common";
-import { createNoteFromDanbooruArticle, guessContentType } from "../Utility";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
+import { TagCategory } from "../BooruTypes.js";
+import { CategoryMap } from "./Common.js";
+import { createNoteFromDanbooruArticle, guessContentType, parseResolutionString } from "../Utility.js";

 // Small hack to avoid screwing with semver
 enum Version {
@ -11,6 +11,17 @@ enum Version {

 export default class Gelbooru extends ScrapeEngineBase {
  name = "gelbooru";
+  features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
+  notes = ["Supported features might vary between hosts."];
+  supportedHosts = [
+    "safebooru.org",
+    "gelbooru.com",
+    "rule34.xxx",
+    "tbib.org",
+    "xbooru.com",
+    "mspabooru.com",
+    "hypnohub.net",
+  ];

  private readonly classNameToCategoryMap: CategoryMap = {
    "tag-type-copyright": "copyright",
@ -19,19 +30,16 @@ export default class Gelbooru extends ScrapeEngineBase {
    "tag-type-metadata": "meta",
  };

-  canImport(url: Location): boolean {
-    return (
-      url.host == "safebooru.org" || url.host == "gelbooru.com" || url.host == "rule34.xxx" || url.host == "tbib.org"
-    );
-  }
-
  scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
    let version: Version | null = null;
    switch (document.location.host) {
      case "safebooru.org":
      case "rule34.xxx":
      case "tbib.org":
+      case "xbooru.com":
+      case "mspabooru.com":
+      case "hypnohub.net":
        version = Version.v020;
        break;
      case "gelbooru.com":
@ -44,7 +52,7 @@ export default class Gelbooru extends ScrapeEngineBase {

    this.log("Guessed version: " + version);

-    let post = new ScrapedPost();
+    const post = new ScrapedPost();
    post.pageUrl = document.location.href;

    // Set image url
@ -81,10 +89,12 @@ export default class Gelbooru extends ScrapeEngineBase {
        if (matches && matches.length == 3) {
          // If Source element
          if (matches[1] == "Source") {
-            if (version == Version.v020) {
-              post.source = matches[2];
+            const anchor = el.querySelector("a");
+            if (anchor && anchor.href) {
+              post.sources = [anchor.href];
            } else {
-              post.source = (el.children[0] as HTMLAnchorElement).href;
+              // Fallback to plain text just in case
+              post.sources = [matches[2]];
            }
          }
          // If Rating element
@ -93,6 +103,7 @@ export default class Gelbooru extends ScrapeEngineBase {
              case "safe":
                post.rating = "safe";
                break;
+              case "sensitive":
              case "questionable":
                post.rating = "sketchy";
                break;
@ -103,18 +114,7 @@ export default class Gelbooru extends ScrapeEngineBase {
          }
          // If size element
          else if (matches[1] == "Size") {
-            // Example string: 1600x2200
-            const res = matches[2]
-              .split("x") // Split on 'x' character
-              .map((v, _) => parseInt(v)) // Parse ints
-              .filter(Number); // Filter NaN. Also removes the number 0, but that's fine because the resolution can't be 0.
-
-            if (res.length == 2) {
-              // Kinda hacky, but this is currently the correct way to convert an array to a tuple.
-              post.resolution = [res[0], res[1]];
-            } else {
-              this.log("Couldn't parse post resolution.");
-            }
+            post.resolution = parseResolutionString(matches[2]);
          }
        }
      }
@ -140,7 +140,9 @@ export default class Gelbooru extends ScrapeEngineBase {
          // Some <li/> don't actually contain a name.
          // E.g. the sub-headers ("Copyright", "Character", "General", "Meta") on rule34.xxx
          if (nameElements.length > 0) {
-            tagName = el.getElementsByTagName("a")[0].innerText;
+            // Get the last element, because rule34.xxx also adds a "?" link in front.
+            const linkEls = Array.from(el.getElementsByTagName("a"));
+            tagName = linkEls[linkEls.length - 1].innerText;
          } else {
            continue;
          }
@ -160,7 +162,7 @@ export default class Gelbooru extends ScrapeEngineBase {
          }
        }

-        let tag = new ScrapedTag(tagName, category);
+        const tag = new ScrapedTag(tagName, category);
        post.tags.push(tag);
      }
    }
@ -179,6 +181,7 @@ export default class Gelbooru extends ScrapeEngineBase {
    }
    // End of code duplication with Danbooru engine.

+    post.referrer = document.location.origin;
    result.tryAddPost(post);

    return result;
--- a/src/engines/Inkbunny.test.ts
+++ b/src/engines/Inkbunny.test.ts
@ -1,4 +1,4 @@
-import { scrapeUrl } from "../test/engine";
+import { scrapeUrl } from "../test/engine.js";

 test("get inkbunny.net post 2430530", async () => {
  const pageUrl = "https://inkbunny.net/s/2430530";
@ -7,6 +7,7 @@ test("get inkbunny.net post 2430530", async () => {
  expect(res.posts[0]).toHaveTag("comic");
  expect(res.posts[0]).toHaveTag("flower");
  expect(res.posts[0]).toHaveTag("medusa");
+  expect(res.posts[0]).toHaveResolution([1062, 1521]);
  expect(res.posts[0].pageUrl).toBe(pageUrl);
-  expect(res.posts[0].contentUrl).toBe("https://nl.ib.metapix.net/files/full/3562/3562592_vavacung_medusa.png");
+  expect(res.posts[0].contentUrl).toContain("metapix.net/files/full/3562/3562592_vavacung_medusa.png");
 });
--- a/src/engines/Inkbunny.ts
+++ b/src/engines/Inkbunny.ts
@ -1,16 +1,14 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";

-export default class Inkbunny implements ScrapeEngine {
+export default class Inkbunny extends ScrapeEngineBase {
  name = "inkbunny";
-
-  canImport(url: Location): boolean {
-    return url.host == "inkbunny.net";
-  }
+  features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags"];
+  notes = [];
+  supportedHosts = ["inkbunny.net"];

  scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
-
-    let post = new ScrapedPost();
+    const result = new ScrapeResult(this.name);
+    const post = new ScrapedPost();
    post.pageUrl = document.location.href;

    // Set image url
@ -33,13 +31,20 @@ export default class Inkbunny implements ScrapeEngine {
      return result;
    }

+    // Set post resolution
+    const width = parseInt((<HTMLInputElement>document.getElementById("submission-origwidth"))?.value);
+    const height = parseInt((<HTMLInputElement>document.getElementById("submission-origheight"))?.value);
+    if (!isNaN(width) && !isNaN(height)) {
+      post.resolution = [width, height];
+    }
+
    // Set rating
    const ratingExp = new RegExp("Rating: (.*)");
    const spanEls = Array.from(document.querySelectorAll("div > span"));
    const ratingEls = spanEls.map((x) => x as HTMLDivElement).filter((x) => x.innerText.indexOf("Rating:") != -1);

    if (ratingEls.length > 0) {
-      const matches = ratingEls[0].parentElement!.innerText.match(ratingExp);
+      const matches = ratingEls[0].parentElement?.innerText.match(ratingExp);
      if (matches && matches.length > 0) {
        switch (matches[1].toLowerCase()) {
          case "general":
@ -61,7 +66,7 @@ export default class Inkbunny implements ScrapeEngine {
      const keywords = keywordsContent.split(",").map((x) => x.trim());

      for (const keyword of keywords) {
-        let tag = new ScrapedTag(keyword);
+        const tag = new ScrapedTag(keyword);
        post.tags.push(tag);
      }
    }
--- a/src/engines/Mastodon.ts
+++ b/src/engines/Mastodon.ts
@ -0,0 +1,145 @@
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapeEngineFeature, ContentType} from "../ScrapeEngine.js";
+
+interface Attachment {
+  url: string;          // original media URL
+  contentType: ContentType;
+}
+
+export default class Mastodon extends ScrapeEngineBase {
+  name = "mastodon";
+  features: ScrapeEngineFeature[] = ["content"];
+  notes = [];
+  supportedHosts = [
+	"aethy.com",
+    "baraag.net",
+    "mastodon.online",
+    "mastodon.social",
+	"mastodon.world",
+	"mstdn.jp",
+	"mstdn.social",
+    "pawoo.net"
+  ];
+  /**
+   * Enabled for any other instances matching the pattern: .../@user/<postID>
+   */
+  canImport(url: Location): boolean {
+	return this.supportedHosts.indexOf(url.host.replace("www.", "")) !== -1
+		|| /^\/@[^/]+\/\d+\/?$/.test(url.pathname);
+  }
+
+  scrapeDocument(document: Document): ScrapeResult {
+	const result = new ScrapeResult(this.name);
+    const attachments = this.findAttachments(document);
+	
+    for (const att of attachments) {
+      const post = new ScrapedPost();
+      post.pageUrl = document.location.href;
+      post.contentUrl = att.url;
+      post.contentType = att.contentType;
+      post.referrer = document.location.origin;
+      result.tryAddPost(post);
+    }
+
+    return result;
+  }
+
+  private findAttachments(document: Document): Attachment[] {
+    const out: Attachment[] = [];
+    const seen = new Set<string>();
+
+    const add = (att: Attachment) => {
+      if (seen.has(att.url)) return;
+      seen.add(att.url);
+      out.push(att);
+    };
+
+    /**
+     *  Images -> a[href]
+	 *
+	 *  <a class="media-gallery__item-thumbnail" href=".../original/....png">
+     *    <img src=".../small/....png">
+     *  </a>
+	 */
+    let anchors = Array.from(
+      document.querySelectorAll<HTMLAnchorElement>("a.media-gallery__item-thumbnail[href]")
+    );
+    if (anchors.length === 0) {
+      anchors = Array.from(
+        document.querySelectorAll<HTMLAnchorElement>("a[href*='/media_attachments/']")
+      );
+    }
+    for (const a of anchors) {
+      const rawHref = a.getAttribute("href");
+      if (!rawHref) continue;
+
+      const img = a.querySelector("img");
+
+      const url = this.resolveUrl(rawHref) || this.largestFromSrcset(img);
+      if (!url) continue;
+
+      add({
+        url,
+        contentType: this.contentTypeFor(url),
+      });
+    }
+
+    /**
+	 *  GIFs and Videos -> video[src]
+	 *
+     *  <div class="media-gallery__gifv autoplay">
+     *    <video class="media-gallery__item-gifv-thumbnail" src=".../original/....mp4" autoplay loop></video>
+     *  </div>
+	 *
+     *  <div role="menuitem" class="video-player">
+     *    <video src=".../original/....mp4" poster=".../small/....png"></video>
+     *  </div>
+	 */
+    const videos = Array.from(document.querySelectorAll<HTMLVideoElement>("video[src]"));
+    for (const v of videos) {
+      const rawSrc = v.getAttribute("src") || v.currentSrc;
+      if (!rawSrc) continue;
+
+      const url = this.resolveUrl(rawSrc);
+      if (!url) continue;
+
+      add({
+        url,
+        contentType: "video",
+      });
+    }
+
+    return out;
+  }
+
+  /**
+   * Resolve possible relative media URLs against current page.
+   */
+  private resolveUrl(href: string): string {
+    try {
+	  return new URL(href, document.location.href).toString();
+    } catch {
+      return href;
+    }
+  }
+
+  private largestFromSrcset(img: HTMLImageElement | null): string | undefined {
+    if (!img) return undefined;
+    const srcset = img.getAttribute("srcset");
+    if (!srcset) return img.src || undefined;
+
+    let best: { url: string; w: number } | undefined;
+    for (const part of srcset.split(",")) {
+      const [urlPart, descriptor] = part.trim().split(/\s+/);
+      if (!urlPart) continue;
+      const w = descriptor?.endsWith("w") ? parseInt(descriptor, 10) : 0;
+      if (!best || w > best.w) best = { url: urlPart, w };
+    }
+    return best?.url;
+  }
+
+  private contentTypeFor(url: string): ContentType {
+    const path = url.split("?")[0].toLowerCase();
+    if (/\.(mp4|webm|mov|m4v)$/.test(path)) return "video";
+    return "image";
+  }
+}
--- a/src/engines/Moebooru.test.ts
+++ b/src/engines/Moebooru.test.ts
@ -1,9 +1,13 @@
-import { scrapeUrl, scrapeEvalUrl } from "../test/engine";
+import { scrapeUrl, scrapeEvalUrl } from "../test/engine.js";

 test("get yande.re post 438510", async () => {
  const pageUrl = "https://yande.re/post/show/438510";
  const res = await scrapeUrl(pageUrl);
  expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("the_legend_of_zelda", "copyright");
+  expect(res.posts[0]).toHaveTag("link", "character");
+  expect(res.posts[0]).toHaveTag("landscape");
+  expect(res.posts[0]).toHaveResolution([3000, 1688]);
  expect(res.posts[0].pageUrl).toBe(pageUrl);
  expect(res.posts[0].contentUrl).toBe(
    "https://files.yande.re/image/49b79bc372716a579d8525dcaccb56e2/yande.re%20438510%20landscape%20link%20nintendo%20princess_zelda%20sword%20the_legend_of_zelda%20the_legend_of_zelda%3A_breath_of_the_wild.jpg"
@ -16,6 +20,8 @@ test("get yande.re post 684478 with notes", async () => {
  expect(res.posts.length).toBe(1);
  expect(res.posts[0]).toHaveTag("gym_uniform");
  expect(res.posts[0]).toHaveNote("post #671879");
+  expect(res.posts[0]).toHaveResolution([2480, 3508]);
+  expect(res.posts[0]).toHaveSource("https://www.pixiv.net/artworks/84228323");
  expect(res.posts[0].pageUrl).toBe(pageUrl);
  expect(res.posts[0].contentUrl).toBe(
    "https://files.yande.re/image/8bdf7c54b0a459acee080f0aeb2a259c/yande.re%20684478%20gym_uniform%20pal_lett.jpg"
@ -26,6 +32,9 @@ test("get konachan.com post 289968", async () => {
  const pageUrl = "https://konachan.com/post/show/289968/ahri_-league_of_legends-akali-animal_ears-blonde_h";
  const res = await scrapeUrl(pageUrl);
  expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("league_of_legends", "copyright");
+  expect(res.posts[0]).toHaveResolution([1200, 783]);
+  expect(res.posts[0]).toHaveSource("https://www.pixiv.net/member_illust.php?mode=medium&illust_id=72418059");
  expect(res.posts[0].pageUrl).toBe(pageUrl);
  expect(res.posts[0].contentUrl).toBe(
    "https://konachan.com/image/e40b38fb704434e4da3eae41e26bca39/Konachan.com%20-%20289968%20akali%20blue_eyes%20breasts%20choker%20cleavage%20evelynn%20foxgirl%20gloves%20group%20hat%20kai%27sa%20long_hair%20microphone%20ponytail%20saphirya%20short_hair%20signed%20watermark.jpg"
@ -41,6 +50,8 @@ test("get konachan.com post 327454 with notes", async () => {
  expect(res.posts[0]).toHaveTag("kexue");
  expect(res.posts[0]).toHaveTag("water");
  expect(res.posts[0]).toHaveNote("Under construction.  \nBe careful of your safety");
+  expect(res.posts[0]).toHaveResolution([5629, 2505]);
+  expect(res.posts[0]).toHaveSource("https://www.pixiv.net/en/artworks/86644963");
  expect(res.posts[0].pageUrl).toBe(pageUrl);
  expect(res.posts[0].contentUrl).toBe(
    "https://konachan.com/image/94a13c8a1fcb9c0f6c9a6c6c5c316259/Konachan.com%20-%20327454%20apron%20brown_hair%20building%20car%20gloves%20kexue%20long_hair%20original%20rain%20see_through%20short_hair%20twintails%20uniform%20water.jpg"
@ -58,8 +69,28 @@ test("get konachan.com post 285782 with notes", async () => {
  expect(res.posts[0]).toHaveNote("I'M AWAKE!!");
  expect(res.posts[0]).toHaveNote("Poi!");
  expect(res.posts[0]).toHaveNote("Haaah!");
+  expect(res.posts[0]).toHaveResolution([1000, 830]);
  expect(res.posts[0].pageUrl).toBe(pageUrl);
  expect(res.posts[0].contentUrl).toBe(
    "https://konachan.com/image/24f5974ad8ed725292ccb3f682181bc1/Konachan.com%20-%20285782%20anthropomorphism%20bed%20black_eyes%20blonde_hair%20cropped%20fang%20kantai_collection%20long_hair%20momoniku%20pajamas%20white%20yuudachi_%28kancolle%29.jpg"
  );
+  expect(res.posts[0]).toHaveSource("https://www.pixiv.net/member_illust.php?mode=medium&illust_id=75747354");
+});
+
+test("get konachan.com post 358247 jpeg and png", async () => {
+  const pageUrl = "https://konachan.com/post/show/358247/3d-animal_ears-arknights-black_hair-blonde_hair-br";
+  const res = await scrapeUrl(pageUrl);
+  expect(res.posts.length).toBe(2);
+  expect(res.posts[0]).toHaveTag("texas_(arknights)", "character");
+  expect(res.posts[0]).toHaveResolution([6144, 3456]);
+  expect(res.posts[0].pageUrl).toBe(pageUrl);
+  expect(res.posts[0].contentUrl).toBe(
+    "https://konachan.com/image/56983a2166a13f4af963774ccd424f0e/Konachan.com%20-%20358247%203d%20arknights%20computer%20couch%20dress%20drink%20gloves%20gun%20halo%20hood%20necklace%20pantyhose%20red_eyes%20red_hair%20shirt%20shorts%20skirt%20tie%20twintails%20weapon%20wolfgirl.png"
+  );
+  expect(res.posts[0]).toHaveSource("https://www.pixiv.net/en/artworks/108925945");
+
+  // JPEG variant
+  expect(res.posts[1].contentUrl).toBe(
+    "https://konachan.com/jpeg/56983a2166a13f4af963774ccd424f0e/Konachan.com%20-%20358247%203d%20arknights%20computer%20couch%20dress%20drink%20gloves%20gun%20halo%20hood%20necklace%20pantyhose%20red_eyes%20red_hair%20shirt%20shorts%20skirt%20tie%20twintails%20weapon%20wolfgirl.jpg"
+  );
 });
--- a/src/engines/Moebooru.ts
+++ b/src/engines/Moebooru.ts
@ -1,19 +1,22 @@
-import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapedNote } from "../ScrapeEngine";
-import { TagCategory } from "../BooruTypes";
-import { createNotesFromMoebooruBoxes, guessContentType } from "../Utility";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
+import { TagCategory } from "../BooruTypes.js";
+import { createNotesFromMoebooruBoxes, guessContentType, parseResolutionString } from "../Utility.js";

 export default class Moebooru extends ScrapeEngineBase {
  name = "moebooru";
-
-  canImport(url: Location): boolean {
-    return url.host == "yande.re" || url.host == "konachan.com";
-  }
+  features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
+  notes = [];
+  supportedHosts = ["yande.re", "konachan.com", "konachan.net", "sakugabooru.com", "img.genshiken-itb.org"];

  scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
-    let post = new ScrapedPost();
+    const result = new ScrapeResult(this.name);
+    const post = new ScrapedPost();
    post.pageUrl = document.location.href;

+    if (document.location.host == "yande.re") {
+      post.uploadMode = "content";
+    }
+
    // Set image url
    const originalImageElement = document.querySelector("#highres") as HTMLAnchorElement;
    if (originalImageElement) {
@ -35,8 +38,8 @@ export default class Moebooru extends ScrapeEngineBase {
      const matches = el.innerText.match(regex);
      if (matches && matches.length == 3) {
        // If Source element
-        if (matches[1] == "Source") {
-          post.source = (el.children[0] as HTMLAnchorElement).href;
+        if (matches[1] == "Source" && el.children.length > 0) {
+          post.sources = [(el.children[0] as HTMLAnchorElement).href];
        }
        // If Rating element
        else if (matches[1] == "Rating") {
@ -54,18 +57,7 @@ export default class Moebooru extends ScrapeEngineBase {
        }
        // If size element
        else if (matches[1] == "Size") {
-          // Example string: 1600x2200
-          const res = matches[2]
-            .split("x") // Split on 'x' character
-            .map((v, _) => parseInt(v)) // Parse ints
-            .filter(Number); // Filter NaN. Also removes the number 0, but that's fine because the resolution can't be 0.
-
-          if (res.length == 2) {
-            // Kinda hacky, but this is currently the correct way to convert an array to a tuple.
-            post.resolution = [res[0], res[1]];
-          } else {
-            this.log("Couldn't parse post resolution.");
-          }
+          post.resolution = parseResolutionString(matches[2]);
        }
      }
    }
@ -75,12 +67,14 @@ export default class Moebooru extends ScrapeEngineBase {

    for (const el of tagElements) {
      let tagName: string;
-      let tagNameElements = el.getElementsByTagName("a");
+      const tagNameElements = el.getElementsByTagName("a");

      if (tagNameElements.length > 1) {
        // First <a> is "?"
-        // Second <a> is "actual_tag_name"
-        tagName = tagNameElements[1].innerText;
+        // (opt) Second <a> is "+" (when logged in)
+        // (opt) Third <a> is "-" (when logged in)
+        // Last <a> is the actual tag name
+        tagName = tagNameElements[tagNameElements.length - 1].innerText;
      } else {
        // Skip if no tag name is found
        continue;
@ -107,7 +101,7 @@ export default class Moebooru extends ScrapeEngineBase {
        }
      }

-      let tag = new ScrapedTag(tagName, category);
+      const tag = new ScrapedTag(tagName, category);
      post.tags.push(tag);
    }

@ -120,6 +114,18 @@ export default class Moebooru extends ScrapeEngineBase {
      post.notes = createNotesFromMoebooruBoxes(document, boxSize);
    }

+    // Sometimes there is both a JPEG and a PNG version of a post.
+    // If this is the case, copy all the information from the JPEG
+    // version and only replace the contentUrl with the link to the PNG version.
+    // We want to add this PNG version before the other version, so that it appears on top.
+    const unchangedImgEl = document.querySelector<HTMLAnchorElement>("a.original-file-unchanged");
+    if (unchangedImgEl && unchangedImgEl.href != post.contentUrl) {
+      const copy = Object.assign(new ScrapedPost(), post);
+      copy.contentUrl = unchangedImgEl.href;
+      result.tryAddPost(copy);
+    }
+
+    // Add normal/non-png post.
    result.tryAddPost(post);

    return result;
--- a/src/engines/Nozomi.ts
+++ b/src/engines/Nozomi.ts
@ -0,0 +1,49 @@
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
+import { TagCategory } from "../BooruTypes.js";
+
+export default class e621 extends ScrapeEngineBase {
+  name = "nozomi";
+  features: ScrapeEngineFeature[] = ["content", "tags", "tag_category"];
+  notes = ["Image preview might not work due to hotlink protection."];
+  supportedHosts = ["nozomi.la"];
+
+  scrapeDocument(document: Document): ScrapeResult {
+    const result = new ScrapeResult(this.name);
+    const post = new ScrapedPost();
+    post.pageUrl = document.location.href;
+
+    // Set image url
+    const downloadEl = document.querySelector(".post a") as HTMLAnchorElement;
+    post.contentUrl = downloadEl?.href;
+
+    // Set tags
+    const tagEls = Array.from(document.querySelectorAll(".sidebar li")).map((x) => x as HTMLLIElement);
+
+    for (const el of tagEls) {
+      const tagName = el.innerText;
+      let category: TagCategory | undefined;
+
+      switch (el.querySelector("a")?.className) {
+        // https://e621.net/help/api#tags_listing
+        case "copyright":
+          category = "copyright";
+          break;
+        case "character":
+          category = "character";
+          break;
+        case "artist":
+          category = "artist";
+          break;
+      }
+
+      if (tagName) {
+        const tag = new ScrapedTag(tagName, category);
+        post.tags.push(tag);
+      }
+    }
+
+    result.tryAddPost(post);
+
+    return result;
+  }
+}
--- a/src/engines/Philomena.test.ts
+++ b/src/engines/Philomena.test.ts
@ -0,0 +1,43 @@
+import { scrapeUrl } from "../test/engine.js";
+
+test("get derpibooru.org post 3153524", async () => {
+  const pageUrl = "https://derpibooru.org/images/3153524";
+  const res = await scrapeUrl(pageUrl);
+  expect(res.posts.length).toBe(1);
+  expect(res.posts[0].rating).toBe("safe");
+  expect(res.posts[0]).not.toHaveTag("safe");
+  expect(res.posts[0]).toHaveTag("yay");
+  expect(res.posts[0]).toHaveTag("flutteryay");
+  expect(res.posts[0]).toHaveSource("https://twitter.com/Confetti_Cakez/status/1674145455666544640");
+  expect(res.posts[0]).toHaveResolution([4000, 4000]);
+  expect(res.posts[0].pageUrl).toBe(pageUrl);
+  expect(res.posts[0].contentUrl).toBe("https://derpicdn.net/img/download/2023/6/29/3153524.jpg");
+});
+
+test("get derpibooru.org post 3157356", async () => {
+  const pageUrl = "https://derpibooru.org/images/3157356?q=oc";
+  const res = await scrapeUrl(pageUrl);
+  expect(res.posts.length).toBe(1);
+  expect(res.posts[0].rating).toBe("safe");
+  expect(res.posts[0]).not.toHaveTag("safe");
+  expect(res.posts[0]).toHaveTag("oc");
+  expect(res.posts[0]).toHaveTag("serra20", "artist");
+  expect(res.posts[0]).toHaveTag("pony", "species");
+  expect(res.posts[0]).toHaveTag("eris_(oc)", "character");
+  expect(res.posts[0]).toHaveResolution([960, 960]);
+  expect(res.posts[0].pageUrl).toBe(pageUrl);
+  expect(res.posts[0].contentUrl).toBe("https://derpicdn.net/img/download/2023/7/5/3157356.png");
+});
+
+test("get derpibooru.org video post 3161674", async () => {
+  const pageUrl = "https://derpibooru.org/images/3161674?q=video";
+  const res = await scrapeUrl(pageUrl);
+  expect(res.posts.length).toBe(1);
+  expect(res.posts[0].rating).toBe("safe");
+  expect(res.posts[0]).not.toHaveTag("safe");
+  expect(res.posts[0]).toHaveTag("oc");
+  expect(res.posts[0]).toHaveTag("rainbow_dash", "character");
+  expect(res.posts[0]).toHaveResolution([1920, 1080]);
+  expect(res.posts[0].pageUrl).toBe(pageUrl);
+  expect(res.posts[0].contentUrl).toBe("https://derpicdn.net/img/download/2023/7/12/3161674.webm");
+});
--- a/src/engines/Philomena.ts
+++ b/src/engines/Philomena.ts
@ -0,0 +1,127 @@
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
+import { SafetyRating, TagCategory } from "../BooruTypes.js";
+import { parseResolutionString } from "../Utility.js";
+
+export default class Philomena extends ScrapeEngineBase {
+  name = "Philomena";
+  features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source"];
+  notes = [];
+  supportedHosts = [
+    "derpibooru.org",
+    "trixiebooru.org",
+    "ponybooru.org",
+    "furbooru.org",
+    "ponerpics.org",
+    "manebooru.art",
+    "twibooru.org",
+  ];
+  removePrefixes = ["artist", "editor", "character", "species", "oc", "ship"];
+  removePrefixesSuffix = new Map([["oc", "_(oc)"]]);
+  artistPrefixes = ["artist", "editor"];
+  safetyMap = new Map<string, SafetyRating>([
+    ["safe", "safe"],
+    ["suggestive", "sketchy"],
+    ["questionable", "sketchy"],
+    ["explicit", "unsafe"],
+  ]);
+  categoryMap = new Map<string, TagCategory>([
+    ["character", "character"],
+    ["species", "species"],
+    ["oc", "character"],
+    ["content-official", "copyright"],
+  ]);
+
+  scrapeDocument(document: Document): ScrapeResult {
+    const result = new ScrapeResult(this.name);
+    const post = new ScrapedPost();
+    post.pageUrl = document.location.href;
+
+    // Set image url
+    const downloadEl = document.querySelector("a[title='Download (no tags in filename)']") as HTMLAnchorElement;
+    post.contentUrl = downloadEl?.href;
+
+    // Set content type
+    if (document.querySelector("video#image-display") != null) {
+      post.contentType = "video";
+    }
+
+    // Set default rating (Will be determined later)
+    post.rating = "unsafe";
+
+    // Set resolution
+    const resStr = (<HTMLElement>document.querySelector("span[class='image-size']"))?.innerText?.match(
+      /(\d+x\d+)/
+    )?.[0];
+    if (resStr) {
+      const res = parseResolutionString(resStr);
+      if (res) {
+        post.resolution = res;
+      }
+    }
+
+    // Set tags
+    const tagEls = Array.from(document.querySelectorAll(".tag.dropdown")).map((x) => x as HTMLSpanElement);
+
+    for (const el of tagEls) {
+      let tagName = el.getAttribute("data-tag-name");
+      if (!tagName) continue;
+
+      // Set rating (philomena uses tags for this)
+      // These tags also have `data-tag-category="rating"` which we could use.
+      const safety = this.safetyMap.get(tagName);
+      if (safety != undefined) {
+        post.rating = safety;
+        // Don't add this as a tag.
+        continue;
+      }
+
+      let category: TagCategory | undefined;
+
+      // Categorize tags using data-tag-category
+      const dataTagCategory = el.getAttribute("data-tag-category");
+      if (dataTagCategory) {
+        const mappedCategory = this.categoryMap.get(dataTagCategory);
+        if (mappedCategory) {
+          category = mappedCategory;
+        }
+      }
+
+      // Artist tag categorize using prefixes
+      for (const prefix of this.artistPrefixes) {
+        if (tagName.includes(prefix + ":")) {
+          category = "artist";
+          break;
+        }
+      }
+
+      // Remove listed prefixes (removePrefixes) from tagName
+      for (const prefix of this.removePrefixes) {
+        if (tagName.includes(prefix + ":")) {
+          tagName = tagName.replace(prefix + ":", "").trim();
+
+          // Add suffix if configured
+          const suffix = this.removePrefixesSuffix.get(prefix);
+          if (suffix) {
+            tagName = tagName + suffix;
+          }
+
+          break;
+        }
+      }
+
+      // Push tag
+      if (tagName) {
+        const tag = new ScrapedTag(tagName, category);
+        post.tags.push(tag);
+      }
+    }
+
+    // Set source
+    const sourceEls = Array.from(document.querySelectorAll("a.js-source-link")).map((x) => x as HTMLAnchorElement);
+    post.sources = sourceEls.map((x) => x.href);
+
+    result.tryAddPost(post);
+
+    return result;
+  }
+}
--- a/src/engines/Pixiv.test.ts
+++ b/src/engines/Pixiv.test.ts
@ -1,5 +1,18 @@
-import { ScrapeResults } from "../ScrapeEngine";
-import { scrapeEvalUrl } from "../test/engine";
+import { ScrapeResults } from "../ScrapeEngine.js";
+import { MobileUserAgent, scrapeEvalUrl, scrapePage } from "../test/engine.js";
+import { upgradeUrlToOriginalQuality } from "./Pixiv.js";
+
+test("upgradeUrlToOriginalQuality", async () => {
+  expect(
+    upgradeUrlToOriginalQuality("https://i.pximg.net/img-master/img/2022/02/15/06/50/28/96268450_p2_master1200.jpg")
+  ).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p2.jpg");
+
+  expect(
+    upgradeUrlToOriginalQuality(
+      "https://i.pximg.net/c/600x1200_90_webp/img-master/img/2022/02/15/06/50/28/96268450_p2_master1200.jpg"
+    )
+  ).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p2.jpg");
+});

 describe("get pixiv post 93964654", () => {
  const pageUrl = "https://www.pixiv.net/en/artworks/93964654";
@ -9,10 +22,72 @@ describe("get pixiv post 93964654", () => {
    res = await scrapeEvalUrl(pageUrl, ".gtm-expand-full-size-illust");
  });

-  it("should grab original image url", async () => {
-    pending("Test does not work.");
-    // expect(res.posts.length).toBe(1);
-    // expect(res.posts[0].pageUrl).toBe(pageUrl);
-    // expect(res.posts[0].contentUrl).toBe("https://i.pximg.net/img-original/img/2021/11/07/00/01/02/93964654_p0.jpg");
+  it("should find exactly one post", () => {
+    expect(res.posts.length).toBe(1);
+  });
+
+  it("should grab original image url", () => {
+    expect(res.posts[0].contentUrl).toBe("https://i.pximg.net/img-original/img/2021/11/07/00/01/02/93964654_p0.jpg");
+  });
+});
+
+describe("get pixiv post 93964654 (mobile)", () => {
+  const pageUrl = "https://www.pixiv.net/en/artworks/93964654";
+  let res: ScrapeResults;
+
+  beforeAll(async () => {
+    res = await scrapeEvalUrl(pageUrl, ".work-thumb", MobileUserAgent);
+  });
+
+  it("should find exactly one post", () => {
+    expect(res.posts.length).toBe(1);
+  });
+
+  it("should grab original image url", () => {
+    expect(res.posts[0].contentUrl).toBe("https://i.pximg.net/img-original/img/2021/11/07/00/01/02/93964654_p0.jpg");
+  });
+});
+
+describe("get pixiv post 96268450 (multipage, not expanded)", () => {
+  const pageUrl = "https://www.pixiv.net/en/artworks/96268450";
+  let res: ScrapeResults;
+
+  beforeAll(async () => {
+    res = await scrapeEvalUrl(pageUrl, "div[role='presentation'] img");
+  });
+
+  it("should find exactly one post", () => {
+    expect(res.posts.length).toBe(1);
+  });
+
+  it("should grab original image url", () => {
+    expect(res.posts[0].contentUrl).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p0.jpg");
+  });
+});
+
+describe("get pixiv post 96268450 (multipage, expanded)", () => {
+  const pageUrl = "https://www.pixiv.net/en/artworks/96268450";
+  let res: ScrapeResults;
+
+  beforeAll(async () => {
+    await jestPuppeteer.resetPage();
+    await page.goto(pageUrl);
+    await page.waitForSelector("div[role='presentation'] img", { visible: true });
+    await expect(page).toClick("button", { text: "Show all" });
+    await page.waitForSelector(".gtm-expand-full-size-illust", { visible: true });
+    // await jestPuppeteer.debug();
+    await page.waitForTimeout(2000);
+    res = await scrapePage(pageUrl, page);
+  });
+
+  it("should find exactly four posts", () => {
+    expect(res.posts.length).toBe(4);
+  });
+
+  it("should grab original image url", () => {
+    expect(res.posts[0].contentUrl).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p0.jpg");
+    expect(res.posts[1].contentUrl).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p1.jpg");
+    expect(res.posts[2].contentUrl).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p2.jpg");
+    expect(res.posts[3].contentUrl).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p3.jpg");
  });
 });
--- a/src/engines/Pixiv.ts
+++ b/src/engines/Pixiv.ts
@ -1,28 +1,78 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
-import { TagCategory } from "../BooruTypes";
-import { guessContentType } from "../Utility";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapeEngineFeature } from "../ScrapeEngine.js";
+import { guessContentType } from "../Utility.js";

-export default class Pixiv implements ScrapeEngine {
-  name = "pixiv";
+export function upgradeUrlToOriginalQuality(imgLink: string) {
+  if (imgLink.indexOf("img-master") != -1) {
+    // Example:
+    // Input:  https://i.pximg.net/img-master/img/2022/02/15/06/50/28/96268450_p2_master1200.jpg
+    // Input:  https://i.pximg.net/c/600x1200_90_webp/img-master/img/2022/02/15/06/50/28/96268450_p2_master1200.jpg
+    // Output: https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p2.jpg
+    //
+    // Warning:
+    // This will NOT work for a srcset like this:
+    // https://i.pximg.net/c/540x540_70/img-master/img/2022/02/15/06/50/28/96268450_p0_master1200.jpg 540w,https://i.pximg.net/img-master/img/2022/02/15/06/50/28/96268450_p0_master1200.jpg
+    //
+    // See regexr.com/6rcag for an example.

-  canImport(url: Location): boolean {
-    return url.host == "pixiv.net" || url.host == "www.pixiv.net";
+    // TODO: Add a check to verify the newly constructed/guessed url?
+    return imgLink.replace(/(https?:\/\/i\.pximg\.net).*\/img-master\/(.*)_master\d+(.*)/, "$1/img-original/$2$3");
  }

-  scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+  return imgLink;
+}

-    const imgLinks = Array.from(document.querySelectorAll("div[role='presentation'] > a")).map(
-      (x) => x as HTMLAnchorElement
+export default class Pixiv extends ScrapeEngineBase {
+  name = "pixiv";
+  features: ScrapeEngineFeature[] = ["content"];
+  notes = [];
+  supportedHosts = ["pixiv.net"];
+
+  scrapeDocument(document: Document): ScrapeResult {
+    const result = new ScrapeResult(this.name);
+    const imgLinks = [] as string[];
+
+    // Query for when the user is on the desktop site, is logged in, and has optionally clicked on "Show all".
+    // If the user has not clicked on "Show all" then this will only grab the first post.
+    imgLinks.push(
+      ...Array.from(document.querySelectorAll<HTMLAnchorElement>("div[role='presentation'] > a")).map((x) => x.href)
    );

+    if (imgLinks.length == 0) {
+      // Query for when the user is on the desktop site and not logged in.
+      imgLinks.push(
+        ...Array.from(document.querySelectorAll<HTMLImageElement>("div[role='presentation'] img")).map((x) => x.src)
+      );
+    }
+
+    if (imgLinks.length == 0) {
+      // Query for when the user is on the mobile site, and has clicked on "Show all".
+      // This only works on pages with multiple posts. The user can optionally be logged in.
+      imgLinks.push(
+        ...Array.from(document.querySelectorAll<HTMLImageElement>(".manga-page > a > img[data-big]")).map(
+          // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+          (x) => x.getAttribute("data-big")!
+        )
+      );
+    }
+
+    if (imgLinks.length == 0) {
+      // Query for when the user is on the mobile site, and has not clicked on "See all" -or- there is only one post.
+      const imgLink = document.querySelector<HTMLImageElement>(".work-main-image img")?.src;
+      if (imgLink) {
+        imgLinks.push(imgLink);
+      }
+    }
+
    for (const imgLink of imgLinks) {
-      let post = new ScrapedPost();
+      const post = new ScrapedPost();
      post.pageUrl = document.location.href;
-      post.contentUrl = imgLink.href;
+      post.contentUrl = upgradeUrlToOriginalQuality(imgLink);
      post.contentType = guessContentType(post.contentUrl);
      post.referrer = document.location.origin;
      result.tryAddPost(post);
+      if (document.querySelector('a[href*="/tags/R-18/artworks"]')) {
+        post.rating = "unsafe";
+      }
    }

    return result;
--- a/src/engines/Reddit.test.ts
+++ b/src/engines/Reddit.test.ts
@ -1,4 +1,4 @@
-import { scrapeUrl } from "../test/engine";
+import { scrapeUrl } from "../test/engine.js";

 test("get old.reddit.com post", async () => {
  const pageUrl = "https://old.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/";
--- a/src/engines/Reddit.ts
+++ b/src/engines/Reddit.ts
@ -1,16 +1,15 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine";
-import { guessContentType } from "../Utility";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapeEngineFeature } from "../ScrapeEngine.js";
+import { guessContentType } from "../Utility.js";

-export default class Reddit implements ScrapeEngine {
+export default class Reddit extends ScrapeEngineBase {
  name = "reddit";
-
-  canImport(url: Location): boolean {
-    return url.host.endsWith("reddit.com");
-  }
+  features: ScrapeEngineFeature[] = ["content"];
+  notes = [];
+  supportedHosts = ["reddit.com", "new.reddit.com", "old.reddit.com"];

  scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
-    let post = new ScrapedPost();
+    const result = new ScrapeResult(this.name);
+    const post = new ScrapedPost();
    post.pageUrl = document.location.href;

    // Set image url
--- a/src/engines/Rule34us.test.ts
+++ b/src/engines/Rule34us.test.ts
@ -0,0 +1,18 @@
+import { scrapeUrl } from "../test/engine.js";
+
+test("get rule34.us post 4646579", async () => {
+  // Kinda sketchy.
+  const pageUrl = "https://rule34.us/index.php?r=posts/view&id=4646579";
+  const res = await scrapeUrl(pageUrl);
+  expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("sciamano240", "artist");
+  expect(res.posts[0]).toHaveTag("mario", "character");
+  expect(res.posts[0]).toHaveTag("nintendo", "copyright");
+  expect(res.posts[0]).toHaveTag("crown");
+  expect(res.posts[0]).toHaveTag("solo");
+  expect(res.posts[0]).not.toHaveTag("original");
+  expect(res.posts[0]).not.toHaveTag("edit");
+  expect(res.posts[0]).toHaveResolution([2100, 3000]);
+  expect(res.posts[0].pageUrl).toBe(pageUrl);
+  expect(res.posts[0].contentUrl).toBe("https://img2.rule34.us/images/91/16/91160933dc1b3c737b3ef4a25b12680d.png");
+});
--- a/src/engines/Rule34us.ts
+++ b/src/engines/Rule34us.ts
@ -1,29 +1,27 @@
 // Technically speaking rule34.us also uses gelbooru, albeit a (heavily?) modified one.
 // Because pretty much no scraper code can be shared with other gelbooru-likes it makes more sense to create a new engine for rule34.us.

-import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
-import { TagCategory } from "../BooruTypes";
-import { CategoryMap } from "./Common";
-import { guessContentType } from "../Utility";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
+import { TagCategory } from "../BooruTypes.js";
+import { CategoryMap } from "./Common.js";
+import { guessContentType, parseResolutionString } from "../Utility.js";

-export default class Gelbooru implements ScrapeEngine {
+export default class Gelbooru extends ScrapeEngineBase {
  name = "rule34us";
+  features: ScrapeEngineFeature[] = ["content", "resolution", "tags", "tag_category"];
+  notes = ["Rating is assumed to be unsafe."];
+  supportedHosts = ["rule34.us"];

  private readonly classNameToCategoryMap: CategoryMap = {
    "character-tag": "character",
-    "artist-tag": "character",
+    "artist-tag": "artist",
    "copyright-tag": "copyright",
    "metadata-tag": "meta",
  };

-  canImport(url: Location): boolean {
-    return url.host == "rule34.us";
-  }
-
  scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
-
-    let post = new ScrapedPost();
+    const result = new ScrapeResult(this.name);
+    const post = new ScrapedPost();
    post.pageUrl = document.location.href;

    // Set image url
@ -40,6 +38,13 @@ export default class Gelbooru implements ScrapeEngine {
    // Set content type
    post.contentType = guessContentType(post.contentUrl);

+    // Set resolution
+    const statEls = Array.from(document.querySelectorAll<HTMLLIElement>(".container > ul > div > li"));
+    const sizeEl = statEls.find((x) => x.innerText.indexOf("Size:") != -1);
+    if (sizeEl) {
+      post.resolution = parseResolutionString(sizeEl.innerText);
+    }
+
    // Patch contentUrl if contentType == "video"
    // This is because the "Original" button does link to "302 Found" for videos. Idk why.
    // Shit's fucked, there are multiple workarounds for that and this is one of them.
--- a/src/engines/SankakuComplex.test.ts
+++ b/src/engines/SankakuComplex.test.ts
@ -1,37 +1,15 @@
-import { scrapeUrl } from "../test/engine";
+import { scrapeUrl } from "../test/engine.js";

 test("get chan.sankakucomplex.com post 19614481 with notes", async () => {
-  const pageUrl = "https://chan.sankakucomplex.com/post/show/19614481";
+  const pageUrl = "https://chan.sankakucomplex.com/en/posts/porqpNvZBa7";
  const res = await scrapeUrl(pageUrl);
  expect(res.posts.length).toBe(1);
  expect(res.posts[0]).toHaveTag("bangs");
  expect(res.posts[0]).toHaveTag("bare_shoulders");
-  expect(res.posts[0]).toHaveTag("solo");
-  expect(res.posts[0]).toHaveTag("translated");
-  expect(res.posts[0]).toHaveTag("text");
  expect(res.posts[0]).toHaveTag("idolmaster");
  expect(res.posts[0]).toHaveTag("mayuzumi_fuyuko");
-  expect(res.posts[0]).toHaveNote("What are you looking at...");
-  expect(res.posts[0]).toHaveNote("What...?");
+  // expect(res.posts[0]).toHaveNote("What are you looking at...");
+  // expect(res.posts[0]).toHaveNote("What...?");
  expect(res.posts[0].pageUrl).toBe(pageUrl);
-  expect(res.posts[0].contentUrl).toBe(
-    "https://s.sankakucomplex.com/data/50/7e/507e986a62cc5b866d92ad491dcfb1a4.jpg?e=1637625160&m=e0n_DCg-3fnmrhGzpI0Ftg"
-  );
-});
-
-test("get chan.sankakucomplex.com post 29354061 with notes", async () => {
-  // Kinda sketchy
-  const pageUrl = "https://chan.sankakucomplex.com/post/show/29354061";
-  const res = await scrapeUrl(pageUrl);
-  expect(res.posts.length).toBe(1);
-  expect(res.posts[0]).toHaveTag("bangs");
-  expect(res.posts[0]).toHaveTag("braid");
-  expect(res.posts[0]).toHaveTag("sankuro_(agoitei)");
-  expect(res.posts[0]).toHaveTag("translated");
-  expect(res.posts[0]).toHaveTag("note_translated");
-  expect(res.posts[0]).toHaveNote("No.  \nI already told you...");
-  expect(res.posts[0].pageUrl).toBe(pageUrl);
-  expect(res.posts[0].contentUrl).toBe(
-    "https://s.sankakucomplex.com/data/c8/37/c8376f02608548365d394a658e85cb3f.jpg?e=1637625020&m=A9xq-zht_FoVfoCiSRA-GA"
-  );
+  expect(res.posts[0].contentUrl).toContain("sankakucomplex.com/data/50/7e/507e986a62cc5b866d92ad491dcfb1a4.jpg");
 });
--- a/src/engines/SankakuComplex.ts
+++ b/src/engines/SankakuComplex.ts
@ -1,17 +1,16 @@
-import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapedNote } from "../ScrapeEngine";
-import { TagCategory } from "../BooruTypes";
-import { createNotesFromMoebooruBoxes, guessContentType } from "../Utility";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
+import { TagCategory } from "../BooruTypes.js";
+import { createNotesFromMoebooruBoxes, guessContentType } from "../Utility.js";

 export default class SankakuComplex extends ScrapeEngineBase {
  name = "sankakucomplex";
-
-  canImport(url: Location): boolean {
-    return url.host == "chan.sankakucomplex.com";
-  }
+  features: ScrapeEngineFeature[] = ["content", "rating", "tags", "tag_category", "notes"];
+  notes = [];
+  supportedHosts = ["chan.sankakucomplex.com"];

  scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
-    let post = new ScrapedPost();
+    const result = new ScrapeResult(this.name);
+    const post = new ScrapedPost();
    post.pageUrl = document.location.href;

    // Set image url
@ -28,25 +27,14 @@ export default class SankakuComplex extends ScrapeEngineBase {
    post.contentType = guessContentType(post.contentUrl);

    // Set rating
-    const safetyExp = new RegExp("Rating: (.*)");
-    const safetyElements = Array.from(document.querySelectorAll("#stats > ul > li"))
-      .map((x) => x as HTMLLIElement)
-      .filter((x) => x.innerText.startsWith("Rating"));
-
-    if (safetyElements.length > 0) {
-      const matches = safetyElements[0].innerText.match(safetyExp);
-      if (matches && matches.length > 0) {
-        switch (matches[1].toLowerCase()) {
-          case "safe":
-            post.rating = "safe";
-            break;
-          case "questionable":
-            post.rating = "sketchy";
-            break;
-          case "explicit":
-            post.rating = "unsafe";
-            break;
-        }
+    const ratingElement = document.querySelector("#stats .rating-s, #stats .rating-q, #stats .rating-e");
+    if (ratingElement) {
+      if (ratingElement.classList.contains("rating-s")) {
+        post.rating = "safe";
+      } else if (ratingElement.classList.contains("rating-q")) {
+        post.rating = "sketchy";
+      } else if (ratingElement.classList.contains("rating-e")) {
+        post.rating = "unsafe";
      }
    }

@ -76,7 +64,7 @@ export default class SankakuComplex extends ScrapeEngineBase {
      }

      if (tagName) {
-        let tag = new ScrapedTag(tagName, category);
+        const tag = new ScrapedTag(tagName, category);
        post.tags.push(tag);
      }
    }
--- a/src/engines/Shimmie2.test.ts
+++ b/src/engines/Shimmie2.test.ts
@ -1,11 +1,13 @@
-import { scrapeUrl } from "../test/engine";
+import { scrapeUrl } from "../test/engine.js";

 test("get rule34.paheal.net post 2505982", async () => {
  const pageUrl = "https://rule34.paheal.net/post/view/2505982"; // Somewhat sketchy
  const res = await scrapeUrl(pageUrl);
  expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("kai'sa");
  expect(res.posts[0].pageUrl).toBe(pageUrl);
-  expect(res.posts[0].contentUrl).toContain(
-    ".paheal.net/_images/15427ab5ed2cddbeef947e30f5c9cab6/2505982%20-%20Kai%27Sa%20League_of_Legends%20lawzilla.png"
+  expect(res.posts[0]).toHaveSource(
+    "http://law-zilla.tumblr.com/post/171249285285/finished-subdraw-37-kaisa-from-league-of-legends"
  );
+  expect(res.posts[0].contentUrl).toContain("https://r34i.paheal-cdn.net");
 });
--- a/src/engines/Shimmie2.ts
+++ b/src/engines/Shimmie2.ts
@ -1,31 +1,33 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
-import { guessContentType } from "../Utility";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
+import { guessContentType } from "../Utility.js";

-export default class Shimmie2 implements ScrapeEngine {
+export default class Shimmie2 extends ScrapeEngineBase {
  name = "shimmie2";
-
-  canImport(url: Location): boolean {
-    return url.host == "rule34.paheal.net" || url.host == "rule34hentai.net";
-  }
+  features: ScrapeEngineFeature[] = ["content", "tags", "source"];
+  notes = ["Rating is assumed to be unsafe."];
+  supportedHosts = ["rule34.paheal.net", "rule34hentai.net"];

  scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
-    let post = new ScrapedPost();
+    const result = new ScrapeResult(this.name);
+    const post = new ScrapedPost();
    post.pageUrl = document.location.href;

    // Set image url
-    const originalImageElements = Array.from(document.querySelectorAll(".image_info > tbody > tr > td > a"))
-      .map((x) => x as HTMLAnchorElement)
-      .filter((x) => x.innerText == "Image Only" || x.innerText == "File Only");
+    // const originalImageElements = Array.from(document.querySelectorAll(".image_info > tbody > tr > td > a"))
+    //   .map((x) => x as HTMLAnchorElement)
+    //   .filter((x) => x.innerText == "Image Only" || x.innerText == "File Only");

-    if (originalImageElements.length > 0) {
-      post.contentUrl = originalImageElements[0].href;
-    } else {
-      const dowloadBtnEl = document.querySelector("a[download]") as HTMLAnchorElement;
-      if (dowloadBtnEl) {
-        post.contentUrl = dowloadBtnEl.href;
-      }
-    }
+    // if (originalImageElements.length > 0) {
+    //   post.contentUrl = originalImageElements[0].href;
+    // } else {
+    //   const dowloadBtnEl = document.querySelector("a[download]") as HTMLAnchorElement;
+    //   if (dowloadBtnEl) {
+    //     post.contentUrl = dowloadBtnEl.href;
+    //   }
+    // }
+
+    // @ts-expect-error
+    post.contentUrl = document.getElementById("main_image").src;

    // Set content type
    post.contentType = guessContentType(post.contentUrl);
@ -45,7 +47,7 @@ export default class Shimmie2 implements ScrapeEngine {
    // Set source
    const sourceEl = document.querySelector(".image_info > tbody > tr > td > div > a") as HTMLAnchorElement;
    if (sourceEl) {
-      post.source = sourceEl.href;
+      post.sources = [sourceEl.href];
    }

    result.tryAddPost(post);
--- a/src/engines/Shuushuu.test.ts
+++ b/src/engines/Shuushuu.test.ts
@ -1,9 +1,15 @@
-import { scrapeUrl } from "../test/engine";
+import { scrapeUrl } from "../test/engine.js";

 test("get e-shuushuu.net post 1063586", async () => {
  const pageUrl = "https://e-shuushuu.net/image/1063586/";
  const res = await scrapeUrl(pageUrl);
  expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("monster");
+  expect(res.posts[0]).toHaveTag("wings");
+  expect(res.posts[0]).toHaveTag("celebi", "character");
+  expect(res.posts[0]).toHaveTag("shiina_mirin", "artist");
+  expect(res.posts[0]).toHaveTag("pokémon", "copyright");
+  expect(res.posts[0]).toHaveResolution([1447, 1023]);
  expect(res.posts[0].pageUrl).toBe(pageUrl);
  expect(res.posts[0].contentUrl).toBe("https://e-shuushuu.net/images/2021-08-28-1063586.jpeg");
 });
--- a/src/engines/Shuushuu.ts
+++ b/src/engines/Shuushuu.ts
@ -1,17 +1,16 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
-import { TagCategory } from "../BooruTypes";
-import { guessContentType } from "../Utility";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
+import { TagCategory } from "../BooruTypes.js";
+import { guessContentType, parseResolutionString } from "../Utility.js";

-export default class Shuushuu implements ScrapeEngine {
+export default class Shuushuu extends ScrapeEngineBase {
  name = "shuushuu";
-
-  canImport(url: Location): boolean {
-    return url.host == "e-shuushuu.net";
-  }
+  features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category"];
+  notes = ["Rating is assumed to be safe."];
+  supportedHosts = ["e-shuushuu.net"];

  scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
-    let post = new ScrapedPost();
+    const result = new ScrapeResult(this.name);
+    const post = new ScrapedPost();
    post.pageUrl = document.location.href;

    // Set image url
@ -26,6 +25,19 @@ export default class Shuushuu implements ScrapeEngine {
    // Set content type
    post.contentType = guessContentType(post.contentUrl);

+    // Set resolution
+    const metaEls = Array.from(document.querySelector<HTMLDataListElement>(".meta > dl")?.children ?? []);
+    const dimsDtEl = metaEls.find((x) => (<any>x).innerText == "Dimensions:");
+    if (dimsDtEl) {
+      const dimsDtIdx = metaEls.indexOf(dimsDtEl);
+      let str = (<any>metaEls[dimsDtIdx + 1])?.innerText;
+      if (str) {
+        // When str is "1447x1023 (1.480 MPixel)"
+        str = str.split("(")[0]; // Remove the "(1.480 MPixel)" part
+        post.resolution = parseResolutionString(str);
+      }
+    }
+
    // Set rating
    post.rating = "safe";

--- a/src/engines/Twitter.test.ts
+++ b/src/engines/Twitter.test.ts
@ -1,5 +1,5 @@
-import { ScrapeResults } from "../ScrapeEngine";
-import { scrapeEvalUrl } from "../test/engine";
+import { ScrapeResults } from "../ScrapeEngine.js";
+import { scrapeEvalUrl } from "../test/engine.js";

 describe("get twitter.com post", () => {
  const pageUrl = "https://twitter.com/Dishwasher1910/status/1171162651461746688/photo/1";
--- a/src/engines/Twitter.ts
+++ b/src/engines/Twitter.ts
@ -1,30 +1,29 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapeEngineFeature } from "../ScrapeEngine.js";

 export function getOriginalImageUrl(image: HTMLImageElement) {
  // if the link includes a resolution, change it to orig:
  // https://pbs.twimg.com/media/stuff?format=jpg&name=900x900
  // -> https://pbs.twimg.com/media/stuff?format=jpg&name=orig
-  let img_url = new URL((image as HTMLImageElement).src);
+  const img_url = new URL((image as HTMLImageElement).src);
  if (img_url.searchParams.has("name") && img_url.searchParams.get("name") != "orig") {
    img_url.searchParams.set("name", "orig");
  }
  return img_url;
 }

-export default class Twitter implements ScrapeEngine {
+export default class Twitter extends ScrapeEngineBase {
  name = "twitter";
-
-  canImport(url: Location): boolean {
-    return url.host == "twitter.com" || url.host == "mobile.twitter.com";
-  }
+  features: ScrapeEngineFeature[] = ["content"];
+  notes = [];
+  supportedHosts = ["twitter.com", "mobile.twitter.com", "x.com"];

  scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);

    // first look for a focused image on desktop:
    const center_img = document.querySelector("div[role=dialog] div[role=dialog] div[aria-label=Image] img");
    if (center_img) {
-      let post = new ScrapedPost();
+      const post = new ScrapedPost();
      post.pageUrl = document.location.href;
      post.contentUrl = getOriginalImageUrl(center_img as HTMLImageElement).href;
      result.tryAddPost(post);
@ -34,7 +33,7 @@ export default class Twitter implements ScrapeEngine {
    const imgs = [...document.querySelectorAll("article a[role='link'] div[data-testid='tweetPhoto'] img")];

    for (const img of imgs) {
-      let post = new ScrapedPost();
+      const post = new ScrapedPost();
      post.pageUrl = document.location.href;
      post.contentUrl = getOriginalImageUrl(img as HTMLImageElement).href;
      result.tryAddPost(post);
--- a/src/engines/Zerochan.test.ts
+++ b/src/engines/Zerochan.test.ts
@ -1,19 +1,23 @@
-import { scrapeUrl } from "../test/engine";
+import { scrapeUrl } from "../test/engine.js";

 test("get zerochan.net post 2934998", async () => {
  const pageUrl = "https://www.zerochan.net/2934998";
  const res = await scrapeUrl(pageUrl);
  expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("nintendo", "copyright");
+  expect(res.posts[0]).toHaveTag("akihorisu", "artist");
+  expect(res.posts[0]).toHaveTag("shirona_(pokémon)", "character");
+  expect(res.posts[0]).toHaveResolution([1186, 1009]);
  expect(res.posts[0].pageUrl).toBe(pageUrl);
-  expect(res.posts[0].tags.find((x) => x.name == "nintendo")).not.toBeUndefined();
  expect(res.posts[0].contentUrl).toBe("https://static.zerochan.net/Shirona.%28Pok%C3%A9mon%29.full.2934998.png");
 });

 test("get zerochan.net gif post 3511809", async () => {
-  const pageUrl = "https://www.zerochan.net/3511809";
+  const pageUrl = "https://www.zerochan.net/4356451";
  const res = await scrapeUrl(pageUrl);
  expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("honkai_star_rail");
+  expect(res.posts[0]).toHaveResolution([2192, 1644]);
  expect(res.posts[0].pageUrl).toBe(pageUrl);
-  expect(res.posts[0]).toHaveTag("rwby");
-  expect(res.posts[0].contentUrl).toBe("https://static.zerochan.net/Ruby.Rose.full.3511809.gif");
+  expect(res.posts[0].contentUrl).toBe("https://static.zerochan.net/Honkai.Star.Rail.full.4356451.jpg");
 });
--- a/src/engines/Zerochan.ts
+++ b/src/engines/Zerochan.ts
@ -1,22 +1,21 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
-import { TagCategory } from "../BooruTypes";
-import { guessContentType } from "../Utility";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
+import { TagCategory } from "../BooruTypes.js";
+import { guessContentType, parseResolutionString } from "../Utility.js";

-export default class Zerochan implements ScrapeEngine {
+export default class Zerochan extends ScrapeEngineBase {
  name = "zerochan";
-
-  canImport(url: Location): boolean {
-    return url.host.endsWith("zerochan.net");
-  }
+  features: ScrapeEngineFeature[] = ["content", "resolution", "tags", "tag_category"];
+  notes = ["Rating is assumed to be safe."];
+  supportedHosts = ["zerochan.net"];

  scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
-    let post = new ScrapedPost();
+    const result = new ScrapeResult(this.name);
+    const post = new ScrapedPost();
    post.pageUrl = document.location.href;

    // Set image url
    const previewElements = document.getElementsByClassName("preview");
-    if (previewElements.length > 0) {
+    if (previewElements.length > 0 && (previewElements[0] as HTMLAnchorElement).href) {
      // The preview element only exists when there is a larger size available. Preview's img element contains a downscaled version.
      post.contentUrl = (previewElements[0] as HTMLAnchorElement).href;
    } else {
@ -30,6 +29,12 @@ export default class Zerochan implements ScrapeEngine {
    // Set content type
    post.contentType = guessContentType(post.contentUrl);

+    // Set resolution
+    const resSizeEl = document.querySelector("#image-info")?.firstElementChild;
+    if (resSizeEl) {
+      post.resolution = parseResolutionString(resSizeEl.textContent?.replace("×", "x"));
+    }
+
    // Set rating
    post.rating = "safe";

@ -47,9 +52,11 @@ export default class Zerochan implements ScrapeEngine {
      }

      let category: TagCategory | undefined;
-      switch (tagElement.className) {
+      const className = tagElement.classList[0];
+      switch (className) {
        case "game":
        case "series":
+        case "studio":
          category = "copyright";
          break;
        case "character":
@ -59,12 +66,12 @@ export default class Zerochan implements ScrapeEngine {
          category = "artist";
          break;
        case "source":
-          // Skip these type of tags, because they are not 'booru-like'
+          // Skip these type of tags, because they are not 'booru-like'.
          continue;
      }

      if (tagName) {
-        let tag = new ScrapedTag(tagName, category);
+        const tag = new ScrapedTag(tagName, category);
        post.tags.push(tag);
      }
    }
--- a/src/engines/e621.test.ts
+++ b/src/engines/e621.test.ts
@ -0,0 +1,15 @@
+import { scrapeUrl } from "../test/engine.js";
+
+test("get e621.net post 380315", async () => {
+  const pageUrl = "https://e621.net/posts/380315?q=league_of_legends+rating%3Asafe+order%3Ascore";
+  const res = await scrapeUrl(pageUrl);
+  expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("inkinesss", "artist");
+  expect(res.posts[0]).toHaveTag("nasus_(lol)", "character");
+  expect(res.posts[0]).toHaveTag("ambiguous_gender");
+  expect(res.posts[0]).toHaveTag("low_res", "meta");
+  expect(res.posts[0]).toHaveResolution([500, 500]);
+  expect(res.posts[0]).toHaveSource("https://inkinesss.deviantart.com/art/Nasus-388347996");
+  expect(res.posts[0].pageUrl).toBe(pageUrl);
+  expect(res.posts[0].contentUrl).toBe("https://static1.e621.net/data/92/e7/92e71bd73ab219d80516c4d9adad3db5.jpg");
+});
--- a/src/engines/e621.ts
+++ b/src/engines/e621.ts
@ -1,21 +1,19 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
-import { TagCategory } from "../BooruTypes";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
+import { TagCategory } from "../BooruTypes.js";

-export default class e621 implements ScrapeEngine {
+export default class e621 extends ScrapeEngineBase {
  name = "e621";
-
-  canImport(url: Location): boolean {
-    return url.host == "e621.net" || url.host == "e926.net";
-  }
+  features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source"];
+  notes = [];
+  supportedHosts = ["e621.net", "e926.net", "e6ai.net"];

  scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
-
-    let post = new ScrapedPost();
+    const result = new ScrapeResult(this.name);
+    const post = new ScrapedPost();
    post.pageUrl = document.location.href;

    // Set image url
-    const downloadEl = document.querySelector("#image-download-link > a") as HTMLAnchorElement;
+    const downloadEl = document.querySelector(".ptbr-fullscreen-button") as HTMLAnchorElement;
    post.contentUrl = downloadEl?.href;

    // Set content type
@ -37,50 +35,54 @@ export default class e621 implements ScrapeEngine {
        break;
    }

+    // Set resolution
+    const width = parseInt((<HTMLSpanElement>document.querySelector("span[itemprop='width']"))?.innerText);
+    const height = parseInt((<HTMLSpanElement>document.querySelector("span[itemprop='height']"))?.innerText);
+    if (!isNaN(width) && !isNaN(height)) {
+      post.resolution = [width, height];
+    }
+
    // Set tags
    const tagEls = Array.from(document.querySelectorAll("#tag-list > ul > li")).map((x) => x as HTMLLIElement);

    for (const el of tagEls) {
-      const tagName = (el.querySelector(".search-tag") as HTMLAnchorElement)?.innerText;
+      const tagName = decodeURIComponent(el.getAttribute("data-name") as string);
      let category: TagCategory | undefined;

-      switch (el.className) {
+      switch (el.getAttribute("data-category")) {
        // https://e621.net/help/api#tags_listing
-        case "category-3":
+        case "copyright":
          category = "copyright";
          break;
-        case "category-4":
+        case "character":
          category = "character";
          break;
-        case "category-1":
+        case "artist":
          category = "artist";
          break;
-        case "category-7":
+        case "meta":
          category = "meta";
          break;
-        case "category-5":
+        case "species":
          category = "species";
          break;
-        // case "category-6":
+        // case "invalid":
        //   category = "invalid";
        //   break;
-        case "category-8":
+        case "lore":
          category = "lore";
          break;
      }

      if (tagName) {
-        let tag = new ScrapedTag(tagName, category);
+        const tag = new ScrapedTag(tagName, category);
        post.tags.push(tag);
      }
    }

    // Set source
    const sourceEls = Array.from(document.querySelectorAll(".source-link > a")).map((x) => x as HTMLAnchorElement);
-    // We currently only support one source.
-    if (sourceEls.length > 0) {
-      post.source = sourceEls[0].href;
-    }
+    post.sources = sourceEls.map((x) => x.href);

    result.tryAddPost(post);

--- a/src/engines/index.ts
+++ b/src/engines/index.ts
@ -1,16 +1,21 @@
-export { default as AnimePictures } from "./AnimePictures";
-export { default as Danbooru } from "./Danbooru";
-export { default as e621 } from "./e621";
-export { default as Fallback } from "./Fallback";
-export { default as FurAffinity } from "./FurAffinity";
-export { default as Gelbooru } from "./Gelbooru";
-export { default as Inkbunny } from "./Inkbunny";
-export { default as Moebooru } from "./Moebooru";
-export { default as Pixiv } from "./Pixiv";
-export { default as Reddit } from "./Reddit";
-export { default as Rule34us } from "./Rule34us";
-export { default as SankakuComplex } from "./SankakuComplex";
-export { default as Shimmie2 } from "./Shimmie2";
-export { default as Shuushuu } from "./Shuushuu";
-export { default as Twitter } from "./Twitter";
-export { default as Zerochan } from "./Zerochan";
+export { default as AnimePictures } from "./AnimePictures.js";
+export { default as Bluesky } from "./Bluesky.js";
+export { default as Danbooru } from "./Danbooru.js";
+export { default as e621 } from "./e621.js";
+export { default as Ehentai } from "./Ehentai.js";
+export { default as Fallback } from "./Fallback.js";
+export { default as FurAffinity } from "./FurAffinity.js";
+export { default as Gelbooru } from "./Gelbooru.js";
+export { default as Inkbunny } from "./Inkbunny.js";
+export { default as Mastodon } from "./Mastodon.js";
+export { default as Moebooru } from "./Moebooru.js";
+export { default as Nozomi } from "./Nozomi.js";
+export { default as Philomena } from "./Philomena.js";
+export { default as Pixiv } from "./Pixiv.js";
+export { default as Reddit } from "./Reddit.js";
+export { default as Rule34us } from "./Rule34us.js";
+export { default as SankakuComplex } from "./SankakuComplex.js";
+export { default as Shimmie2 } from "./Shimmie2.js";
+export { default as Shuushuu } from "./Shuushuu.js";
+export { default as Twitter } from "./Twitter.js";
+export { default as Zerochan } from "./Zerochan.js";
--- a/src/index.ts
+++ b/src/index.ts
@ -1,8 +1,8 @@
-export { default as NeoScraper } from "./NeoScraper";
-export * from "./ScrapeEngine";
+export { default as NeoScraper } from "./NeoScraper.js";
+export * from "./ScrapeEngine.js";

-import * as Engines from "./engines";
+import * as Engines from "./ScrapeEngine.js";
 export { Engines };

-import * as BooruTypes from "./BooruTypes";
+import * as BooruTypes from "./BooruTypes.js";
 export { BooruTypes };
--- a/src/test/engine.ts
+++ b/src/test/engine.ts
@ -1,5 +1,7 @@
 import { JSDOM, ConstructorOptions } from "jsdom";
-import NeoScraper from "../NeoScraper";
+import NeoScraper from "../NeoScraper.js";
+
+export const MobileUserAgent = "Mozilla/5.0 (Android 12; Mobile; rv:68.0) Gecko/68.0 Firefox/103.0";

 /**
 * Go to an URL and parse it's raw HTML without executing any client-side code.
@ -7,7 +9,9 @@ import NeoScraper from "../NeoScraper";
 * @returns
 */
 export async function scrapeUrl(url: string) {
-  const dom = await JSDOM.fromURL(url);
+  const dom = await JSDOM.fromURL(url, {
+    userAgent: "PostmanRuntime/7.37.3",
+  });
  return scrapeDom(dom.window.document);
 }

@ -17,9 +21,26 @@ export async function scrapeUrl(url: string) {
 * @param waitForSelector
 * @returns
 */
-export async function scrapeEvalUrl(url: string, waitForSelector: string) {
+export async function scrapeEvalUrl(
+  url: string,
+  waitForSelector: string,
+  userAgent: string | undefined = "PostmanRuntime/7.37.3"
+) {
+  await jestPuppeteer.resetPage();
+
+  if (userAgent) {
+    page.setUserAgent(userAgent);
+  }
+
  await page.goto(url);
-  await page.waitForSelector(waitForSelector, { visible: true });
+
+  const res = await page.waitForSelector(waitForSelector, { visible: true, timeout: 10_000 });
+  if (res == null) throw new Error("waitForSelector timed out.");
+
+  return scrapePage(url, page);
+}
+
+export async function scrapePage(url: string, page: any) {
  const document = await page.content();
  return scrapeHtml(url, document);
 }
--- a/src/test/jest.d.ts
+++ b/src/test/jest.d.ts
@ -1,10 +1,15 @@
 import "jest-extended";
+import { TagCategory } from "../BooruTypes.js";

 declare global {
  namespace jest {
    interface Matchers<R> {
-      toHaveTag(tagName: string): CustomMatcherResult;
+      toHaveTag(tagName: string, tagCategory?: TagCategory): CustomMatcherResult;
      toHaveNote(noteText: string): CustomMatcherResult;
+      toHaveResolution(resolution: [number, number]): CustomMatcherResult;
+      toHaveSource(source: string): CustomMatcherResult;
    }
  }
 }
+
+export {};
--- a/src/test/jest.setup.ts
+++ b/src/test/jest.setup.ts
@ -1,11 +1,11 @@
 import { JSDOM } from "jsdom";
-import { ScrapedPost } from "../ScrapeEngine";
+import { TagCategory } from "../BooruTypes.js";
+import { ScrapedPost } from "../ScrapeEngine.js";

 // Expose JSDOM Element constructor
-// @ts-ignore
 global.Element = new JSDOM().window.Element;
 // 'Implement' innerText in JSDOM: https://github.com/jsdom/jsdom/issues/1245
-// @ts-ignore
+// This implementation is not 100% correct, but usually good enough so that the tests pass.
 Object.defineProperty(global.Element.prototype, "innerText", {
  get() {
    return this.textContent.replace(/(\r\n|\n|\r|^\s+|\s+$)/gm, "");
@ -14,22 +14,51 @@ Object.defineProperty(global.Element.prototype, "innerText", {

 // Add our own functions
 expect.extend({
-  toHaveTag(received: any, tagName: string): jest.CustomMatcherResult {
-    let pass: boolean = false;
-    const message = () => (pass ? "" : `post does not have the '${tagName}' tag.`);
+  toHaveTag(
+    received: any,
+    tagName: string,
+    tagCategory: TagCategory | undefined = undefined
+  ): jest.CustomMatcherResult {
+    let pass = false;
+    const message = () => (pass ? "" : `Post does not have the '${tagName}' tag, or the category is incorrect.`);

    if (received instanceof ScrapedPost) {
-      pass = received.tags.find((x) => x.name == tagName) != undefined;
+      const found = received.tags.find((x) => x.name == tagName);
+      if (found) {
+        if (tagCategory) {
+          pass = found.category == tagCategory;
+        } else {
+          pass = true;
+        }
+      }
    }
    return { pass, message };
  },
  toHaveNote(received: any, noteText: string): jest.CustomMatcherResult {
-    let pass: boolean = false;
-    const message = () => (pass ? "" : `post does not have a note with '${noteText}'.`);
+    let pass = false;
+    const message = () => (pass ? "" : `Post does not have a note with '${noteText}'.`);

    if (received instanceof ScrapedPost) {
      pass = received.notes.find((x) => x.text == noteText) != undefined;
    }
    return { pass, message };
  },
+  toHaveResolution(received: any, resolution: [number, number]): jest.CustomMatcherResult {
+    let pass = false;
+    const message = () => (pass ? "" : `Expected: ${resolution}\nReceived: ${received?.resolution}`);
+
+    if (received instanceof ScrapedPost && received.resolution?.length == 2) {
+      pass = received.resolution[0] == resolution[0] && received.resolution[1] == resolution[1];
+    }
+    return { pass, message };
+  },
+  toHaveSource(received: any, source: string): jest.CustomMatcherResult {
+    let pass = false;
+    const message = () => (pass ? "" : `Post does not have source '${source}'.`);
+
+    if (received instanceof ScrapedPost) {
+      pass = received.sources.find((x) => x == source) != undefined;
+    }
+    return { pass, message };
+  },
 });
--- a/tsconfig.json
+++ b/tsconfig.json
@ -1,16 +1,17 @@
 {
  "compilerOptions": {
    "lib": [
-      "es2019",
-      "dom",
-      "dom.iterable"
+      "ES2019",
+      "DOM",
+      "DOM.Iterable"
    ],
+    "rootDir": "./src/",
    "outDir": "./dist/",
    "sourceMap": true,
    "strict": true,
-    "module": "es2015",
-    "moduleResolution": "node",
-    "target": "es2015",
+    "module": "node16",
+    "moduleResolution": "node16",
+    "target": "es2020",
    "skipLibCheck": true,
    "esModuleInterop": true,
    "removeComments": true,
--- a/tsconfig.test.json
+++ b/tsconfig.test.json
@ -1,16 +1,16 @@
 {
  "compilerOptions": {
    "lib": [
-      "es2019",
-      "dom",
-      "dom.iterable"
+      "ES2019",
+      "DOM",
+      "DOM.Iterable"
    ],
    "outDir": "./dist/",
    "sourceMap": true,
    "strict": true,
-    "module": "es2015",
-    "moduleResolution": "node",
-    "target": "es2015",
+    "module": "ESNext",
+    "moduleResolution": "NodeNext",
+    "target": "ESNext",
    "skipLibCheck": true,
    "esModuleInterop": true,
    "removeComments": true,
--- a/yarn.lock
+++ b/yarn.lock
Author	SHA1	Message	Date
Nes370	15bec7b115	Bugfixes Fixed safety detection on chan.sankakucomplex.com, and improved source parsing on Gelbooru v0.2.0 sites.	2026-10-17 19:29:36 -07:00
Nes370	fe0a543bf4	Update Bluesky.ts Added a filter to only visible elements to reduce clutter from posts in feed.	2026-10-13 23:46:36 -07:00
Nes370	1575a7a2f1	Update Bluesky.ts An attempt to order posts by their position in the page.	2026-10-13 23:14:14 -07:00
Nes370	38ad7b1156	Update Bluesky.ts Added support for Bluesky videos.	2026-10-12 20:50:39 -07:00
Nes370	f853f10c69	Update Danbooru.ts Added ATFBooru to host list.	2026-10-11 16:46:03 -07:00
Nes370	8004d0a183	Update Mastodon.ts Cleaned up code a bit.	2026-10-11 16:41:41 -07:00
Nes370	a6a4308276	Update Mastodon.ts Remove all references to video preview images.	2026-10-11 10:14:32 -07:00
Nes370	7f51092362	Update Mastodon.ts Removed thumbnail previews for videos.	2026-10-11 09:59:10 -07:00
Nes370	568660c6a9	Update Mastodon.ts Added video support.	2026-10-11 09:39:09 -07:00
Nes370	f9014fff03	Removed rating detection. Removed rating detection logic due to inconsistent behavior across domains.	2026-10-11 09:36:35 -07:00
Nes370	cecb0f0f30	Update Mastodon.ts Fixed terminology.	2026-10-11 07:27:26 -07:00
Nes370	0a69309627	Add Mastodon support. Added a scraper engine for Mastodon.	2026-10-11 07:04:15 -07:00
neobooru	0232ac2d8c	Add page referrer on Gelbooru	2026-08-19 00:39:26 +02:00
neobooru	6437c21d95	Bump version to 0.11.0	2026-07-31 02:02:09 +02:00
neobooru	9df26e0685	Fix summary table script	2026-07-31 02:01:32 +02:00
neobooru	fccd1d7cbb	Fix for New e621 Download Button and e6ai Domain Addition Co-authored-by: Kaias Blackwood <kaias.blackwood@gmail.com>	2026-07-31 02:01:22 +02:00
neobooru	eda941d889	feat: mark unsafe for R-18 posts on Pixiv Co-authored-by: Keo Cheung <keocheung@outlook.com>	2026-07-31 01:50:51 +02:00
neobooru	51be6dfdcd	Fix + bump version	2025-04-12 01:03:38 +02:00
neobooru	4692fe757d	Fix some things	2025-04-12 00:10:28 +02:00
Shae	832b3d286d	feat: add more domains	2025-04-11 22:43:56 +02:00
Kaias Blackwood	eeee42c389	Updated tag scraping to new e621 DOM	2025-04-11 22:39:35 +02:00
neobooru	ed9580ea58	Add basic bluesky support (image only)	2025-02-18 23:32:50 +01:00
neobooru	ad2d1fe585	Fix tests and some small issues	2024-12-08 18:03:32 +01:00
neobooru	cfbd9652ec	Fix crash when yande.re post source has no link	2024-07-11 23:06:54 +02:00
neobooru	593c4cef0d	yande.re hotlink protection workaround	2024-07-11 22:55:57 +02:00
neobooru	17c84b16db	UploadMode tweaks + add Nozomi + add x.com as twitter domain	2024-06-25 23:37:35 +02:00
neobooru	df9b1b2cfe	Add uploadMode to engines	2024-06-14 16:55:20 +02:00
neobooru	5253297809	Update danbooru tag category scraper	2024-01-09 23:19:10 +01:00
neobooru	6cd6bb1745	Add sensitive -> sketchy conversion for gelbooru	2024-01-01 20:15:43 +01:00
neobooru	56c3e78afd	Fix bug where exhentai full resolution picture would not be found	2023-12-01 23:00:38 +01:00
neobooru	fa6baa75b7	Fix and update tests and use pnpm instead of yarn	2023-11-05 15:09:49 +01:00
neobooru	18e9b57e12	Add Philomena video resolution test	2023-07-22 14:35:56 +02:00
GlobalGit3	7a12310bb4	Update Philomena.ts Fix resolution grabbing extra data, on video/animated images specifically. This is only a visual issue and not a problem for importing.	2023-07-22 11:54:23 +00:00
neobooru	b2937d3c94	Find PNG version of posts on konachan + test fixes	2023-07-05 13:58:23 +02:00
neobooru	c149992fb2	Bump version to 0.7.0	2023-07-05 13:11:28 +02:00
neobooru	da0593e432	Fix typescript definitions for test utilities	2023-07-05 13:07:29 +02:00
Neo	c9daac63b0	Merge pull request #15 from GlobalGit3/master Added Philomena Scraper	2023-07-05 11:06:07 +00:00
neobooru	5beea8cc69	Small fixes and add extra test	2023-07-05 13:04:41 +02:00
GlobalGit3	e208d69695	Updated Features & Added resolution. Resolution now supported. Feature list was updated to reflect both resolution and tag category being used now.	2023-07-03 16:43:39 -05:00
GlobalGit3	8d215729f3	Map and Arrays for prefixes Categorize tags and remove prefixes using Maps and Arrays. A quick write-up, but works for the time being to get an idea for what is wanted. Due to the data-tag-category's 'origin' being used for more than artist's names, prefixes are used to categorize the artist tag. There are also other prefixes such as 'editor' that are the same as 'artist'. This should be no problem as all artists' tags start with the 'artist' or 'editor' prefix. Suffixes only applies when removing a prefix, this will probably be changed later to apply without removing the prefix. It was coded to give an idea of wanting to change tagNames from "oc:originalcharacter" to "originalcharacter_(oc)".	2023-07-02 01:49:40 -05:00
neobooru	580e308f8f	Remove safety tags from list and add categories	2023-07-02 00:20:12 +02:00
GlobalGit3	4c0c790c0c	Attempt to 'prettier' code * Hopefully this is 'prettier'	2023-07-01 14:24:08 -05:00
GlobalGit3	3407ae2a8c	Update Philomena.ts * Removed useless check	2023-07-01 14:14:44 -05:00
GlobalGit3	b9ae6a1e8a	Update README.md Used the wrong readme, oopsies.	2023-07-01 08:32:35 -05:00
GlobalGit3	e10baedddc	Update Philomena.test.ts Fixed a small oversight. No worries with the old id.	2023-07-01 08:30:07 -05:00
GlobalGit3	8b89e7b527	Added Philomena * Philomena support	2023-07-01 08:17:53 -05:00
neobooru	4254c5f5d8	Bump version to 0.6.0	2023-06-19 21:31:19 +02:00
neobooru	52c48526af	tsconfig.json change which shouldn't break anything	2023-06-19 21:30:02 +02:00
neobooru	b67d024274	Fix broken engines and update tests where needed	2023-06-19 21:22:23 +02:00
neobooru	bf1e9d9e62	Add experimental e-hentai engine	2023-06-19 21:05:24 +02:00
neobooru	f89ab0b949	Upgrade dependencies and maybe fix tests	2023-02-11 00:27:39 +01:00
neobooru	8c12ad4d49	Fix ES module crap I hope	2023-01-15 16:42:30 +01:00
neobooru	1c9e0934d1	Fix zerochan issue	2022-10-22 13:30:13 +02:00
neobooru	be6111e166	Map sensitive as sketchy on danbooru + small fix	2022-10-22 00:50:51 +02:00
neobooru	de16edd143	Bump version	2022-08-12 01:31:16 +02:00
neobooru	535b358e7d	Build fixes	2022-08-12 01:27:33 +02:00
neobooru	75f5c201b4	Add support for multiple source fields + add tests	2022-08-10 14:01:17 +02:00
neobooru	73ffee047e	Set user who submitted the art as the artists on furaffinity	2022-08-10 14:00:27 +02:00
neobooru	d127ec755b	Fix broken tag names when logged into moebooru site	2022-08-09 18:08:18 +02:00
neobooru	3d602869ed	Fix jest puppeteer config	2022-08-09 00:41:19 +02:00
neobooru	a599710276	Add feature info to engines + project cleanup	2022-08-07 20:48:28 +02:00
neobooru	198da89bc4	Update Pixiv engine and tests	2022-08-07 15:24:04 +02:00
neobooru	e30e55116c	Minor version bump	2022-01-30 19:11:43 +01:00
neobooru	5bb418dae6	Add resolution scraping feature to various sites	2022-01-30 19:11:04 +01:00
neobooru	cbeea1c665	Fix bug where videos couldn't be uploaded from a direct link Possibly a firefox-only issue.	2022-01-30 19:10:12 +01:00
neobooru	5322d0158a	Security stuff	2022-01-26 20:32:26 +01:00
neobooru	6249bbd10c	Bump version	2022-01-26 20:22:58 +01:00
neobooru	754c31eb0b	Some tweaks + more tests	2021-11-23 18:24:18 +01:00
neobooru	905c36b2ff	AnimePictures parse resolution + more tests	2021-11-23 13:44:42 +01:00