Add feature info to engines + project cleanup

.eslintrc.cjs

@@ -0,0 +1,17 @@
+module.exports = {
+  env: {
+    browser: true,
+    commonjs: true,
+    es2021: true,
+    node: true,
+  },
+  extends: ["eslint:recommended", "plugin:@typescript-eslint/recommended"],
+  parser: "@typescript-eslint/parser",
+  parserOptions: {
+    ecmaVersion: "latest",
+  },
+  plugins: ["@typescript-eslint"],
+  rules: {
+    semi: ["error", "always"],
+  },
+};

README.md

@@ -1,3 +1,24 @@
 # neo-scraper
 
 Scrape a booru-like post object from the DOM.
+
+## Supported sites
+
+| Engine         | Sites                                                         | Features                                                       | Notes                                                         |
+| -------------- | ------------------------------------------------------------- | -------------------------------------------------------------- | ------------------------------------------------------------- |
+| animepictures  | anime-pictures.net                                            | content, resolution, tags, tag_category                        | Rating is assumed to be safe.                                 |
+| danbooru       | danbooru.donmai.us, safebooru.donmai.us                       | content, rating, resolution, tags, tag_category, source, notes |                                                               |
+| e621           | e621.net, e926.net                                            | content, rating, resolution, tags, tag_category, source        |                                                               |
+| furaffinity    | www.furaffinity.net                                           | content, rating, resolution, tags, tag_category                |                                                               |
+| gelbooru       | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts.                  |
+| inkbunny       | inkbunny.net                                                  | content, rating, resolution, tags                              |                                                               |
+| moebooru       | yande.re, konachan.com                                        | content, rating, resolution, tags, tag_category, source, notes |                                                               |
+| pixiv          | pixiv.net, www.pixiv.net                                      | content                                                        |                                                               |
+| reddit         | reddit.com, new.reddit.com, old.reddit.com                    | content                                                        |                                                               |
+| rule34us       | rule34.us                                                     | content, resolution, tags, tag_category                        | Rating is assumed to be unsafe.                               |
+| sankakucomplex | chan.sankakucomplex.com                                       | content, rating, tags, tag_category, notes                     |                                                               |
+| shimmie2       | rule34.paheal.net, rule34hentai.net                           | content, tags, source                                          | Rating is assumed to be unsafe.                               |
+| shuushuu       | e-shuushuu.net                                                | content, rating, resolution, tags, tag_category                | Rating is assumed to be safe.                                 |
+| twitter        | twitter.com, mobile.twitter.com                               | content                                                        |                                                               |
+| zerochan       | www.zerochan.net                                              | content, resolution, tags, tag_category                        | Rating is assumed to be safe.                                 |
+| fallback       |                                                               | content                                                        | Tries to find the largest image or video on the current page. |

package.json

@@ -1,7 +1,8 @@
 {
   "name": "neo-scraper",
-  "version": "0.5.4",
+  "version": "0.5.5",
   "description": "Scrape a booru-like post object from the DOM.",
+  "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.js",
   "scripts": {
@@ -10,7 +11,9 @@
     "test:debug": "cross-env HEADLESS=false jest --testTimeout=100000000",
     "build": "tsc",
     "watch": "tsc -w",
-    "prepare": "npm run build"
+    "prepare": "npm run build",
+    "lint": "eslint src scripts",
+    "summary-table": "npm run build && node --experimental-specifier-resolution=node --loader ts-node/esm scripts/summary-table.ts"
   },
   "author": "neobooru",
   "license": "MIT",
@@ -24,15 +27,23 @@
     "@types/jsdom": "^12.2.4",
     "@types/puppeteer": "^5.4.4",
     "@types/turndown": "^5.0.1",
+    "@typescript-eslint/eslint-plugin": "^5.32.0",
+    "@typescript-eslint/parser": "^5.32.0",
+    "eslint": "8.7.0",
+    "eslint-plugin-import": "^2.25.2",
+    "eslint-plugin-n": "^15.0.0",
+    "eslint-plugin-promise": "^6.0.0",
     "jest": "^26.6.3",
     "jest-puppeteer": "^6.0.0",
     "jsdom": "^15.2.1",
+    "prettier": "2.5.1",
     "puppeteer": "^13.1.2",
     "ts-jest": "^26.5.6",
+    "ts-node": "^10.9.1",
     "typescript": "^4.5.2"
   },
   "jest": {
-    "preset": "./jest.preset.js",
+    "preset": "./jest.preset.cjs",
     "transform": {
       "^.+\\.tsx?$": "ts-jest"
     },

scripts/summary-table.ts

@@ -0,0 +1,15 @@
+import NeoScraper from "../src/NeoScraper";
+import FallbackEngine from "../src/engines/Fallback";
+
+const engines = [...new NeoScraper().engines, new FallbackEngine()];
+const txt = ["| Engine | Sites | Features | Notes |", "|---|---|---|---|---|"];
+
+for (const engine of engines) {
+  txt.push(
+    `| ${engine.name} | ${engine.supportedHosts.join(", ")} | ${engine.features.join(", ")} | ${engine.notes.join(
+      " "
+    )} |`
+  );
+}
+
+console.log(txt.join("\n"));

src/NeoScraper.test.ts

@@ -1,12 +1,12 @@
 import NeoScraper from "./NeoScraper";
 
 test("create scraper with default engines", async () => {
-  const scraper = new NeoScraper();
+  new NeoScraper();
 });
 
 test("all default scrapers have unique names", () => {
   const scraper = new NeoScraper();
-  let alreadySeen: string[] = [];
+  const alreadySeen: string[] = [];
 
   for (const engine of scraper.engines) {
     if (alreadySeen.indexOf(engine.name) != -1) {

src/NeoScraper.ts

@@ -5,7 +5,7 @@ export default class NeoScraper {
   engines: ScrapeEngine[] = [];
   fallbackEngine: ScrapeEngine | null = null;
 
-  constructor(useDefaultEngines: boolean = true) {
+  constructor(useDefaultEngines = true) {
     if (useDefaultEngines) {
       this.engines.push(
         new engines.AnimePictures(),
@@ -28,12 +28,7 @@ export default class NeoScraper {
     }
   }
 
-  scrapeDocument(
+  scrapeDocument(document: Document, allowFallbackEngine = false, ignoreCanImport = false): ScrapeResults {
-    document: Document,
-    engines: string[] | null = null,
-    allowFallbackEngine: boolean = false,
-    ignoreCanImport: boolean = false
-  ): ScrapeResults {
     const res = new ScrapeResults();
 
     for (const engine of this.engines) {

src/ScrapeEngine.ts

@@ -3,8 +3,8 @@ import { TagCategory, SafetyRating } from "./BooruTypes";
 export type ContentType = "image" | "video";
 
 export class ScrapedPost {
-  contentUrl: string = "";
+  contentUrl = "";
-  pageUrl: string = "";
+  pageUrl = "";
   contentType: ContentType = "image";
   resolution: [number, number] | undefined;
   rating: SafetyRating = "safe";
@@ -30,7 +30,7 @@ export class ScrapedNote {
 
 export class ScrapeResult {
   engine: string;
-  description: string = "";
+  description = "";
   posts: ScrapedPost[] = [];
 
   constructor(engine: string) {
@@ -50,9 +50,9 @@ export class ScrapeResults {
   results: ScrapeResult[] = [];
 
   get posts(): ScrapedPost[] {
-    var posts: ScrapedPost[] = [];
+    let posts: ScrapedPost[] = [];
 
-    for (var res of this.results) {
+    for (const res of this.results) {
       posts = posts.concat(res.posts);
     }
 
@@ -60,15 +60,28 @@ export class ScrapeResults {
   }
 }
 
+export type ScrapeEngineFeature = "content" | "rating" | "resolution" | "tags" | "tag_category" | "source" | "notes";
+
 export interface ScrapeEngine {
   name: string;
+  features: ScrapeEngineFeature[];
+  notes: string[];
+  supportedHosts: string[];
+
   canImport(url: Location): boolean;
   scrapeDocument(document: Document): ScrapeResult;
 }
 
 export abstract class ScrapeEngineBase implements ScrapeEngine {
   abstract name: string;
-  abstract canImport(url: Location): boolean;
+  abstract features: ScrapeEngineFeature[];
+  abstract notes: string[];
+  abstract supportedHosts: string[];
+
+  canImport(url: Location): boolean {
+    return this.supportedHosts.indexOf(url.host) != -1;
+  }
+
   abstract scrapeDocument(document: Document): ScrapeResult;
 
   protected log(str: string) {

src/Utility.ts

@@ -50,7 +50,7 @@ export function getUrlExtension(url: string) {
  * @param url
  * @returns "image" | "video"
  */
-export function guessContentType(url: string): ContentType {
+export function guessContentType(url: string | undefined): ContentType {
   if (url != undefined) {
     const ext = getUrlExtension(url);
     if (videoExtensions.indexOf(ext) != -1) return "video";
@@ -109,7 +109,7 @@ export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement
       return undefined;
     }
     const num = parseInt(value);
-    if (num == NaN) {
+    if (isNaN(num)) {
       logFail(`parseInt on '${value}' returned NaN.`);
     }
     return num;
@@ -132,10 +132,10 @@ export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement
     text = htmlNoteToMarkdownNote(text);
   }
 
-  let x = getInt(data, "x");
+  const x = getInt(data, "x");
-  let y = getInt(data, "y");
+  const y = getInt(data, "y");
-  let w = getInt(data, "width");
+  const w = getInt(data, "width");
-  let h = getInt(data, "height");
+  const h = getInt(data, "height");
 
   if (!text || !x || !y || !w || !h) {
     logFail("A required data field could not be parsed.");
@@ -159,7 +159,7 @@ export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement
  * @returns
  */
 export function createNotesFromMoebooruBoxes(document: Document, boxSize: [number, number]) {
-  let notes: ScrapedNote[] = [];
+  const notes: ScrapedNote[] = [];
   const noteBoxes = Array.from(document.querySelectorAll(".note-box")).map((x) => x as HTMLLIElement);
   const noteBodies = Array.from(document.querySelectorAll(".note-body")).map((x) => x as HTMLLIElement);
 

src/engines/AnimePictures.ts

@@ -1,19 +1,18 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine";
 import { TagCategory } from "../BooruTypes";
 import { guessContentType, parseResolutionString } from "../Utility";
 
-export default class AnimePictures implements ScrapeEngine {
+export default class AnimePictures extends ScrapeEngineBase {
   name = "animepictures";
-
+  features: ScrapeEngineFeature[] = ["content", "resolution", "tags", "tag_category"];
-  canImport(url: Location): boolean {
+  notes = ["Rating is assumed to be safe."];
-    return url.host == "anime-pictures.net";
+  supportedHosts = ["anime-pictures.net"];
-  }
 
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
-    let post = new ScrapedPost();
+    const post = new ScrapedPost();
     post.pageUrl = document.location.href;
-    post.source = document.location.href;
+    
 
     // Set image url
     const originalImageElements = Array.from(document.getElementsByClassName("download_icon"));
@@ -40,7 +39,7 @@ export default class AnimePictures implements ScrapeEngine {
     // Set tags
     const tagElements = Array.from(document.querySelectorAll<HTMLAnchorElement>("ul.tags > li > a"));
     for (const tagElement of tagElements) {
-      let tagName = tagElement.innerText;
+      const tagName = tagElement.innerText;
 
       let category: TagCategory | undefined;
       for (const className of tagElement.classList) {
@@ -58,7 +57,7 @@ export default class AnimePictures implements ScrapeEngine {
       }
 
       if (tagName) {
-        let tag = new ScrapedTag(tagName, category);
+        const tag = new ScrapedTag(tagName, category);
         post.tags.push(tag);
       }
     }

src/engines/Danbooru.ts

@@ -1,17 +1,16 @@
-import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapedNote } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine";
 import { TagCategory } from "../BooruTypes";
 import { createNoteFromDanbooruArticle, guessContentType } from "../Utility";
 
 export default class Danbooru extends ScrapeEngineBase {
   name = "danbooru";
-
+  features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
-  canImport(url: Location): boolean {
+  notes = [];
-    return url.host == "danbooru.donmai.us" || url.host == "safebooru.donmai.us";
+  supportedHosts = ["danbooru.donmai.us", "safebooru.donmai.us"];
-  }
 
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
-    let post = new ScrapedPost();
+    const post = new ScrapedPost();
     post.pageUrl = document.location.href;
 
     // Set image url
@@ -92,7 +91,7 @@ export default class Danbooru extends ScrapeEngineBase {
       }
 
       if (tagName) {
-        let tag = new ScrapedTag(tagName, category);
+        const tag = new ScrapedTag(tagName, category);
         post.tags.push(tag);
       }
     }

src/engines/Fallback.ts

@@ -1,4 +1,4 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ContentType } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ContentType, ScrapeEngineFeature } from "../ScrapeEngine";
 import { guessContentType } from "../Utility";
 
 class MediaElement {
@@ -9,22 +9,21 @@ class MediaElement {
   ) {}
 }
 
-export default class Fallback implements ScrapeEngine {
+export default class Fallback extends ScrapeEngineBase {
   name = "fallback";
-
+  features: ScrapeEngineFeature[] = ["content"];
-  canImport(location: Location): boolean {
+  notes = ["Tries to find the largest image or video on the current page."];
-    return false;
+  supportedHosts = [];
-  }
 
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
 
     // Check if the current page is a link to a video file.
     // If it is then use the document.location.href as media source and immediately return.
     // This is needed because firefox does not add a <source> to a <video> element
     // when going to a direct link. It looks like chromium-based browsers do not have this issue.
     if (guessContentType(document.location.href) == "video") {
-      let post = new ScrapedPost();
+      const post = new ScrapedPost();
       post.pageUrl = document.location.href;
       post.contentUrl = document.location.href;
       post.contentType = "video";
@@ -50,7 +49,7 @@ export default class Fallback implements ScrapeEngine {
     );
 
     if (largestMediaElement) {
-      let post = new ScrapedPost();
+      const post = new ScrapedPost();
       post.pageUrl = document.location.href;
       post.contentUrl = largestMediaElement.contentUrl;
       post.contentType = largestMediaElement.contentType;

src/engines/FurAffinity.ts

@@ -1,17 +1,15 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine";
 import { guessContentType, parseResolutionString } from "../Utility";
 
-export default class FurAffinity implements ScrapeEngine {
+export default class FurAffinity extends ScrapeEngineBase {
   name = "furaffinity";
-
+  features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category"];
-  canImport(url: Location): boolean {
+  notes = [];
-    return url.host == "www.furaffinity.net";
+  supportedHosts = ["www.furaffinity.net"];
-  }
 
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
-
+    const post = new ScrapedPost();
-    let post = new ScrapedPost();
     post.pageUrl = document.location.href;
 
     // Set image url
@@ -53,9 +51,6 @@ export default class FurAffinity implements ScrapeEngine {
       .map((x) => (x as HTMLSpanElement).innerText)
       .map((x) => new ScrapedTag(x));
 
-    // Set source
-    post.source = document.location.href;
-
     result.tryAddPost(post);
 
     return result;

src/engines/Gelbooru.ts

@@ -1,4 +1,4 @@
-import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine";
 import { TagCategory } from "../BooruTypes";
 import { CategoryMap } from "./Common";
 import { createNoteFromDanbooruArticle, guessContentType, parseResolutionString } from "../Utility";
@@ -11,6 +11,9 @@ enum Version {
 
 export default class Gelbooru extends ScrapeEngineBase {
   name = "gelbooru";
+  features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
+  notes = ["Supported features might vary between hosts."];
+  supportedHosts = ["safebooru.org", "gelbooru.com", "rule34.xxx", "tbib.org", "xbooru.com"];
 
   private readonly classNameToCategoryMap: CategoryMap = {
     "tag-type-copyright": "copyright",
@@ -19,18 +22,8 @@ export default class Gelbooru extends ScrapeEngineBase {
     "tag-type-metadata": "meta",
   };
 
-  canImport(url: Location): boolean {
-    return (
-      url.host == "safebooru.org" ||
-      url.host == "gelbooru.com" ||
-      url.host == "rule34.xxx" ||
-      url.host == "tbib.org" ||
-      url.host == "xbooru.com"
-    );
-  }
-
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
     let version: Version | null = null;
     switch (document.location.host) {
       case "safebooru.org":
@@ -49,7 +42,7 @@ export default class Gelbooru extends ScrapeEngineBase {
 
     this.log("Guessed version: " + version);
 
-    let post = new ScrapedPost();
+    const post = new ScrapedPost();
     post.pageUrl = document.location.href;
 
     // Set image url
@@ -154,7 +147,7 @@ export default class Gelbooru extends ScrapeEngineBase {
           }
         }
 
-        let tag = new ScrapedTag(tagName, category);
+        const tag = new ScrapedTag(tagName, category);
         post.tags.push(tag);
       }
     }

src/engines/Inkbunny.ts

@@ -1,16 +1,14 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine";
 
-export default class Inkbunny implements ScrapeEngine {
+export default class Inkbunny extends ScrapeEngineBase {
   name = "inkbunny";
-
+  features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags"];
-  canImport(url: Location): boolean {
+  notes = [];
-    return url.host == "inkbunny.net";
+  supportedHosts = ["inkbunny.net"];
-  }
 
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
-
+    const post = new ScrapedPost();
-    let post = new ScrapedPost();
     post.pageUrl = document.location.href;
 
     // Set image url
@@ -36,7 +34,7 @@ export default class Inkbunny implements ScrapeEngine {
     // Set post resolution
     const width = parseInt((<HTMLInputElement>document.getElementById("submission-origwidth"))?.value);
     const height = parseInt((<HTMLInputElement>document.getElementById("submission-origheight"))?.value);
-    if (width != NaN && height != NaN) {
+    if (!isNaN(width) && !isNaN(height)) {
       post.resolution = [width, height];
     }
 
@@ -46,7 +44,7 @@ export default class Inkbunny implements ScrapeEngine {
     const ratingEls = spanEls.map((x) => x as HTMLDivElement).filter((x) => x.innerText.indexOf("Rating:") != -1);
 
     if (ratingEls.length > 0) {
-      const matches = ratingEls[0].parentElement!.innerText.match(ratingExp);
+      const matches = ratingEls[0].parentElement?.innerText.match(ratingExp);
       if (matches && matches.length > 0) {
         switch (matches[1].toLowerCase()) {
           case "general":
@@ -68,7 +66,7 @@ export default class Inkbunny implements ScrapeEngine {
       const keywords = keywordsContent.split(",").map((x) => x.trim());
 
       for (const keyword of keywords) {
-        let tag = new ScrapedTag(keyword);
+        const tag = new ScrapedTag(keyword);
         post.tags.push(tag);
       }
     }

src/engines/Moebooru.ts

@@ -1,17 +1,16 @@
-import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapedNote } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine";
 import { TagCategory } from "../BooruTypes";
 import { createNotesFromMoebooruBoxes, guessContentType, parseResolutionString } from "../Utility";
 
 export default class Moebooru extends ScrapeEngineBase {
   name = "moebooru";
-
+  features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
-  canImport(url: Location): boolean {
+  notes = [];
-    return url.host == "yande.re" || url.host == "konachan.com";
+  supportedHosts = ["yande.re", "konachan.com"];
-  }
 
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
-    let post = new ScrapedPost();
+    const post = new ScrapedPost();
     post.pageUrl = document.location.href;
 
     // Set image url
@@ -64,7 +63,7 @@ export default class Moebooru extends ScrapeEngineBase {
 
     for (const el of tagElements) {
       let tagName: string;
-      let tagNameElements = el.getElementsByTagName("a");
+      const tagNameElements = el.getElementsByTagName("a");
 
       if (tagNameElements.length > 1) {
         // First <a> is "?"
@@ -96,7 +95,7 @@ export default class Moebooru extends ScrapeEngineBase {
         }
       }
 
-      let tag = new ScrapedTag(tagName, category);
+      const tag = new ScrapedTag(tagName, category);
       post.tags.push(tag);
     }
 

src/engines/Pixiv.ts

@@ -1,4 +1,4 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapeEngineFeature } from "../ScrapeEngine";
 import { guessContentType } from "../Utility";
 
 export function upgradeUrlToOriginalQuality(imgLink: string) {
@@ -21,17 +21,15 @@ export function upgradeUrlToOriginalQuality(imgLink: string) {
   return imgLink;
 }
 
-export default class Pixiv implements ScrapeEngine {
+export default class Pixiv extends ScrapeEngineBase {
   name = "pixiv";
-
+  features: ScrapeEngineFeature[] = ["content"];
-  canImport(url: Location): boolean {
+  notes = [];
-    return url.host == "pixiv.net" || url.host == "www.pixiv.net";
+  supportedHosts = ["pixiv.net", "www.pixiv.net"];
-  }
 
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
-
+    const imgLinks = [] as string[];
-    let imgLinks = [] as string[];
 
     // Query for when the user is on the desktop site, is logged in, and has optionally clicked on "See all".
     // If the user has not clicked on "See all" then this will only grab the first post.
@@ -49,6 +47,7 @@ export default class Pixiv implements ScrapeEngine {
       // This only works on pages with multiple posts. The user can optionally be logged in.
       imgLinks.push(
         ...Array.from(document.querySelectorAll<HTMLImageElement>(".manga-page > a > img[data-big]")).map(
+          // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
           (x) => x.getAttribute("data-big")!
         )
       );
@@ -63,7 +62,7 @@ export default class Pixiv implements ScrapeEngine {
     }
 
     for (const imgLink of imgLinks) {
-      let post = new ScrapedPost();
+      const post = new ScrapedPost();
       post.pageUrl = document.location.href;
       post.contentUrl = upgradeUrlToOriginalQuality(imgLink);
       post.contentType = guessContentType(post.contentUrl);

src/engines/Reddit.ts

@@ -1,16 +1,15 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapeEngineFeature } from "../ScrapeEngine";
 import { guessContentType } from "../Utility";
 
-export default class Reddit implements ScrapeEngine {
+export default class Reddit extends ScrapeEngineBase {
   name = "reddit";
-
+  features: ScrapeEngineFeature[] = ["content"];
-  canImport(url: Location): boolean {
+  notes = [];
-    return url.host.endsWith("reddit.com");
+  supportedHosts = ["reddit.com", "new.reddit.com", "old.reddit.com"];
-  }
 
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
-    let post = new ScrapedPost();
+    const post = new ScrapedPost();
     post.pageUrl = document.location.href;
 
     // Set image url

src/engines/Rule34us.ts

@@ -1,13 +1,16 @@
 // Technically speaking rule34.us also uses gelbooru, albeit a (heavily?) modified one.
 // Because pretty much no scraper code can be shared with other gelbooru-likes it makes more sense to create a new engine for rule34.us.
 
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine";
 import { TagCategory } from "../BooruTypes";
 import { CategoryMap } from "./Common";
 import { guessContentType, parseResolutionString } from "../Utility";
 
-export default class Gelbooru implements ScrapeEngine {
+export default class Gelbooru extends ScrapeEngineBase {
   name = "rule34us";
+  features: ScrapeEngineFeature[] = ["content", "resolution", "tags", "tag_category"];
+  notes = ["Rating is assumed to be unsafe."];
+  supportedHosts = ["rule34.us"];
 
   private readonly classNameToCategoryMap: CategoryMap = {
     "character-tag": "character",
@@ -16,14 +19,9 @@ export default class Gelbooru implements ScrapeEngine {
     "metadata-tag": "meta",
   };
 
-  canImport(url: Location): boolean {
-    return url.host == "rule34.us";
-  }
-
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
-
+    const post = new ScrapedPost();
-    let post = new ScrapedPost();
     post.pageUrl = document.location.href;
 
     // Set image url

src/engines/SankakuComplex.ts

@@ -1,17 +1,16 @@
-import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapedNote } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine";
 import { TagCategory } from "../BooruTypes";
 import { createNotesFromMoebooruBoxes, guessContentType } from "../Utility";
 
 export default class SankakuComplex extends ScrapeEngineBase {
   name = "sankakucomplex";
-
+  features: ScrapeEngineFeature[] = ["content", "rating", "tags", "tag_category", "notes"];
-  canImport(url: Location): boolean {
+  notes = [];
-    return url.host == "chan.sankakucomplex.com";
+  supportedHosts = ["chan.sankakucomplex.com"];
-  }
 
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
-    let post = new ScrapedPost();
+    const post = new ScrapedPost();
     post.pageUrl = document.location.href;
 
     // Set image url
@@ -76,7 +75,7 @@ export default class SankakuComplex extends ScrapeEngineBase {
       }
 
       if (tagName) {
-        let tag = new ScrapedTag(tagName, category);
+        const tag = new ScrapedTag(tagName, category);
         post.tags.push(tag);
       }
     }

src/engines/Shimmie2.ts

@@ -1,16 +1,15 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine";
 import { guessContentType } from "../Utility";
 
-export default class Shimmie2 implements ScrapeEngine {
+export default class Shimmie2 extends ScrapeEngineBase {
   name = "shimmie2";
-
+  features: ScrapeEngineFeature[] = ["content", "tags", "source"];
-  canImport(url: Location): boolean {
+  notes = ["Rating is assumed to be unsafe."];
-    return url.host == "rule34.paheal.net" || url.host == "rule34hentai.net";
+  supportedHosts = ["rule34.paheal.net", "rule34hentai.net"];
-  }
 
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
-    let post = new ScrapedPost();
+    const post = new ScrapedPost();
     post.pageUrl = document.location.href;
 
     // Set image url

src/engines/Shuushuu.ts

@@ -1,17 +1,16 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine";
 import { TagCategory } from "../BooruTypes";
 import { guessContentType, parseResolutionString } from "../Utility";
 
-export default class Shuushuu implements ScrapeEngine {
+export default class Shuushuu extends ScrapeEngineBase {
   name = "shuushuu";
-
+  features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category"];
-  canImport(url: Location): boolean {
+  notes = ["Rating is assumed to be safe."];
-    return url.host == "e-shuushuu.net";
+  supportedHosts = ["e-shuushuu.net"];
-  }
 
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
-    let post = new ScrapedPost();
+    const post = new ScrapedPost();
     post.pageUrl = document.location.href;
 
     // Set image url

src/engines/Twitter.ts

@@ -1,30 +1,29 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapeEngineFeature } from "../ScrapeEngine";
 
 export function getOriginalImageUrl(image: HTMLImageElement) {
   // if the link includes a resolution, change it to orig:
   // https://pbs.twimg.com/media/stuff?format=jpg&name=900x900
   // -> https://pbs.twimg.com/media/stuff?format=jpg&name=orig
-  let img_url = new URL((image as HTMLImageElement).src);
+  const img_url = new URL((image as HTMLImageElement).src);
   if (img_url.searchParams.has("name") && img_url.searchParams.get("name") != "orig") {
     img_url.searchParams.set("name", "orig");
   }
   return img_url;
 }
 
-export default class Twitter implements ScrapeEngine {
+export default class Twitter extends ScrapeEngineBase {
   name = "twitter";
-
+  features: ScrapeEngineFeature[] = ["content"];
-  canImport(url: Location): boolean {
+  notes = [];
-    return url.host == "twitter.com" || url.host == "mobile.twitter.com";
+  supportedHosts = ["twitter.com", "mobile.twitter.com"];
-  }
 
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
 
     // first look for a focused image on desktop:
     const center_img = document.querySelector("div[role=dialog] div[role=dialog] div[aria-label=Image] img");
     if (center_img) {
-      let post = new ScrapedPost();
+      const post = new ScrapedPost();
       post.pageUrl = document.location.href;
       post.contentUrl = getOriginalImageUrl(center_img as HTMLImageElement).href;
       result.tryAddPost(post);
@@ -34,7 +33,7 @@ export default class Twitter implements ScrapeEngine {
     const imgs = [...document.querySelectorAll("article a[role='link'] div[data-testid='tweetPhoto'] img")];
 
     for (const img of imgs) {
-      let post = new ScrapedPost();
+      const post = new ScrapedPost();
       post.pageUrl = document.location.href;
       post.contentUrl = getOriginalImageUrl(img as HTMLImageElement).href;
       result.tryAddPost(post);

src/engines/Zerochan.ts

@@ -1,17 +1,16 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine";
 import { TagCategory } from "../BooruTypes";
 import { guessContentType, parseResolutionString } from "../Utility";
 
-export default class Zerochan implements ScrapeEngine {
+export default class Zerochan extends ScrapeEngineBase {
   name = "zerochan";
-
+  features: ScrapeEngineFeature[] = ["content", "resolution", "tags", "tag_category"];
-  canImport(url: Location): boolean {
+  notes = ["Rating is assumed to be safe."];
-    return url.host.endsWith("zerochan.net");
+  supportedHosts = ["www.zerochan.net"];
-  }
 
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
-    let post = new ScrapedPost();
+    const post = new ScrapedPost();
     post.pageUrl = document.location.href;
 
     // Set image url
@@ -67,12 +66,12 @@ export default class Zerochan implements ScrapeEngine {
           category = "artist";
           break;
         case "source":
-          // Skip these type of tags, because they are not 'booru-like'
+          // Skip these type of tags, because they are not 'booru-like'.
           continue;
       }
 
       if (tagName) {
-        let tag = new ScrapedTag(tagName, category);
+        const tag = new ScrapedTag(tagName, category);
         post.tags.push(tag);
       }
     }

src/engines/e621.ts

@@ -1,17 +1,15 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine";
 import { TagCategory } from "../BooruTypes";
 
-export default class e621 implements ScrapeEngine {
+export default class e621 extends ScrapeEngineBase {
   name = "e621";
-
+  features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source"];
-  canImport(url: Location): boolean {
+  notes = [];
-    return url.host == "e621.net" || url.host == "e926.net";
+  supportedHosts = ["e621.net", "e926.net"];
-  }
 
   scrapeDocument(document: Document): ScrapeResult {
-    let result = new ScrapeResult(this.name);
+    const result = new ScrapeResult(this.name);
-
+    const post = new ScrapedPost();
-    let post = new ScrapedPost();
     post.pageUrl = document.location.href;
 
     // Set image url
@@ -40,7 +38,7 @@ export default class e621 implements ScrapeEngine {
     // Set resolution
     const width = parseInt((<HTMLSpanElement>document.querySelector("span[itemprop='width']"))?.innerText);
     const height = parseInt((<HTMLSpanElement>document.querySelector("span[itemprop='height']"))?.innerText);
-    if (width != NaN && height != NaN) {
+    if (!isNaN(width) && !isNaN(height)) {
       post.resolution = [width, height];
     }
 
@@ -77,7 +75,7 @@ export default class e621 implements ScrapeEngine {
       }
 
       if (tagName) {
-        let tag = new ScrapedTag(tagName, category);
+        const tag = new ScrapedTag(tagName, category);
         post.tags.push(tag);
       }
     }

src/index.ts

@@ -1,7 +1,7 @@
 export { default as NeoScraper } from "./NeoScraper";
 export * from "./ScrapeEngine";
 
-import * as Engines from "./engines";
+import * as Engines from "./ScrapeEngine";
 export { Engines };
 
 import * as BooruTypes from "./BooruTypes";

src/test/jest.d.ts

@@ -5,9 +5,9 @@ declare global {
   namespace jest {
     interface Matchers<R> {
       // The `= undefined` part is incorrect according to VSCode, but it is actually required.
-      toHaveTag(tagName: string, tagCategory: TagCategory | undefined = undefined): CustomMatcherResult;
+      toHaveTag(tagName: string, tagCategory: TagCategory | undefined): R;
-      toHaveNote(noteText: string): CustomMatcherResult;
+      toHaveNote(noteText: string): R;
-      toHaveResolution(resolution: [number, number]): CustomMatcherResult;
+      toHaveResolution(resolution: [number, number]): R;
     }
   }
 }

src/test/jest.setup.ts

@@ -19,7 +19,7 @@ expect.extend({
     tagName: string,
     tagCategory: TagCategory | undefined = undefined
   ): jest.CustomMatcherResult {
-    let pass: boolean = false;
+    let pass = false;
     const message = () => (pass ? "" : `Post does not have the '${tagName}' tag, or the category is incorrect.`);
 
     if (received instanceof ScrapedPost) {
@@ -35,7 +35,7 @@ expect.extend({
     return { pass, message };
   },
   toHaveNote(received: any, noteText: string): jest.CustomMatcherResult {
-    let pass: boolean = false;
+    let pass = false;
     const message = () => (pass ? "" : `Post does not have a note with '${noteText}'.`);
 
     if (received instanceof ScrapedPost) {
@@ -44,7 +44,7 @@ expect.extend({
     return { pass, message };
   },
   toHaveResolution(received: any, resolution: [number, number]): jest.CustomMatcherResult {
-    let pass: boolean = false;
+    let pass = false;
     const message = () => (pass ? "" : `Expected: ${resolution}\nReceived: ${received?.resolution}`);
 
     if (received instanceof ScrapedPost && received.resolution?.length == 2) {

tsconfig.json

@@ -1,16 +1,16 @@
 {
   "compilerOptions": {
     "lib": [
-      "es2019",
+      "ES2019",
-      "dom",
+      "DOM",
-      "dom.iterable"
+      "DOM.Iterable"
     ],
     "outDir": "./dist/",
     "sourceMap": true,
     "strict": true,
-    "module": "es2015",
+    "module": "ESNext",
-    "moduleResolution": "node",
+    "moduleResolution": "Node",
-    "target": "es2015",
+    "target": "ESNext",
     "skipLibCheck": true,
     "esModuleInterop": true,
     "removeComments": true,