Add rule34.us engine

package.json

@@ -1,12 +1,13 @@
 {
   "name": "neo-scraper",
-  "version": "0.4.5",
+  "version": "0.4.6",
   "description": "Scrape a booru-like post object from the DOM.",
   "main": "dist/index.js",
   "types": "dist/index.d.js",
   "scripts": {
     "test": "jest",
     "build": "tsc",
+    "watch": "tsc -w",
     "prepare": "npm run build"
   },
   "author": "neobooru",

src/NeoScraper.ts

@@ -17,6 +17,7 @@ export default class NeoScraper {
         new engines.Moebooru(),
         new engines.Pixiv(),
         new engines.Reddit(),
+        new engines.Rule34us(),
         new engines.SankakuComplex(),
         new engines.Shimmie2(),
         new engines.Shuushuu(),

src/engines/Common.ts

@@ -0,0 +1,3 @@
+import { TagCategory } from "../BooruTypes";
+
+export type CategoryMap = Record<string, TagCategory>;

src/engines/Gelbooru.ts

@@ -1,5 +1,6 @@
 import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
 import { TagCategory } from "../BooruTypes";
+import { CategoryMap } from "./Common";
 import { guessContentType } from "../Utility";
 
 // Small hack to avoid screwing with semver
@@ -8,11 +9,16 @@ enum Version {
   v025, // 0.2.5
 }
 
-type CategoryMap = Record<string, TagCategory>;
-
 export default class Gelbooru implements ScrapeEngine {
   name = "gelbooru";
 
+  private readonly classNameToCategoryMap: CategoryMap = {
+    "tag-type-copyright": "copyright",
+    "tag-type-character": "character",
+    "tag-type-artist": "artist",
+    "tag-type-metadata": "meta",
+  };
+
   canImport(url: Location): boolean {
     return (
       url.host == "safebooru.org" || url.host == "gelbooru.com" || url.host == "rule34.xxx" || url.host == "tbib.org"
@@ -135,17 +141,10 @@ export default class Gelbooru implements ScrapeEngine {
 
         let category: TagCategory | undefined;
 
-        const classNameToCategoryMap: CategoryMap = {
-          "tag-type-copyright": "copyright",
-          "tag-type-character": "character",
-          "tag-type-artist": "artist",
-          "tag-type-metadata": "meta",
-        };
-
         // Loop over all classes, because some websites use multiple. E.g. `class="tag tag-type-copyright"`
         for (const className of el.classList) {
-          if (classNameToCategoryMap[className]) {
+          if (this.classNameToCategoryMap[className]) {
-            category = classNameToCategoryMap[className];
+            category = this.classNameToCategoryMap[className];
             break;
           }
         }

src/engines/Rule34us.ts

@@ -0,0 +1,74 @@
+// Technically speaking rule34.us also uses gelbooru, albeit a (heavily?) modified one.
+// Because pretty much no scraper code can be shared with other gelbooru-likes it makes more sense to create a new engine for rule34.us.
+
+import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
+import { TagCategory } from "../BooruTypes";
+import { CategoryMap } from "./Common";
+import { guessContentType } from "../Utility";
+
+export default class Gelbooru implements ScrapeEngine {
+  name = "rule34us";
+
+  private readonly classNameToCategoryMap: CategoryMap = {
+    "character-tag": "character",
+    "artist-tag": "character",
+    "copyright-tag": "copyright",
+    "metadata-tag": "meta",
+  };
+
+  canImport(url: Location): boolean {
+    return url.host == "rule34.us";
+  }
+
+  scrapeDocument(document: Document): ScrapeResult {
+    let result = new ScrapeResult(this.name);
+
+    let post = new ScrapedPost();
+    post.pageUrl = document.location.href;
+
+    // Set image url
+    const origImgEls = Array.from(document.querySelectorAll("ul > a"))
+      .map((x) => x as HTMLAnchorElement)
+      .filter((x) => x.innerText == "Original");
+
+    if (origImgEls.length > 0) {
+      post.contentUrl = origImgEls[0].href;
+    } else {
+      return result;
+    }
+
+    // Set content type
+    post.contentType = guessContentType(post.contentUrl);
+
+    // Patch contentUrl if contentType == "video"
+    // This is because the "Original" button does link to "302 Found" for videos. Idk why.
+    // Shit's fucked, there are multiple workarounds for that and this is one of them.
+    if (post.contentType == "video") {
+      post.contentUrl = post.contentUrl.replace("img2.rule34", "video.rule34");
+    }
+
+    // This is where we would set the rating and source,
+    // but rule34.us doesn't display those as far as I can see.
+    post.rating = "unsafe"; // Lucky guess.
+
+    // Set tags
+    const possibleTagEls = Array.from(document.querySelectorAll(".container > ul > li")).map((x) => x as HTMLLIElement);
+
+    for (const el of possibleTagEls) {
+      // Idk why booru sites can't have consistent a HTML DOM.
+      // Not all elements inside `possibleTagEls` are actually tags,
+      // only elements that have a `a` tag as a direct child are valid tags.
+      // E.g. the 'Edit' button also has the `general-tag` class, but it certainly is not a tag.
+
+      const anchor = el.firstElementChild as HTMLAnchorElement | null;
+      if (anchor?.tagName == "A") {
+        const name = anchor.innerText;
+        const category = this.classNameToCategoryMap[el.className] as TagCategory | undefined;
+        post.tags.push(new ScrapedTag(name, category));
+      }
+    }
+
+    result.tryAddPost(post);
+    return result;
+  }
+}

src/engines/index.ts

@@ -8,6 +8,7 @@ export { default as Inkbunny } from "./Inkbunny";
 export { default as Moebooru } from "./Moebooru";
 export { default as Pixiv } from "./Pixiv";
 export { default as Reddit } from "./Reddit";
+export { default as Rule34us } from "./Rule34us";
 export { default as SankakuComplex } from "./SankakuComplex";
 export { default as Shimmie2 } from "./Shimmie2";
 export { default as Shuushuu } from "./Shuushuu";