UploadMode tweaks + add Nozomi + add x.com as twitter domain

package.json

@@ -1,6 +1,6 @@
 {
   "name": "neo-scraper",
-  "version": "0.8.0",
+  "version": "0.9.0",
   "description": "Scrape a booru-like post object from the DOM.",
   "type": "module",
   "main": "dist/index.js",
@@ -39,7 +39,7 @@
     "jest-ts-webcompat-resolver": "^1.0.0",
     "jsdom": "^15.2.1",
     "prettier": "2.5.1",
-    "puppeteer": "^21.5.0",
+    "puppeteer": "^22.12.0",
     "ts-jest": "^29.1.1",
     "ts-node": "^10.9.1",
     "typescript": "^4.9.5"
@@ -52,7 +52,12 @@
     "preset": "./jest.preset.cjs",
     "resolver": "jest-ts-webcompat-resolver",
     "transform": {
-      "^.+\\.tsx?$": "ts-jest"
+      "^.+\\.tsx?$": [
+        "ts-jest",
+        {
+          "tsconfig": "tsconfig.test.json"
+        }
+      ]
     },
     "moduleFileExtensions": [
       "ts",
@@ -62,11 +67,6 @@
     "testRegex": "^.+\\.test\\.ts$",
     "setupFilesAfterEnv": [
       "./src/test/jest.setup.ts"
-    ],
-    "globals": {
-      "ts-jest": {
-        "tsconfig": "tsconfig.test.json"
-      }
-    }
+    ]
   }
 }

src/NeoScraper.ts

@@ -16,6 +16,7 @@ export default class NeoScraper {
         new engines.Gelbooru(),
         new engines.Inkbunny(),
         new engines.Moebooru(),
+        new engines.Nozomi(),
         new engines.Philomena(),
         new engines.Pixiv(),
         new engines.Reddit(),

src/ScrapeEngine.ts

@@ -5,6 +5,7 @@ export type ContentType = "image" | "video";
 export type UploadMode = "content" | "url";
 
 export class ScrapedPost {
+  name: string | undefined = undefined;
   contentUrl = "";
   pageUrl = "";
   contentType: ContentType = "image";
@@ -17,6 +18,7 @@ export class ScrapedPost {
   referrer: string | undefined;
   extraContentUrl: string | undefined;
   cookies: Cookies | undefined;
+  uploadMode: UploadMode = "url";
 }
 
 export class ScrapedTag {
@@ -30,7 +32,7 @@ export class ScrapedTag {
 }
 
 export class ScrapedNote {
-  constructor(public text: string, public polygon: number[][]) {}
+  constructor(public text: string, public polygon: number[][]) { }
 }
 
 export class ScrapeResult {
@@ -82,8 +84,6 @@ export interface ScrapeEngine {
   notes: string[];
   supportedHosts: string[];
 
-  get uploadMode(): UploadMode;
-
   canImport(url: Location): boolean;
   scrapeDocument(document: Document): ScrapeResult;
 }
@@ -94,10 +94,6 @@ export abstract class ScrapeEngineBase implements ScrapeEngine {
   abstract notes: string[];
   abstract supportedHosts: string[];
 
-  get uploadMode(): UploadMode {
-    return "url";
-  }
-
   canImport(url: Location): boolean {
     return this.supportedHosts.indexOf(url.host) != -1;
   }

src/engines/Danbooru.ts

@@ -1,4 +1,4 @@
-import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature, UploadMode } from "../ScrapeEngine.js";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
 import { TagCategory } from "../BooruTypes.js";
 import { createNoteFromDanbooruArticle, guessContentType } from "../Utility.js";
 
@@ -8,13 +8,10 @@ export default class Danbooru extends ScrapeEngineBase {
   notes = [];
   supportedHosts = ["danbooru.donmai.us", "safebooru.donmai.us"];
 
-  override get uploadMode(): UploadMode {
-    return "content";
-  }
-
   scrapeDocument(document: Document): ScrapeResult {
     const result = new ScrapeResult(this.name);
     const post = new ScrapedPost();
+    post.uploadMode = "content";
     post.pageUrl = document.location.href;
 
     // Set image url

src/engines/Ehentai.ts

@@ -1,4 +1,4 @@
-import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapeEngineFeature, UploadMode } from "../ScrapeEngine.js";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapeEngineFeature } from "../ScrapeEngine.js";
 import { parseResolutionString } from "../Utility.js";
 import Cookies from "js-cookie";
 
@@ -8,10 +8,6 @@ export default class Ehentai extends ScrapeEngineBase {
   notes = ["Rating is assumed to be unsafe."];
   supportedHosts = ["e-hentai.org", "exhentai.org"];
 
-  override get uploadMode(): UploadMode {
-    return "content";
-  }
-
   scrapeDocument(document: Document): ScrapeResult {
     const result = new ScrapeResult(this.name);
     const post = new ScrapedPost();

src/engines/Fallback.ts

@@ -6,7 +6,7 @@ class MediaElement {
     public readonly contentType: ContentType,
     public readonly contentUrl: string,
     public readonly size: number
-  ) {}
+  ) { }
 }
 
 export default class Fallback extends ScrapeEngineBase {
@@ -27,7 +27,7 @@ export default class Fallback extends ScrapeEngineBase {
       post.pageUrl = document.location.href;
       post.contentUrl = document.location.href;
       post.contentType = "video";
-      result.posts.push(post);
+      result.tryAddPost(post);
       return result;
     }
 
@@ -50,10 +50,16 @@ export default class Fallback extends ScrapeEngineBase {
 
     if (largestMediaElement) {
       const post = new ScrapedPost();
+      post.name = "Upload as URL";
       post.pageUrl = document.location.href;
       post.contentUrl = largestMediaElement.contentUrl;
       post.contentType = largestMediaElement.contentType;
-      result.posts.push(post);
+      result.tryAddPost(post);
+
+      const clone = Object.assign({}, post);
+      clone.name = "Upload as content";
+      clone.uploadMode = "content";
+      result.tryAddPost(clone);
     }
 
     return result;

src/engines/Nozomi.ts

@@ -0,0 +1,49 @@
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
+import { TagCategory } from "../BooruTypes.js";
+
+export default class e621 extends ScrapeEngineBase {
+  name = "nozomi";
+  features: ScrapeEngineFeature[] = ["content", "tags", "tag_category"];
+  notes = ["Image preview might not work due to hotlink protection."];
+  supportedHosts = ["nozomi.la"];
+
+  scrapeDocument(document: Document): ScrapeResult {
+    const result = new ScrapeResult(this.name);
+    const post = new ScrapedPost();
+    post.pageUrl = document.location.href;
+
+    // Set image url
+    const downloadEl = document.querySelector(".post a") as HTMLAnchorElement;
+    post.contentUrl = downloadEl?.href;
+
+    // Set tags
+    const tagEls = Array.from(document.querySelectorAll(".sidebar li")).map((x) => x as HTMLLIElement);
+
+    for (const el of tagEls) {
+      const tagName = el.innerText;
+      let category: TagCategory | undefined;
+
+      switch (el.querySelector("a")?.className) {
+        // https://e621.net/help/api#tags_listing
+        case "copyright":
+          category = "copyright";
+          break;
+        case "character":
+          category = "character";
+          break;
+        case "artist":
+          category = "artist";
+          break;
+      }
+
+      if (tagName) {
+        const tag = new ScrapedTag(tagName, category);
+        post.tags.push(tag);
+      }
+    }
+
+    result.tryAddPost(post);
+
+    return result;
+  }
+}

src/engines/Twitter.ts

@@ -15,7 +15,7 @@ export default class Twitter extends ScrapeEngineBase {
   name = "twitter";
   features: ScrapeEngineFeature[] = ["content"];
   notes = [];
-  supportedHosts = ["twitter.com", "mobile.twitter.com"];
+  supportedHosts = ["twitter.com", "mobile.twitter.com", "x.com"];
 
   scrapeDocument(document: Document): ScrapeResult {
     const result = new ScrapeResult(this.name);

src/engines/index.ts

@@ -7,6 +7,7 @@ export { default as FurAffinity } from "./FurAffinity.js";
 export { default as Gelbooru } from "./Gelbooru.js";
 export { default as Inkbunny } from "./Inkbunny.js";
 export { default as Moebooru } from "./Moebooru.js";
+export { default as Nozomi } from "./Nozomi.js";
 export { default as Philomena } from "./Philomena.js";
 export { default as Pixiv } from "./Pixiv.js";
 export { default as Reddit } from "./Reddit.js";