Initial commit

.gitignore

@@ -0,0 +1,3 @@
+.vscode/
+node_modules/
+dist/

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 neobooru
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,3 @@
+# neo-scraper
+
+Scrape a booru-like post object from the DOM.

package.json

@@ -0,0 +1,32 @@
+{
+  "name": "neo-scraper",
+  "version": "0.1.0",
+  "description": "Scrape a booru-like post object from the DOM.",
+  "main": "dist/index.js",
+  "types": "dist/index.d.js",
+  "scripts": {
+    "test": "jest",
+    "build": "tsc",
+    "prepare": "npm run build"
+  },
+  "author": "neobooru",
+  "license": "MIT",
+  "devDependencies": {
+    "@types/jest": "^24.0.18",
+    "@types/jsdom": "^12.2.4",
+    "jest": "^24.9.0",
+    "jsdom": "^15.1.1",
+    "ts-jest": "^24.1.0",
+    "typescript": "^3.6.3"
+  },
+  "jest": {
+    "transform": {
+      "^.+\\.tsx?$": "ts-jest"
+    },
+    "moduleFileExtensions": [
+      "ts",
+      "js"
+    ],
+    "testRegex": "^.+\\.test\\.ts$"
+  }
+}

src/BooruTypes.ts

@@ -0,0 +1,2 @@
+export type SafetyRating = "safe" | "sketchy" | "unsafe";
+export type TagCategory = "copyright" | "character" | "artist" | "meta";

src/NeoScraper.test.ts

@@ -0,0 +1,133 @@
+import { JSDOM } from "jsdom";
+import NeoScraper from "./NeoScraper";
+
+async function scrapeUrl(url: string) {
+    const scraper = new NeoScraper();
+    const dom = await JSDOM.fromURL(url);
+    return scraper.scrapeDocument(dom.window.document);
+}
+
+beforeAll(() => {
+    // Expose JSDOM Element constructor
+    // @ts-ignore
+    global.Element = (new JSDOM()).window.Element;
+    // 'Implement' innerText in JSDOM: https://github.com/jsdom/jsdom/issues/1245
+    // @ts-ignore
+    Object.defineProperty(global.Element.prototype, 'innerText', {
+        get() {
+            return this.textContent.replace(/(\r\n|\n|\r|^\s+|\s+$)/gm, "");;
+        },
+    });
+});
+
+test("create scraper with default engines", async () => {
+    const scraper = new NeoScraper();
+});
+
+test("all default scrapers have unique names", () => {
+    const scraper = new NeoScraper();
+    let alreadySeen: string[] = [];
+
+    for (const engine of scraper.engines) {
+        if (alreadySeen.indexOf(engine.name) != -1) {
+            fail(engine.name + " is not a unique name!");
+        } else {
+            alreadySeen.push(engine.name);
+        }
+    }
+});
+
+// Danbooru
+// All links are SFW unless stated otherwise
+
+test("get danbooru.donmai.us post 3555872", async () => {
+    var source = "https://danbooru.donmai.us/posts/3555872";
+    const res = await scrapeUrl(source);
+    expect(res.posts.length).toBe(1);
+    expect(res.posts[0].source).toBe(source);
+    expect(res.posts[0].imageUrl).toBe("https://danbooru.donmai.us/data/__flint_girls_und_panzer_drawn_by_omachi_slabco__90332299216db81b8793084e4cb7c15d.jpg");
+});
+
+// Gelbooru
+
+test("get safebooru.org post 2871532", async () => {
+    var source = "https://safebooru.org/index.php?page=post&s=view&id=2871532";
+    const res = await scrapeUrl(source);
+    expect(res.posts.length).toBe(1);
+    expect(res.posts[0].source).toBe(source);
+    expect(res.posts[0].imageUrl).toBe("https://safebooru.org//images/2757/5f19c7744db918c3d4c4c6035bc65dfa9e18866f.jpg");
+});
+
+test("get gelbooru.com post 4925055", async () => {
+    var source = "https://gelbooru.com/index.php?page=post&s=view&id=4925055&tags=kimetsu_no_yaiba";
+    const res = await scrapeUrl(source);
+    expect(res.posts.length).toBe(1);
+    expect(res.posts[0].source).toBe(source);
+    expect(res.posts[0].imageUrl).toBe("https://img2.gelbooru.com//images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
+});
+
+test("get rule34.xxx post 2740994", async () => {
+    var source = "https://rule34.xxx/index.php?page=post&s=view&id=2740994";
+    const res = await scrapeUrl(source);
+    expect(res.posts.length).toBe(1);
+    expect(res.posts[0].source).toBe(source);
+    expect(res.posts[0].imageUrl).toBe("https://img.rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg");
+});
+
+// Moebooru
+
+test("get yande.re post 438510", async () => {
+    var source = "https://yande.re/post/show/438510";
+    const res = await scrapeUrl(source);
+    expect(res.posts.length).toBe(1);
+    expect(res.posts[0].source).toBe(source);
+    expect(res.posts[0].imageUrl).toBe("https://files.yande.re/image/49b79bc372716a579d8525dcaccb56e2/yande.re%20438510%20landscape%20link%20nintendo%20princess_zelda%20sword%20the_legend_of_zelda%20the_legend_of_zelda%3A_breath_of_the_wild.jpg");
+});
+
+test("get konachan.com post 289968", async () => {
+    var source = "http://konachan.com/post/show/289968/ahri_-league_of_legends-akali-animal_ears-blonde_h";
+    const res = await scrapeUrl(source);
+    expect(res.posts.length).toBe(1);
+    expect(res.posts[0].source).toBe(source);
+    expect(res.posts[0].imageUrl).toBe("https://konachan.com/image/e40b38fb704434e4da3eae41e26bca39/Konachan.com%20-%20289968%20akali%20blue_eyes%20breasts%20choker%20cleavage%20evelynn%20foxgirl%20gloves%20group%20hat%20kai%27sa%20long_hair%20microphone%20ponytail%20saphirya%20short_hair%20signed%20watermark.jpg");
+});
+
+// Reddit
+
+test("get old.reddit.com post", async () => {
+    var source = "https://old.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/";
+    const res = await scrapeUrl(source);
+    expect(res.posts.length).toBe(1);
+    expect(res.posts[0].source).toBe(source);
+    expect(res.posts[0].imageUrl).toBe("https://i.redd.it/ln8ahz17kpa31.jpg");
+});
+
+test("get new.reddit.com post", async () => {
+    var source = "https://new.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/";
+    const res = await scrapeUrl(source);
+    expect(res.posts.length).toBe(1);
+    expect(res.posts[0].source).toBe(source);
+    expect(res.posts[0].imageUrl).toBe("https://i.redd.it/ln8ahz17kpa31.jpg");
+});
+
+// Shimmie2
+
+test("get rule34.paheal.net post 2505982", async () => {
+    var source = "https://rule34.paheal.net/post/view/2505982"; // Somewhat sketchy
+    const res = await scrapeUrl(source);
+    expect(res.posts.length).toBe(1);
+    expect(res.posts[0].source).toBe(source);
+    expect(res.posts[0].imageUrl).toBe("https://pansy.paheal.net/_images/15427ab5ed2cddbeef947e30f5c9cab6/2505982%20-%20Kai%27Sa%20League_of_Legends%20lawzilla.png");
+});
+
+// Twitter
+
+test("get twitter.com post", async () => {
+    pending("JSDOM can't parse this page");
+    // TODO: JSDOM sucks
+    // var source = "https://twitter.com/Dishwasher1910/status/1171162651461746688/photo/1";
+    // const res = await scrapeUrl(source);
+    // expect(res.posts.length).toBe(1);
+    // expect(res.posts[0].source).toBe(source);
+    // expect(res.posts[0].imageUrl).toBe("https://pbs.twimg.com/media/EEDOOCdU0AAHX00?format=jpg&name=orig");
+});

src/NeoScraper.ts

@@ -0,0 +1,41 @@
+import { ScrapeEngine, ScrapeResults, } from './ScrapeEngine';
+import * as engines from "./engines";
+
+export default class NeoScraper {
+    engines: ScrapeEngine[] = [];
+    fallbackEngine: ScrapeEngine | null = null;
+
+    constructor(useDefaultEngines: boolean = true) {
+        if (useDefaultEngines) {
+            this.engines.push(
+                new engines.Danbooru(),
+                new engines.Gelbooru(),
+                new engines.Moebooru(),
+                new engines.Reddit(),
+                new engines.Shimmie2(),
+                new engines.Twitter()
+            );
+            this.fallbackEngine = new engines.Fallback();
+        }
+    }
+
+    scrapeDocument(document: Document, engines: string[] | null = null, allowFallbackEngine: boolean = false, ignoreCanImport: boolean = false): ScrapeResults {
+        let res = new ScrapeResults();
+
+        for (const engine of this.engines) {
+            if (ignoreCanImport || engine.canImport(document.location)) {
+                res.results.push(engine.scrapeDocument(document));
+            }
+        }
+
+        if (allowFallbackEngine) {
+            if (!this.fallbackEngine) {
+                console.error("this.fallbackEngine is unset!");
+            } else {
+                res.results.push(this.fallbackEngine.scrapeDocument(document));
+            }
+        }
+
+        return res;
+    }
+}

src/ScrapeEngine.ts

@@ -0,0 +1,48 @@
+import { TagCategory, SafetyRating } from "./BooruTypes";
+
+export class ScrapedPost {
+    imageUrl: string = "";
+    source: string = "";
+    rating: SafetyRating = "safe";
+    tags: ScrapedTag[] = [];
+}
+
+export class ScrapedTag {
+    name: string = "";
+    category?: TagCategory;
+
+    constructor(name: string, category?: TagCategory) {
+        this.name = name.replace(/\s/g, "_"); // Replace all spaces with underscores
+        this.category = category;
+    }
+}
+
+export class ScrapeResult {
+    engine: string;
+    description: string = "";
+    posts: ScrapedPost[] = [];
+
+    constructor(engine: string) {
+        this.engine = engine;
+    }
+}
+
+export class ScrapeResults {
+    results: ScrapeResult[] = [];
+
+    get posts(): ScrapedPost[] {
+        var posts: ScrapedPost[] = [];
+
+        for (var res of this.results) {
+            posts = posts.concat(res.posts);
+        }
+
+        return posts;
+    }
+}
+
+export interface ScrapeEngine {
+    name: string;
+    canImport(url: Location): boolean;
+    scrapeDocument(document: Document): ScrapeResult;
+}

src/engines/Danbooru.ts

@@ -0,0 +1,87 @@
+import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
+import { TagCategory } from "../BooruTypes";
+
+export default class Gelbooru implements ScrapeEngine {
+    name = "danbooru";
+
+    canImport(url: Location): boolean {
+        return (
+            url.host == "danbooru.donmai.us" ||
+            url.host == "safebooru.donmai.us"
+        );
+    }
+
+    scrapeDocument(document: Document): ScrapeResult {
+        let result = new ScrapeResult(this.name);
+        let post = new ScrapedPost();
+        post.source = document.location.href;
+
+        // Set image url
+        const originalImageElements = Array.from(document.querySelectorAll("#post-information > ul > li > a"))
+            .filter(x => x.parentElement && x.parentElement.innerText.startsWith("Size"));
+
+        if (originalImageElements.length > 0) {
+            post.imageUrl = (originalImageElements[0] as HTMLAnchorElement).href;
+        }
+
+        // Set rating
+        const safetyExp = new RegExp("Rating: (.*)");
+        const safetyElements = Array.from(document.querySelectorAll("#post-information > ul > li"))
+            .map(x => x as HTMLLIElement)
+            .filter(x => x.innerText.startsWith("Rating"));
+
+        if (safetyElements.length > 0) {
+            const matches = safetyElements[0].innerText.match(safetyExp);
+            if (matches && matches.length > 0) {
+                switch (matches[1].toLowerCase()) {
+                    case "safe":
+                        post.rating = "safe";
+                        break;
+                    case "questionable":
+                        post.rating = "sketchy";
+                        break;
+                    case "explicit":
+                        post.rating = "unsafe";
+                        break;
+                }
+            }
+        }
+
+        // Set tags
+        const tagElements = Array.from(document.querySelectorAll("#tag-list > ul > li")).map(x => x as HTMLLIElement);
+        for (const tagElement of tagElements) {
+            let tagName: string | undefined;
+            const tagNameElement = tagElement.querySelector(".search-tag") as HTMLAnchorElement;
+            if (tagNameElement) {
+                tagName = tagNameElement.innerText;
+            }
+
+            let category: TagCategory | undefined;
+            switch (tagElement.className) {
+                case "category-3":
+                    category = "copyright";
+                    break;
+                case "category-4":
+                    category = "character";
+                    break;
+                case "category-1":
+                    category = "artist";
+                    break;
+                case "category-5":
+                    category = "meta";
+                    break;
+            }
+
+            if (tagName) {
+                let tag = new ScrapedTag(tagName, category);
+                post.tags.push(tag);
+            }
+        }
+
+        if (post.imageUrl) {
+            result.posts.push(post);
+        }
+
+        return result;
+    }
+}

src/engines/Fallback.ts

@@ -0,0 +1,30 @@
+import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine";
+
+export default class Fallback implements ScrapeEngine {
+    name = "fallback";
+
+    canImport(location: Location): boolean {
+        return false;
+    }
+
+    scrapeDocument(document: Document): ScrapeResult {
+        function getSize(el: HTMLImageElement) {
+            return el.width * el.height;
+        }
+
+        // Get largest img element on this page
+        const largestImgElement = Array.from(document.getElementsByTagName('img'))
+            .reduce((a, b) => getSize(a) > getSize(b) ? a : b);
+
+        let result = new ScrapeResult(this.name);
+
+        if (largestImgElement) {
+            let post = new ScrapedPost();
+            post.source = document.location.href;
+            post.imageUrl = largestImgElement.src;
+            result.posts.push(post);
+        }
+
+        return result;
+    }
+}

src/engines/Gelbooru.ts

@@ -0,0 +1,140 @@
+import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
+import { TagCategory } from "../BooruTypes";
+
+// Small hack to avoid screwing with semver
+enum Version {
+    v020 = 1, // 0.2.0
+    v025, // 0.2.5
+}
+
+export default class Gelbooru implements ScrapeEngine {
+    name = "gelbooru";
+
+    canImport(url: Location): boolean {
+        return (
+            url.host == "safebooru.org" ||
+            url.host == "gelbooru.com" ||
+            url.host == "rule34.xxx"
+        );
+    }
+
+    scrapeDocument(document: Document): ScrapeResult {
+        let result = new ScrapeResult(this.name);
+        let version: Version | null = null;
+        switch (document.location.host) {
+            case "safebooru.org":
+            case "rule34.xxx":
+                version = Version.v020;
+                break;
+            case "gelbooru.com":
+                version = Version.v025;
+                break;
+            default:
+                console.error("Couldn't guess version");
+                return result;
+        }
+
+        console.log("Gelbooru guessed version: " + version);
+
+        let post = new ScrapedPost();
+        post.source = document.location.href;
+
+        // Set image url
+        Array.from(document.querySelectorAll("li > a"))
+            .map(x => x as HTMLAnchorElement)
+            .filter(x => x && x.innerText == "Original image")
+            .map(x => post.imageUrl = x.href);
+
+        // Set rating
+        const safetyExp = new RegExp("Rating: (.*)");
+        let safetyElements: NodeListOf<Element>;
+
+        if (version == Version.v020) {
+            safetyElements = document.querySelectorAll("#stats > ul > li");
+        } else {
+            safetyElements = document.querySelectorAll("#tag-list > div > li");
+        }
+
+        for (const idx in safetyElements) {
+            const el = safetyElements[idx] as HTMLLIElement;
+            if (el && el.innerText) {
+                // matches is an array where (index)
+                // 0: full string
+                // 1: capture group 1 (rating, so 'Safe', 'Questionable', 'Explicit')
+                const matches = el.innerText.match(safetyExp);
+                if (matches && matches.length > 0) {
+                    switch (matches[1].toLowerCase()) {
+                        case "safe":
+                            post.rating = "safe";
+                            break;
+                        case "questionable":
+                            post.rating = "sketchy";
+                            break;
+                        case "explicit":
+                            post.rating = "unsafe";
+                            break;
+                    }
+                }
+            }
+        }
+
+        // Set tags
+        let tagElements: NodeListOf<Element>;
+
+        if (version == Version.v020) {
+            tagElements = document.querySelectorAll("#tag-sidebar > li");
+        } else {
+            tagElements = document.querySelectorAll("#tag-list > div > li[class^='tag-type']");
+        }
+
+        for (const idx in tagElements) {
+            const el = tagElements[idx] as HTMLLIElement;
+            if (el && el.innerText) {
+                let tagName: string;
+
+                if (version == Version.v020) {
+                    const nameElements = el.getElementsByTagName("a");
+
+                    // Some <li/> don't actually contain a name.
+                    // E.g. the sub-headers ("Copyright", "Character", "General", "Meta") on rule34.xxx
+                    if (nameElements.length > 0) {
+                        tagName = el.getElementsByTagName("a")[0].innerText;
+                    }
+                    else {
+                        continue;
+                    }
+                } else {
+                    // First <a> is "?"
+                    // Second <a> is "actual_tag_name"
+                    tagName = el.getElementsByTagName("a")[1].innerText;
+                }
+
+                let category: TagCategory | undefined;
+
+                switch (el.className) {
+                    case "tag-type-copyright":
+                        category = "copyright";
+                        break;
+                    case "tag-type-character":
+                        category = "character";
+                        break;
+                    case "tag-type-artist":
+                        category = "artist";
+                        break;
+                    case "tag-type-metadata":
+                        category = "meta";
+                        break;
+                }
+
+                let tag = new ScrapedTag(tagName, category);
+                post.tags.push(tag);
+            }
+        }
+
+        if (post.imageUrl) {
+            result.posts.push(post);
+        }
+
+        return result;
+    }
+}

src/engines/Moebooru.ts

@@ -0,0 +1,98 @@
+import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
+import { TagCategory } from "../BooruTypes";
+
+export default class Moebooru implements ScrapeEngine {
+    name = "moebooru";
+
+    canImport(url: Location): boolean {
+        return (
+            url.host == "yande.re" ||
+            url.host == "konachan.com"
+        );
+    }
+
+    scrapeDocument(document: Document): ScrapeResult {
+        let result = new ScrapeResult(this.name);
+        let post = new ScrapedPost();
+        post.source = document.location.href;
+
+        // Set image url
+        const originalImageElement = document.querySelector("#highres") as HTMLAnchorElement;
+        if (originalImageElement) {
+            post.imageUrl = originalImageElement.href;
+        }
+
+        // Set safety
+        // Same method as Gelbooru but cleaner code
+        const safetyExp = new RegExp("Rating: (.*)");
+        const safetyElements = Array.from(document.querySelectorAll("#stats > ul > li"))
+            .map(x => x as HTMLLIElement)
+            .filter(x => x.innerText.startsWith("Rating:"));
+
+        if (safetyElements.length > 0) {
+            const matches = safetyElements[0].innerText.match(safetyExp);
+            if (matches && matches.length > 0) {
+                switch (matches[1].toLowerCase()) {
+                    case "safe":
+                        post.rating = "safe";
+                        break;
+                    case "questionable":
+                        post.rating = "sketchy";
+                        break;
+                    case "explicit":
+                        post.rating = "unsafe";
+                        break;
+                }
+            }
+        }
+
+        // Set tags
+        const tagElements = Array.from(document.querySelectorAll("#tag-sidebar > li"))
+            .map(x => x as HTMLLIElement);
+
+        for (const el of tagElements) {
+            let tagName: string;
+            let tagNameElements = el.getElementsByTagName("a");
+
+            if (tagNameElements.length > 1) {
+                // First <a> is "?"
+                // Second <a> is "actual_tag_name"
+                tagName = tagNameElements[1].innerText;
+            } else {
+                // Skip if no tag name is found
+                continue;
+            }
+
+            let category: TagCategory | undefined;
+
+            // Loop over all classes because some sites (konachan)
+            // have mutliple classes, e.g. "tag-link tag-type-copyright"
+            for (const className of Array.from(el.classList)) {
+                switch (className) {
+                    case "tag-type-copyright":
+                        category = "copyright";
+                        break;
+                    case "tag-type-character":
+                        category = "character";
+                        break;
+                    case "tag-type-artist":
+                        category = "artist";
+                        break;
+                    case "tag-type-metadata":
+                        category = "meta";
+                        break;
+                }
+            }
+
+            let tag = new ScrapedTag(tagName, category);
+            post.tags.push(tag);
+        }
+        
+        
+        if (post.imageUrl) {
+            result.posts.push(post);
+        }
+
+        return result;
+    }
+}

src/engines/Reddit.ts

@@ -0,0 +1,34 @@
+import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine";
+
+export default class Reddit implements ScrapeEngine {
+    name = "reddit";
+
+    canImport(url: Location): boolean {
+        return url.host.endsWith("reddit.com");
+    }
+
+    scrapeDocument(document: Document): ScrapeResult {
+        let result = new ScrapeResult(this.name);
+        let post = new ScrapedPost();
+        post.source = document.location.href;
+
+        // Set image url
+        // For old.reddit.com
+        const oldLinkElements = document.querySelectorAll("a.title");
+        if (oldLinkElements.length > 0) {
+            post.imageUrl = (oldLinkElements[0] as HTMLAnchorElement).href;
+        }
+
+        // For new reddit
+        const newLinkElements = document.querySelectorAll("div[data-test-id='post-content'] > div > a");
+        if (newLinkElements.length > 0) {
+            post.imageUrl = (newLinkElements[0] as HTMLAnchorElement).href;
+        }
+
+        if (post.imageUrl) {
+            result.posts.push(post);
+        }
+
+        return result;
+    }
+}

src/engines/Shimmie2.ts

@@ -0,0 +1,42 @@
+import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
+
+export default class Shimmie2 implements ScrapeEngine {
+    name = "shimmie2";
+
+    canImport(url: Location): boolean {
+        return url.host == "rule34.paheal.net";
+    }
+
+    scrapeDocument(document: Document): ScrapeResult {
+        let result = new ScrapeResult(this.name);
+        let post = new ScrapedPost();
+        post.source = document.location.href;
+
+        // Set image url
+        const originalImageElements = Array.from(document.querySelectorAll(".image_info > tbody > tr > td > a"))
+            .map(x => x as HTMLAnchorElement)
+            .filter(x => x.innerText == "Image Only");
+
+        if (originalImageElements.length > 0) {
+            post.imageUrl = originalImageElements[0].href;
+        }
+
+        // Set safety
+        post.rating = "unsafe"; // Usually it is unsafe
+
+        // Set tags
+        // Shimmie2 doesn't support categories so we don't need to worry about those.
+        // They also use uppercase tags, so we toLowerCase() those.
+        // Tags might not match with other booru-likes, but that is for the user to figure out.
+        // For example by deleting 'wrong' tags or setting up some aliases in szurubooru.
+        post.tags = Array.from(document.querySelectorAll("a.tag_name"))
+            .map(x => (x as HTMLAnchorElement).innerText.toLowerCase())
+            .map(x => new ScrapedTag(x));
+        
+        if (post.imageUrl) {
+            result.posts.push(post);
+        }
+
+        return result;
+    }
+}

src/engines/Twitter.ts

@@ -0,0 +1,45 @@
+import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine";
+
+export function getOriginalImageUrl(imageUrl: string) {
+    const x = imageUrl.replace(/:large|:med|:small|:orig$/, ""); // Remove `:large`, `:small` etc from url
+    return x + ":orig"; // Add :orig to url (to get original image)
+}
+
+export default class Twitter implements ScrapeEngine {
+    name = "twitter";
+
+    canImport(url: Location): boolean {
+        return (
+            url.host == "danbooru.donmai.us" ||
+            url.host == "safebooru.donmai.us"
+        );
+    }
+
+    scrapeDocument(document: Document): ScrapeResult {
+        let result = new ScrapeResult(this.name);
+        let post = new ScrapedPost();
+        post.source = document.location.href;
+
+        // Set image url
+        // When clicked on post (not fullscreen, with post text etc)
+        const overlayElement = document.querySelector("#permalink-overlay-dialog");
+        if (overlayElement) {
+            const cardImageElement = overlayElement.querySelector("div[data-element-context='platform_photo_card'] > img");
+            if (cardImageElement) {
+                post.imageUrl = getOriginalImageUrl((cardImageElement as HTMLImageElement).src);
+            }
+        }
+
+        // When clicked on image (fullscreen)
+        const mediaImageElements = document.querySelectorAll("div.Gallery-media > img.media-image");
+        if (mediaImageElements.length > 0) {
+            post.imageUrl = getOriginalImageUrl((mediaImageElements[0] as HTMLImageElement).src);
+        }
+        
+        if (post.imageUrl) {
+            result.posts.push(post);
+        }
+
+        return result;
+    }
+}

src/engines/index.ts

@@ -0,0 +1,7 @@
+export { default as Danbooru } from "./Danbooru";
+export { default as Fallback } from "./Fallback";
+export { default as Gelbooru } from "./Gelbooru";
+export { default as Moebooru } from "./Moebooru";
+export { default as Reddit } from "./Reddit";
+export { default as Shimmie2 } from "./Shimmie2";
+export { default as Twitter } from "./Twitter";

src/index.ts

@@ -0,0 +1,5 @@
+export { default as NeoScraper } from "./NeoScraper";
+export * from "./ScrapeEngine";
+
+import * as Engines from "./engines";
+export { Engines };

tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "compilerOptions": {
+    "outDir": "./dist/",
+    "sourceMap": true,
+    "strict": true,
+    "module": "es2015",
+    "moduleResolution": "node",
+    "target": "es2015",
+    "skipLibCheck": true,
+    "esModuleInterop": true,
+    "removeComments": true,
+    "declaration": true
+  },
+  "include": [
+    "src/index.ts"
+  ]
+}