From 352ba505e1b1395a84368734dafe01f80b87ec7d Mon Sep 17 00:00:00 2001 From: neobooru <50623835+neobooru@users.noreply.github.com> Date: Sun, 12 Sep 2021 23:45:05 +0200 Subject: [PATCH] Add rule34.us engine --- package.json | 3 +- src/NeoScraper.ts | 1 + src/engines/Common.ts | 3 ++ src/engines/Gelbooru.ts | 21 ++++++------ src/engines/Rule34us.ts | 74 +++++++++++++++++++++++++++++++++++++++++ src/engines/index.ts | 1 + 6 files changed, 91 insertions(+), 12 deletions(-) create mode 100644 src/engines/Common.ts create mode 100644 src/engines/Rule34us.ts diff --git a/package.json b/package.json index bc8f122..0b5fd3b 100644 --- a/package.json +++ b/package.json @@ -1,12 +1,13 @@ { "name": "neo-scraper", - "version": "0.4.5", + "version": "0.4.6", "description": "Scrape a booru-like post object from the DOM.", "main": "dist/index.js", "types": "dist/index.d.js", "scripts": { "test": "jest", "build": "tsc", + "watch": "tsc -w", "prepare": "npm run build" }, "author": "neobooru", diff --git a/src/NeoScraper.ts b/src/NeoScraper.ts index ccf471e..519cd19 100644 --- a/src/NeoScraper.ts +++ b/src/NeoScraper.ts @@ -17,6 +17,7 @@ export default class NeoScraper { new engines.Moebooru(), new engines.Pixiv(), new engines.Reddit(), + new engines.Rule34us(), new engines.SankakuComplex(), new engines.Shimmie2(), new engines.Shuushuu(), diff --git a/src/engines/Common.ts b/src/engines/Common.ts new file mode 100644 index 0000000..21fd41c --- /dev/null +++ b/src/engines/Common.ts @@ -0,0 +1,3 @@ +import { TagCategory } from "../BooruTypes"; + +export type CategoryMap = Record; diff --git a/src/engines/Gelbooru.ts b/src/engines/Gelbooru.ts index 0b6d0d8..dc41d95 100644 --- a/src/engines/Gelbooru.ts +++ b/src/engines/Gelbooru.ts @@ -1,5 +1,6 @@ import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; import { TagCategory } from "../BooruTypes"; +import { CategoryMap } from "./Common"; import { guessContentType } from "../Utility"; // Small hack to avoid screwing with semver @@ -8,11 +9,16 @@ enum Version { v025, // 0.2.5 } -type CategoryMap = Record; - export default class Gelbooru implements ScrapeEngine { name = "gelbooru"; + private readonly classNameToCategoryMap: CategoryMap = { + "tag-type-copyright": "copyright", + "tag-type-character": "character", + "tag-type-artist": "artist", + "tag-type-metadata": "meta", + }; + canImport(url: Location): boolean { return ( url.host == "safebooru.org" || url.host == "gelbooru.com" || url.host == "rule34.xxx" || url.host == "tbib.org" @@ -135,17 +141,10 @@ export default class Gelbooru implements ScrapeEngine { let category: TagCategory | undefined; - const classNameToCategoryMap: CategoryMap = { - "tag-type-copyright": "copyright", - "tag-type-character": "character", - "tag-type-artist": "artist", - "tag-type-metadata": "meta", - }; - // Loop over all classes, because some websites use multiple. E.g. `class="tag tag-type-copyright"` for (const className of el.classList) { - if (classNameToCategoryMap[className]) { - category = classNameToCategoryMap[className]; + if (this.classNameToCategoryMap[className]) { + category = this.classNameToCategoryMap[className]; break; } } diff --git a/src/engines/Rule34us.ts b/src/engines/Rule34us.ts new file mode 100644 index 0000000..18f4118 --- /dev/null +++ b/src/engines/Rule34us.ts @@ -0,0 +1,74 @@ +// Technically speaking rule34.us also uses gelbooru, albeit a (heavily?) modified one. +// Because pretty much no scraper code can be shared with other gelbooru-likes it makes more sense to create a new engine for rule34.us. + +import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; +import { TagCategory } from "../BooruTypes"; +import { CategoryMap } from "./Common"; +import { guessContentType } from "../Utility"; + +export default class Gelbooru implements ScrapeEngine { + name = "rule34us"; + + private readonly classNameToCategoryMap: CategoryMap = { + "character-tag": "character", + "artist-tag": "character", + "copyright-tag": "copyright", + "metadata-tag": "meta", + }; + + canImport(url: Location): boolean { + return url.host == "rule34.us"; + } + + scrapeDocument(document: Document): ScrapeResult { + let result = new ScrapeResult(this.name); + + let post = new ScrapedPost(); + post.pageUrl = document.location.href; + + // Set image url + const origImgEls = Array.from(document.querySelectorAll("ul > a")) + .map((x) => x as HTMLAnchorElement) + .filter((x) => x.innerText == "Original"); + + if (origImgEls.length > 0) { + post.contentUrl = origImgEls[0].href; + } else { + return result; + } + + // Set content type + post.contentType = guessContentType(post.contentUrl); + + // Patch contentUrl if contentType == "video" + // This is because the "Original" button does link to "302 Found" for videos. Idk why. + // Shit's fucked, there are multiple workarounds for that and this is one of them. + if (post.contentType == "video") { + post.contentUrl = post.contentUrl.replace("img2.rule34", "video.rule34"); + } + + // This is where we would set the rating and source, + // but rule34.us doesn't display those as far as I can see. + post.rating = "unsafe"; // Lucky guess. + + // Set tags + const possibleTagEls = Array.from(document.querySelectorAll(".container > ul > li")).map((x) => x as HTMLLIElement); + + for (const el of possibleTagEls) { + // Idk why booru sites can't have consistent a HTML DOM. + // Not all elements inside `possibleTagEls` are actually tags, + // only elements that have a `a` tag as a direct child are valid tags. + // E.g. the 'Edit' button also has the `general-tag` class, but it certainly is not a tag. + + const anchor = el.firstElementChild as HTMLAnchorElement | null; + if (anchor?.tagName == "A") { + const name = anchor.innerText; + const category = this.classNameToCategoryMap[el.className] as TagCategory | undefined; + post.tags.push(new ScrapedTag(name, category)); + } + } + + result.tryAddPost(post); + return result; + } +} diff --git a/src/engines/index.ts b/src/engines/index.ts index bc7285a..c110629 100644 --- a/src/engines/index.ts +++ b/src/engines/index.ts @@ -8,6 +8,7 @@ export { default as Inkbunny } from "./Inkbunny"; export { default as Moebooru } from "./Moebooru"; export { default as Pixiv } from "./Pixiv"; export { default as Reddit } from "./Reddit"; +export { default as Rule34us } from "./Rule34us"; export { default as SankakuComplex } from "./SankakuComplex"; export { default as Shimmie2 } from "./Shimmie2"; export { default as Shuushuu } from "./Shuushuu";