From bf1e9d9e62b1b2553c24f12a4e8cabeaeda7ba2f Mon Sep 17 00:00:00 2001 From: neobooru <50623835+neobooru@users.noreply.github.com> Date: Mon, 19 Jun 2023 21:05:24 +0200 Subject: [PATCH] Add experimental e-hentai engine --- package.json | 2 ++ src/NeoScraper.ts | 1 + src/ScrapeEngine.ts | 15 ++++++++++++- src/engines/Ehentai.ts | 49 ++++++++++++++++++++++++++++++++++++++++++ src/engines/index.ts | 1 + yarn.lock | 10 +++++++++ 6 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 src/engines/Ehentai.ts diff --git a/package.json b/package.json index 7d18686..b52203b 100644 --- a/package.json +++ b/package.json @@ -17,12 +17,14 @@ "author": "neobooru", "license": "MIT", "dependencies": { + "js-cookie": "^3.0.5", "turndown": "^7.1.1" }, "devDependencies": { "@types/expect-puppeteer": "^4.4.7", "@types/jest": "^26.0.14", "@types/jest-environment-puppeteer": "^4.4.1", + "@types/js-cookie": "^3.0.3", "@types/jsdom": "^12.2.4", "@types/puppeteer": "^5.4.4", "@types/turndown": "^5.0.1", diff --git a/src/NeoScraper.ts b/src/NeoScraper.ts index 24c97ee..16dd8ce 100644 --- a/src/NeoScraper.ts +++ b/src/NeoScraper.ts @@ -11,6 +11,7 @@ export default class NeoScraper { new engines.AnimePictures(), new engines.Danbooru(), new engines.e621(), + new engines.Ehentai(), new engines.FurAffinity(), new engines.Gelbooru(), new engines.Inkbunny(), diff --git a/src/ScrapeEngine.ts b/src/ScrapeEngine.ts index 4f5c648..1a657d4 100644 --- a/src/ScrapeEngine.ts +++ b/src/ScrapeEngine.ts @@ -1,17 +1,21 @@ import { TagCategory, SafetyRating } from "./BooruTypes.js"; +export type Cookies = { [key: string]: string }; export type ContentType = "image" | "video"; export class ScrapedPost { contentUrl = ""; pageUrl = ""; contentType: ContentType = "image"; + contentSize: number | undefined; resolution: [number, number] | undefined; rating: SafetyRating = "safe"; tags: ScrapedTag[] = []; notes: ScrapedNote[] = []; sources: string[] = []; referrer: string | undefined; + extraContentUrl: string | undefined; + cookies: Cookies | undefined; } export class ScrapedTag { @@ -60,7 +64,16 @@ export class ScrapeResults { } } -export type ScrapeEngineFeature = "content" | "rating" | "resolution" | "tags" | "tag_category" | "source" | "notes"; +export type ScrapeEngineFeature = + | "content" + | "rating" + | "resolution" + | "tags" + | "tag_category" + | "source" + | "notes" + | "extra_content" + | "cookies"; export interface ScrapeEngine { name: string; diff --git a/src/engines/Ehentai.ts b/src/engines/Ehentai.ts new file mode 100644 index 0000000..23012f3 --- /dev/null +++ b/src/engines/Ehentai.ts @@ -0,0 +1,49 @@ +import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapeEngineFeature } from "../ScrapeEngine.js"; +import { parseResolutionString } from "../Utility.js"; +import Cookies from "js-cookie"; + +export default class Ehentai extends ScrapeEngineBase { + name = "E-hentai"; + features: ScrapeEngineFeature[] = ["content", "resolution", "extra_content", "cookies"]; + notes = ["Rating is assumed to be unsafe."]; + supportedHosts = ["e-hentai.org", "exhentai.org"]; + + scrapeDocument(document: Document): ScrapeResult { + const result = new ScrapeResult(this.name); + const post = new ScrapedPost(); + post.pageUrl = document.location.href; + + // Set image url + const imgUrl = document.getElementById("img") as HTMLImageElement; + if (imgUrl) { + post.contentUrl = imgUrl.src; + } + + const originalSourceUrl = document.querySelector("a[href*='fullimg.php']") as HTMLAnchorElement; + if (originalSourceUrl) { + // Set "extra" image url. This URL only gives a good response if the cookies are included. + // This might also only work on certain accounts. + post.extraContentUrl = originalSourceUrl.href; + + // Set resolution + const m = originalSourceUrl.innerText.match(/\d+ x \d+/); + if (m?.length == 1) { + post.resolution = parseResolutionString(m[0]); + } + } + + // Set rating hardcoded unsafe + post.rating = "unsafe"; + + // No tags available on this page. + // On the parent page you could probably get some tags, though these don't say anything about the individual image. + // The tags only say something about the gallery as a whole. + + // Set cookies + post.cookies = Cookies.get(); + + result.tryAddPost(post); + + return result; + } +} diff --git a/src/engines/index.ts b/src/engines/index.ts index af68fc0..a46b9a6 100644 --- a/src/engines/index.ts +++ b/src/engines/index.ts @@ -1,6 +1,7 @@ export { default as AnimePictures } from "./AnimePictures.js"; export { default as Danbooru } from "./Danbooru.js"; export { default as e621 } from "./e621.js"; +export { default as Ehentai } from "./Ehentai.js"; export { default as Fallback } from "./Fallback.js"; export { default as FurAffinity } from "./FurAffinity.js"; export { default as Gelbooru } from "./Gelbooru.js"; diff --git a/yarn.lock b/yarn.lock index 611d2fa..bf2582a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -815,6 +815,11 @@ jest-diff "^26.0.0" pretty-format "^26.0.0" +"@types/js-cookie@^3.0.3": + version "3.0.3" + resolved "https://registry.yarnpkg.com/@types/js-cookie/-/js-cookie-3.0.3.tgz#d6bfbbdd0c187354ca555213d1962f6d0691ff4e" + integrity sha512-Xe7IImK09HP1sv2M/aI+48a20VX+TdRJucfq4vfRVy6nWN8PYPOEnlMRSgxJAgYQIXJVL8dZ4/ilAM7dWNaOww== + "@types/jsdom@^12.2.4": version "12.2.4" resolved "https://registry.yarnpkg.com/@types/jsdom/-/jsdom-12.2.4.tgz#845cd4d43f95b8406d9b724ec30c03edadcd9528" @@ -3823,6 +3828,11 @@ joi@^17.6.0: "@sideway/formula" "^3.0.1" "@sideway/pinpoint" "^2.0.0" +js-cookie@^3.0.5: + version "3.0.5" + resolved "https://registry.yarnpkg.com/js-cookie/-/js-cookie-3.0.5.tgz#0b7e2fd0c01552c58ba86e0841f94dc2557dcdbc" + integrity sha512-cEiJEAEoIbWfCZYKWhVwFuvPX1gETRYPw6LlaTKoxD3s2AkXzkCjnp6h0V77ozyqj0jakteJ4YqDJT830+lVGw== + js-tokens@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499"