Nes
/
neo-scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: Nes:master
Branches Tags
Nes:master
Nes:v0.5.2
Nes:v0.5.1
..
compare: Nes:v0.5.2
Branches Tags
Nes:master
Nes:v0.5.2
Nes:v0.5.1

No commits in common. "master" and "v0.5.2" have entirely different histories.
master ... v0.5.2

60 changed files with 5151 additions and 7643 deletions
Show Stats Expand all files Collapse all files

17
.eslintrc.cjs
View File

@ -1,17 +0,0 @@
module.exports = {
env: {
browser: true,
commonjs: true,
es2021: true,
node: true,
},
extends: ["eslint:recommended", "plugin:@typescript-eslint/recommended"],
parser: "@typescript-eslint/parser",
parserOptions: {
ecmaVersion: "latest",
},
plugins: ["@typescript-eslint"],
rules: {
semi: ["error", "always"],
},
};

2
.gitignore vendored
View File

@ -2,5 +2,3 @@
node_modules/ node_modules/
dist/ dist/
coverage/ coverage/
package-lock.json
package.json

39
README.md
View File

@ -1,42 +1,3 @@
# neo-scraper # neo-scraper
Scrape a booru-like post object from the DOM. Scrape a booru-like post object from the DOM.
## Supported sites
| Engine | Sites | Features | Notes |
|---|---|---|---|
| animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
| bluesky | bsky.app | content | |
| danbooru | danbooru.donmai.us, safebooru.donmai.us, sonohara.donmai.us, hijiribe.donmai.us | content, rating, resolution, tags, tag_category, source, notes | |
| e621 | e621.net, e926.net, e6ai.net | content, rating, resolution, tags, tag_category, source | |
| E-hentai | e-hentai.org, exhentai.org | content, resolution, extra_content, cookies | Rating is assumed to be unsafe. |
| furaffinity | furaffinity.net | content, rating, resolution, tags, tag_category | |
| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com, mspabooru.com, hypnohub.net | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. |
| inkbunny | inkbunny.net | content, rating, resolution, tags | |
| moebooru | yande.re, konachan.com, konachan.net, sakugabooru.com, img.genshiken-itb.org | content, rating, resolution, tags, tag_category, source, notes | |
| nozomi | nozomi.la | content, tags, tag_category | Image preview might not work due to hotlink protection. |
| Philomena | derpibooru.org, trixiebooru.org, ponybooru.org, furbooru.org, ponerpics.org, manebooru.art, twibooru.org | content, rating, resolution, tags, tag_category, source | |
| pixiv | pixiv.net | content | |
| reddit | reddit.com, new.reddit.com, old.reddit.com | content | |
| rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. |
| sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes | |
| shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. |
| shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. |
| twitter | twitter.com, mobile.twitter.com, x.com | content | |
| zerochan | zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
| fallback | | content | Tries to find the largest image or video on the current page. |
## Development
Build SzuruChrome with your local version of neo-scraper.
```sh
cd neo-scraper
pnpm install
pnpm build # or pnpm watch in another terminal
pnpm link --global
cd ../SzuruChrome
pnpm link --global neo-scraper
pnpm dev
```

24
docs/TESTING.md
View File

@ -1,24 +0,0 @@
# Testing notes
## Run all tests in testfiles matching some pattern
```sh
pnpm test pixiv
```
## Run one specific test ({{describe name}} {{it name}})
```sh
pnpm test pixiv -t "get pixiv post 96268450 \(multipage, expanded\) should find exactly four posts"
```
For some reason you need to escape the parentheses.
## Debug puppeteer tests
```sh
pnpm test:debug # you can add more arguments if you want
```
This will disable headless mode.
Use `await jestPuppeteer.debug();` in the test code to pause code execution.

2
jest-puppeteer.config.cjs → jest-puppeteer.config.js
View File

@ -1,6 +1,6 @@
module.exports = { module.exports = {
launch: { launch: {
headless: process.env.HEADLESS !== "false" ? "new" : false, headless: process.env.HEADLESS !== "false",
executablePath: process.env.CHROME_BIN || null, executablePath: process.env.CHROME_BIN || null,
args: process.env.CI ? ["--disable-gpu", "--disable-dev-shm-usage", "--no-sandbox"] : [], args: process.env.CI ? ["--disable-gpu", "--disable-dev-shm-usage", "--no-sandbox"] : [],
}, },

0
jest.preset.cjs → jest.preset.js
View File

73
package.json
View File

@ -1,77 +1,52 @@
{ {
"name": "neo-scraper", "name": "neo-scraper",
"version": "0.11.1", "version": "0.5.2",
"description": "Scrape a booru-like post object from the DOM.", "description": "Scrape a booru-like post object from the DOM.",
"type": "module",
"main": "dist/index.js", "main": "dist/index.js",
"types": "dist/index.d.js", "types": "dist/index.d.js",
"scripts": { "scripts": {
"test": "cross-env JEST_PUPPETEER_CONFIG=jest-puppeteer.config.cjs jest", "test": "jest",
"test:ci": "cross-env JEST_PUPPETEER_CONFIG=jest-puppeteer.config.cjs jest --colors --silent", "test:ci": "jest --colors --silent",
"test:debug": "cross-env JEST_PUPPETEER_CONFIG=jest-puppeteer.config.cjs HEADLESS=false jest --testTimeout=100000000",
"build": "tsc", "build": "tsc",
"watch": "tsc -w", "watch": "tsc -w",
"lint": "eslint src scripts", "prepare": "npm run build"
"summary-table": "npm run build && node --experimental-specifier-resolution=node --loader ts-node/esm scripts/summary-table.ts"
}, },
"author": "neobooru", "author": "neobooru",
"license": "MIT", "license": "MIT",
"dependencies": {
"js-cookie": "^3.0.5",
"turndown": "^7.1.2"
},
"devDependencies": { "devDependencies": {
"@types/expect-puppeteer": "^5.0.5", "@types/expect-puppeteer": "^4.4.7",
"@types/jest": "^29.5.7", "@types/jest": "^26.0.14",
"@types/jest-environment-puppeteer": "^5.0.5", "@types/jest-environment-puppeteer": "^4.4.1",
"@types/js-cookie": "^3.0.5",
"@types/jsdom": "^12.2.4", "@types/jsdom": "^12.2.4",
"@types/turndown": "^5.0.3", "@types/puppeteer": "^5.4.4",
"@typescript-eslint/eslint-plugin": "^5.62.0", "@types/turndown": "^5.0.1",
"@typescript-eslint/parser": "^5.62.0", "jest": "^26.6.3",
"cross-env": "^7.0.3", "jest-puppeteer": "^6.0.0",
"eslint": "8.7.0",
"eslint-plugin-import": "^2.29.0",
"eslint-plugin-n": "^15.7.0",
"eslint-plugin-promise": "^6.1.1",
"jest": "^29.7.0",
"jest-puppeteer": "^10.1.4",
"jest-ts-webcompat-resolver": "^1.0.0",
"jsdom": "^15.2.1", "jsdom": "^15.2.1",
"prettier": "2.5.1", "puppeteer": "^13.1.2",
"puppeteer": "^23.10.1", "ts-jest": "^26.5.6",
"ts-jest": "^29.2.5", "typescript": "^4.5.2"
"ts-node": "^10.9.2",
"typescript": "^4.9.5"
},
"exports": {
".": {
"types": "./dist/index.d.ts",
"import": "./dist/index.js"
}
}, },
"jest": { "jest": {
"preset": "./jest.preset.cjs", "preset": "./jest.preset.js",
"resolver": "jest-ts-webcompat-resolver",
"transform": { "transform": {
"^.+\\.tsx?$": [ "^.+\\.tsx?$": "ts-jest"
"ts-jest",
{
"tsconfig": "tsconfig.test.json"
}
]
}, },
"moduleFileExtensions": [ "moduleFileExtensions": [
"ts", "ts",
"js" "js"
], ],
"testTimeout": 30000,
"testRegex": "^.+\\.test\\.ts$", "testRegex": "^.+\\.test\\.ts$",
"setupFilesAfterEnv": [ "setupFilesAfterEnv": [
"./src/test/jest.setup.ts" "./src/test/jest.setup.ts"
] ],
"globals": {
"ts-jest": {
"tsconfig": "tsconfig.test.json"
}
}
}, },
"overrides": { "dependencies": {
"@types/node": "18.19.115" "turndown": "^7.1.1"
} }
} }

6131
pnpm-lock.yaml
View File

File diff suppressed because it is too large Load Diff

15
scripts/summary-table.ts
View File

@ -1,15 +0,0 @@
import NeoScraper from "../src/NeoScraper.js";
import FallbackEngine from "../src/engines/Fallback.js";
const engines = [...new NeoScraper().engines, new FallbackEngine()];
const txt = ["| Engine | Sites | Features | Notes |", "|---|---|---|---|"];
for (const engine of engines) {
txt.push(
`| ${engine.name} | ${engine.supportedHosts.join(", ")} | ${engine.features.join(", ")} | ${engine.notes.join(
" "
)} |`
);
}
console.log(txt.join("\n"));

6
src/NeoScraper.test.ts
View File

@ -1,12 +1,12 @@
import NeoScraper from "./NeoScraper.js"; import NeoScraper from "./NeoScraper";
test("create scraper with default engines", async () => { test("create scraper with default engines", async () => {
new NeoScraper(); const scraper = new NeoScraper();
}); });
test("all default scrapers have unique names", () => { test("all default scrapers have unique names", () => {
const scraper = new NeoScraper(); const scraper = new NeoScraper();
const alreadySeen: string[] = []; let alreadySeen: string[] = [];
for (const engine of scraper.engines) { for (const engine of scraper.engines) {
if (alreadySeen.indexOf(engine.name) != -1) { if (alreadySeen.indexOf(engine.name) != -1) {

18
src/NeoScraper.ts
View File

@ -1,25 +1,20 @@
import { ScrapeEngine, ScrapeResults } from "./ScrapeEngine.js"; import { ScrapeEngine, ScrapeResults } from "./ScrapeEngine";
import * as engines from "./engines/index.js"; import * as engines from "./engines";
export default class NeoScraper { export default class NeoScraper {
engines: ScrapeEngine[] = []; engines: ScrapeEngine[] = [];
fallbackEngine: ScrapeEngine | null = null; fallbackEngine: ScrapeEngine | null = null;
constructor(useDefaultEngines = true) { constructor(useDefaultEngines: boolean = true) {
if (useDefaultEngines) { if (useDefaultEngines) {
this.engines.push( this.engines.push(
new engines.AnimePictures(), new engines.AnimePictures(),
new engines.Bluesky(),
new engines.Danbooru(), new engines.Danbooru(),
new engines.e621(), new engines.e621(),
new engines.Ehentai(),
new engines.FurAffinity(), new engines.FurAffinity(),
new engines.Gelbooru(), new engines.Gelbooru(),
new engines.Inkbunny(), new engines.Inkbunny(),
new engines.Mastodon(),
new engines.Moebooru(), new engines.Moebooru(),
new engines.Nozomi(),
new engines.Philomena(),
new engines.Pixiv(), new engines.Pixiv(),
new engines.Reddit(), new engines.Reddit(),
new engines.Rule34us(), new engines.Rule34us(),
@ -33,7 +28,12 @@ export default class NeoScraper {
} }
} }
scrapeDocument(document: Document, allowFallbackEngine = false, ignoreCanImport = false): ScrapeResults { scrapeDocument(
document: Document,
engines: string[] | null = null,
allowFallbackEngine: boolean = false,
ignoreCanImport: boolean = false
): ScrapeResults {
const res = new ScrapeResults(); const res = new ScrapeResults();
for (const engine of this.engines) { for (const engine of this.engines) {

45
src/ScrapeEngine.ts
View File

@ -1,24 +1,17 @@
import { TagCategory, SafetyRating } from "./BooruTypes.js"; import { TagCategory, SafetyRating } from "./BooruTypes";
export type Cookies = { [key: string]: string };
export type ContentType = "image" | "video"; export type ContentType = "image" | "video";
export type UploadMode = "content" | "url";
export class ScrapedPost { export class ScrapedPost {
name: string | undefined = undefined; contentUrl: string = "";
contentUrl = ""; pageUrl: string = "";
pageUrl = "";
contentType: ContentType = "image"; contentType: ContentType = "image";
contentSize: number | undefined;
resolution: [number, number] | undefined; resolution: [number, number] | undefined;
rating: SafetyRating = "safe"; rating: SafetyRating = "safe";
tags: ScrapedTag[] = []; tags: ScrapedTag[] = [];
notes: ScrapedNote[] = []; notes: ScrapedNote[] = [];
sources: string[] = []; source: string | undefined;
referrer: string | undefined; referrer: string | undefined;
extraContentUrl: string | undefined;
cookies: Cookies | undefined;
uploadMode: UploadMode = "url";
} }
export class ScrapedTag { export class ScrapedTag {
@ -37,7 +30,7 @@ export class ScrapedNote {
export class ScrapeResult { export class ScrapeResult {
engine: string; engine: string;
description = ""; description: string = "";
posts: ScrapedPost[] = []; posts: ScrapedPost[] = [];
constructor(engine: string) { constructor(engine: string) {
@ -57,9 +50,9 @@ export class ScrapeResults {
results: ScrapeResult[] = []; results: ScrapeResult[] = [];
get posts(): ScrapedPost[] { get posts(): ScrapedPost[] {
let posts: ScrapedPost[] = []; var posts: ScrapedPost[] = [];
for (const res of this.results) { for (var res of this.results) {
posts = posts.concat(res.posts); posts = posts.concat(res.posts);
} }
@ -67,37 +60,15 @@ export class ScrapeResults {
} }
} }
export type ScrapeEngineFeature =
| "content"
| "rating"
| "resolution"
| "tags"
| "tag_category"
| "source"
| "notes"
| "extra_content"
| "cookies";
export interface ScrapeEngine { export interface ScrapeEngine {
name: string; name: string;
features: ScrapeEngineFeature[];
notes: string[];
supportedHosts: string[];
canImport(url: Location): boolean; canImport(url: Location): boolean;
scrapeDocument(document: Document): ScrapeResult; scrapeDocument(document: Document): ScrapeResult;
} }
export abstract class ScrapeEngineBase implements ScrapeEngine { export abstract class ScrapeEngineBase implements ScrapeEngine {
abstract name: string; abstract name: string;
abstract features: ScrapeEngineFeature[]; abstract canImport(url: Location): boolean;
abstract notes: string[];
abstract supportedHosts: string[];
canImport(url: Location): boolean {
return this.supportedHosts.indexOf(url.host.replace("www.", "")) != -1;
}
abstract scrapeDocument(document: Document): ScrapeResult; abstract scrapeDocument(document: Document): ScrapeResult;
protected log(str: string) { protected log(str: string) {

11
src/Utility.test.ts
View File

@ -1,13 +1,4 @@
import { getUrlExtension, guessContentType } from "./Utility.js"; import { guessContentType } from "./Utility";
test("getUrlExtension", () => {
expect(getUrlExtension("image.png")).toBe("png");
expect(getUrlExtension("image.jpeg?34")).toBe("jpeg");
expect(getUrlExtension("image.php?q=test&p=123%204")).toBe("php");
// I would probably rather have this return an empty string, but maybe some code relies on this functionality?
expect(getUrlExtension("http://site/index")).toBe("http://site/index");
});
test("guessContentType", () => { test("guessContentType", () => {
expect(guessContentType("video.mp4")).toBe("video"); expect(guessContentType("video.mp4")).toBe("video");

29
src/Utility.ts
View File

@ -1,5 +1,5 @@
import TurndownService from "turndown"; import TurndownService from "turndown";
import { ContentType, ScrapedNote, ScrapedPost } from "./ScrapeEngine.js"; import { ContentType, ScrapedNote, ScrapedPost } from "./ScrapeEngine";
// https://github.com/sindresorhus/video-extensions/blob/main/video-extensions.json // https://github.com/sindresorhus/video-extensions/blob/main/video-extensions.json
export const videoExtensions = [ export const videoExtensions = [
@ -39,18 +39,10 @@ export const videoExtensions = [
]; ];
export function getUrlExtension(url: string) { export function getUrlExtension(url: string) {
// https://stackoverflow.com/a/47767860
return url.split(/[#?]/)[0].split(".").pop()!.trim(); return url.split(/[#?]/)[0].split(".").pop()!.trim();
} }
/** export function guessContentType(url: string): ContentType {
* Returns "image" when the guessed type is NOT a video.
* This means that it will also return "image" when the content type
* could not be guessed!
* @param url
* @returns "image" | "video"
*/
export function guessContentType(url: string | undefined): ContentType {
if (url != undefined) { if (url != undefined) {
const ext = getUrlExtension(url); const ext = getUrlExtension(url);
if (videoExtensions.indexOf(ext) != -1) return "video"; if (videoExtensions.indexOf(ext) != -1) return "video";
@ -109,7 +101,7 @@ export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement
return undefined; return undefined;
} }
const num = parseInt(value); const num = parseInt(value);
if (isNaN(num)) { if (num == NaN) {
logFail(`parseInt on '${value}' returned NaN.`); logFail(`parseInt on '${value}' returned NaN.`);
} }
return num; return num;
@ -120,7 +112,6 @@ export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement
return undefined; return undefined;
} }
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const data = (<any>el).dataset as DOMStringMap; const data = (<any>el).dataset as DOMStringMap;
if (!data) { if (!data) {
@ -133,10 +124,10 @@ export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement
text = htmlNoteToMarkdownNote(text); text = htmlNoteToMarkdownNote(text);
} }
const x = getInt(data, "x"); let x = getInt(data, "x");
const y = getInt(data, "y"); let y = getInt(data, "y");
const w = getInt(data, "width"); let w = getInt(data, "width");
const h = getInt(data, "height"); let h = getInt(data, "height");
if (!text || !x || !y || !w || !h) { if (!text || !x || !y || !w || !h) {
logFail("A required data field could not be parsed."); logFail("A required data field could not be parsed.");
@ -160,7 +151,7 @@ export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement
* @returns * @returns
*/ */
export function createNotesFromMoebooruBoxes(document: Document, boxSize: [number, number]) { export function createNotesFromMoebooruBoxes(document: Document, boxSize: [number, number]) {
const notes: ScrapedNote[] = []; let notes: ScrapedNote[] = [];
const noteBoxes = Array.from(document.querySelectorAll(".note-box")).map((x) => x as HTMLLIElement); const noteBoxes = Array.from(document.querySelectorAll(".note-box")).map((x) => x as HTMLLIElement);
const noteBodies = Array.from(document.querySelectorAll(".note-body")).map((x) => x as HTMLLIElement); const noteBodies = Array.from(document.querySelectorAll(".note-body")).map((x) => x as HTMLLIElement);
@ -195,8 +186,8 @@ export function parseResolutionString(str: string | undefined | null): [number,
// Example string: 1600x2200 // Example string: 1600x2200
const res = str const res = str
.split("x") // Split on 'x' character .split("x") // Split on 'x' character
.map((v) => v.replace(/\D/g, "")) // Remove all non-digits .map((v, _) => v.replace(/\D/g, "")) // Remove all non-digits
.map((v) => parseInt(v)) // Parse ints .map((v, _) => parseInt(v)) // Parse ints
.filter(Number); // Filter NaN. Also removes the number 0, but that's fine because the resolution can't be 0. .filter(Number); // Filter NaN. Also removes the number 0, but that's fine because the resolution can't be 0.
if (res.length == 2) { if (res.length == 2) {

8
src/engines/AnimePictures.test.ts
View File

@ -1,8 +1,8 @@
import { scrapeEvalUrl } from "../test/engine.js"; import { scrapeUrl } from "../test/engine";
test("get anime-pictures.net post 729833", async () => { test("get anime-pictures.net post 729833", async () => {
const pageUrl = "https://anime-pictures.net/posts/729833?lang=en"; const pageUrl = "https://anime-pictures.net/pictures/view_post/729833?lang=en";
const res = await scrapeEvalUrl(pageUrl, "body"); const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("arknights", "copyright"); expect(res.posts[0]).toHaveTag("arknights", "copyright");
expect(res.posts[0]).toHaveTag("ch'en_(arknights)", "character"); expect(res.posts[0]).toHaveTag("ch'en_(arknights)", "character");
@ -14,6 +14,6 @@ test("get anime-pictures.net post 729833", async () => {
expect(res.posts[0]).toHaveResolution([4000, 6865]); expect(res.posts[0]).toHaveResolution([4000, 6865]);
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe( expect(res.posts[0].contentUrl).toBe(
"https://api.anime-pictures.net/pictures/download_image/729833-4000x6865-arknights-ch'en+(arknights)-ttk+(kirinottk)-single-long+hair-tall+image.jpeg" "https://anime-pictures.net/pictures/download_image/729833-4000x6865-arknights-ch%26%2339%3Ben+%28arknights%29-ttk+%28kirinottk%29-single-long+hair-tall+image.jpeg"
); );
}); });

30
src/engines/AnimePictures.ts
View File

@ -1,20 +1,22 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js"; import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes.js"; import { TagCategory } from "../BooruTypes";
import { guessContentType, parseResolutionString } from "../Utility.js"; import { guessContentType, parseResolutionString } from "../Utility";
export default class AnimePictures extends ScrapeEngineBase { export default class AnimePictures implements ScrapeEngine {
name = "animepictures"; name = "animepictures";
features: ScrapeEngineFeature[] = ["content", "resolution", "tags", "tag_category"];
notes = ["Rating is assumed to be safe."]; canImport(url: Location): boolean {
supportedHosts = ["anime-pictures.net"]; return url.host == "anime-pictures.net";
}
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); let result = new ScrapeResult(this.name);
const post = new ScrapedPost(); let post = new ScrapedPost();
post.pageUrl = document.location.href; post.pageUrl = document.location.href;
post.source = document.location.href;
// Set image url // Set image url
const originalImageElements = Array.from(document.getElementsByClassName("icon-download")); const originalImageElements = Array.from(document.getElementsByClassName("download_icon"));
if (originalImageElements.length > 0) { if (originalImageElements.length > 0) {
post.contentUrl = (originalImageElements[0] as HTMLAnchorElement).href; post.contentUrl = (originalImageElements[0] as HTMLAnchorElement).href;
@ -27,7 +29,9 @@ export default class AnimePictures extends ScrapeEngineBase {
post.rating = "safe"; // Probably post.rating = "safe"; // Probably
// Set resolution // Set resolution
const postContentLinkEls = Array.from(document.querySelectorAll<HTMLAnchorElement>(".post_content > .body a")); const postContentLinkEls = Array.from(
document.querySelectorAll<HTMLAnchorElement>("#content > div > .post_content > a")
);
const resEl = postContentLinkEls.find((x) => x.href.indexOf("res_x") != -1); const resEl = postContentLinkEls.find((x) => x.href.indexOf("res_x") != -1);
if (resEl) { if (resEl) {
post.resolution = parseResolutionString(resEl.innerText); post.resolution = parseResolutionString(resEl.innerText);
@ -36,7 +40,7 @@ export default class AnimePictures extends ScrapeEngineBase {
// Set tags // Set tags
const tagElements = Array.from(document.querySelectorAll<HTMLAnchorElement>("ul.tags > li > a")); const tagElements = Array.from(document.querySelectorAll<HTMLAnchorElement>("ul.tags > li > a"));
for (const tagElement of tagElements) { for (const tagElement of tagElements) {
const tagName = tagElement.innerText; let tagName = tagElement.innerText;
let category: TagCategory | undefined; let category: TagCategory | undefined;
for (const className of tagElement.classList) { for (const className of tagElement.classList) {
@ -54,7 +58,7 @@ export default class AnimePictures extends ScrapeEngineBase {
} }
if (tagName) { if (tagName) {
const tag = new ScrapedTag(tagName, category); let tag = new ScrapedTag(tagName, category);
post.tags.push(tag); post.tags.push(tag);
} }
} }

126
src/engines/Bluesky.ts
View File

@ -1,126 +0,0 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapeEngineFeature } from "../ScrapeEngine.js";
function getFullSizeImg(link: string) {
return link.replace("/feed_thumbnail/", "/feed_fullsize/");
}
/**
* Bluesky serves video via HLS in the player, but the underlying blob is a
* single .mp4 retrievable from the AT Protocol sync endpoint:
*
* https://bsky.social/xrpc/com.atproto.sync.getBlob?did=<DID>&cid=<CID>
*
* The DID and CID are URL-encoded in the <video> poster's path:
*
* https://video.bsky.app/watch/<DID>/<CID>/thumbnail.jpg
*
* We reconstruct the blob URL and mark the post as uploadMode="content"
* so SzuruChrome fetches it in the browser instead of asking Szurubooru's
* server to download it (which would fail on the blob endpoint).
*/
export default class Bluesky extends ScrapeEngineBase {
name = "bluesky";
features: ScrapeEngineFeature[] = ["content"];
notes = [];
supportedHosts = ["bsky.app"];
scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name);
type Candidate = { el: Element; post: ScrapedPost };
const candidates: Candidate[] = [];
// Images
const imgEls = Array.from<HTMLImageElement>(document.querySelectorAll("div > img"));
let els = imgEls.filter((x) => x.src.indexOf("feed_fullsize") != -1);
// Only add the 'normal' feed elements if no post is in fullscreen.
if (els.length == 0) {
this.log("Post is not fullscreen.");
els = imgEls.filter((x) => x.src.indexOf("feed") != -1);
}
for (const el of els) {
if(!isVisible(el)) continue;
const post = new ScrapedPost();
post.pageUrl = document.location.href;
post.contentUrl = getFullSizeImg(el.src);
candidates.push({ el, post });
}
// Videos
const videoEls = Array.from<HTMLVideoElement>(
document.querySelectorAll('video[poster^="https://video.bsky.app/watch"]')
);
for (const v of videoEls) {
if(!isVisible(v)) continue;
const poster = v.getAttribute("poster");
if (!poster) continue;
const blobUrl = this.buildBlobUrl(poster);
if (!blobUrl) {
this.log(`Could not parse poster URL: ${poster}`);
continue;
}
const post = new ScrapedPost();
post.pageUrl = document.location.href;
post.contentUrl = blobUrl;
post.contentType = "video";
candidates.push({ el: v, post });
}
candidates.sort((a, b) => {
const pos = a.el.compareDocumentPosition(b.el);
if (pos & Node.DOCUMENT_POSITION_FOLLOWING) return -1;
if (pos & Node.DOCUMENT_POSITION_PRECEDING) return 1;
return 0;
});
for(const c of candidates) result.tryAddPost(c.post);
return result;
}
/**
* Poster format:
* https://video.bsky.app/watch/<DID-urlencoded>/<CID>/thumbnail.jpg
* Returns the AT Protocol blob URL, or undefined if the poster doesn't
* match the expected shape.
*/
private buildBlobUrl(poster: string): string | undefined {
try {
const u = new URL(poster);
const parts = u.pathname.split("/").filter((p) => p.length > 0);
// Expected: ["watch", "<DID>", "<CID>", "thumbnail.jpg"]
if (parts.length < 3 || parts[0] !== "watch") return undefined;
const did = decodeURIComponent(parts[1]); // did:plc:...
const cid = parts[2]; // bafkrei...
return (
"https://bsky.social/xrpc/com.atproto.sync.getBlob" +
`?did=${encodeURIComponent(did)}&cid=${encodeURIComponent(cid)}`
);
} catch {
return undefined;
}
}
}
/**
* A lightweight visibility check. We use this rather than IntersectionObserver
* because scrapeDocument is synchronous and must return immediately.
*/
function isVisible(el: Element): boolean {
const rect = el.getBoundingClientRect();
if (rect.width === 0 || rect.height === 0) return false;
if (rect.bottom <= 0 || rect.top >= window.innerHeight) return false;
if (rect.right <= 0 || rect.left >= window.innerWidth) return false;
const style = window.getComputedStyle(el);
if (style.display === "none") return false;
if (style.visibility === "hidden") return false;
if (style.opacity === "0") return false;
return true;
}

2
src/engines/Common.ts
View File

@ -1,3 +1,3 @@
import { TagCategory } from "../BooruTypes.js"; import { TagCategory } from "../BooruTypes";
export type CategoryMap = Record<string, TagCategory>; export type CategoryMap = Record<string, TagCategory>;

25
src/engines/Danbooru.test.ts
View File

@ -1,27 +1,27 @@
import { scrapeEvalUrl } from "../test/engine.js"; import { scrapeUrl } from "../test/engine";
test("get danbooru.donmai.us post 3555872", async () => { test("get danbooru.donmai.us post 3555872", async () => {
const pageUrl = "https://danbooru.donmai.us/posts/3555872"; const pageUrl = "https://danbooru.donmai.us/posts/3555872";
const res = await scrapeEvalUrl(pageUrl, "#tag-list"); const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("omachi_(slabco)", "artist"); expect(res.posts[0]).toHaveTag("omachi_(slabco)", "artist");
expect(res.posts[0]).toHaveTag("dr_pepper"); expect(res.posts[0]).toHaveTag("dr_pepper", "copyright");
expect(res.posts[0]).toHaveTag("flint_(girls_und_panzer)", "character"); expect(res.posts[0]).toHaveTag("flint_(girls_und_panzer)", "character");
expect(res.posts[0]).toHaveTag("bangs");
expect(res.posts[0]).toHaveTag("shirt"); expect(res.posts[0]).toHaveTag("shirt");
expect(res.posts[0]).toHaveTag("skirt"); expect(res.posts[0]).toHaveTag("skirt");
expect(res.posts[0]).toHaveTag("t-shirt"); expect(res.posts[0]).toHaveTag("t-shirt");
expect(res.posts[0]).toHaveTag("long_hair"); expect(res.posts[0]).toHaveTag("long_hair");
expect(res.posts[0]).toHaveResolution([1142, 1600]); expect(res.posts[0]).toHaveResolution([1142, 1600]);
expect(res.posts[0]).toHaveSource("https://twitter.com/omatiosake/status/1124306601584672768");
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe( expect(res.posts[0].contentUrl).toBe(
"https://cdn.donmai.us/original/90/33/__flint_girls_und_panzer_drawn_by_omachi_slabco__90332299216db81b8793084e4cb7c15d.jpg" "https://cdn.donmai.us/original/90/33/__flint_girls_und_panzer_and_1_more_drawn_by_omachi_slabco__90332299216db81b8793084e4cb7c15d.jpg"
); );
}); });
test("get danbooru.donmai.us post 4924490", async () => { test("get danbooru.donmai.us post 4924490", async () => {
const pageUrl = "https://danbooru.donmai.us/posts/4924490"; const pageUrl = "https://danbooru.donmai.us/posts/4924490";
const res = await scrapeEvalUrl(pageUrl, "#tag-list"); const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("yuureidoushi_(yuurei6214)", "artist"); expect(res.posts[0]).toHaveTag("yuureidoushi_(yuurei6214)", "artist");
expect(res.posts[0]).toHaveTag("comic"); expect(res.posts[0]).toHaveTag("comic");
@ -33,16 +33,3 @@ test("get danbooru.donmai.us post 4924490", async () => {
expect(res.posts[0].notes).toHaveLength(17); expect(res.posts[0].notes).toHaveLength(17);
expect(res.posts[0]).toHaveNote("Nya~nya ♪"); expect(res.posts[0]).toHaveNote("Nya~nya ♪");
}); });
test("get danbooru.donmai.us post 4530532", async () => {
const pageUrl = "https://danbooru.donmai.us/posts/4530532";
const res = await scrapeEvalUrl(pageUrl, "#tag-list");
expect(res.posts.length).toBe(1);
expect(res.posts[0].rating).toBe("sketchy");
expect(res.posts[0]).toHaveTag("chainsaw_man", "copyright");
expect(res.posts[0]).toHaveResolution([1105, 1803]);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://cdn.donmai.us/original/fc/e3/__makima_power_and_reze_chainsaw_man_drawn_by_hews__fce35c7a8515a00c566b288a591fd5f9.png"
);
});

33
src/engines/Danbooru.ts
View File

@ -1,23 +1,17 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js"; import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapedNote } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes.js"; import { TagCategory } from "../BooruTypes";
import { createNoteFromDanbooruArticle, guessContentType } from "../Utility.js"; import { createNoteFromDanbooruArticle, guessContentType } from "../Utility";
export default class Danbooru extends ScrapeEngineBase { export default class Danbooru extends ScrapeEngineBase {
name = "danbooru"; name = "danbooru";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
notes = []; canImport(url: Location): boolean {
supportedHosts = [ return url.host == "danbooru.donmai.us" || url.host == "safebooru.donmai.us";
"danbooru.donmai.us", }
"safebooru.donmai.us",
"sonohara.donmai.us",
"hijiribe.donmai.us",
"booru.allthefallen.moe"
];
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); let result = new ScrapeResult(this.name);
const post = new ScrapedPost(); let post = new ScrapedPost();
post.uploadMode = "content";
post.pageUrl = document.location.href; post.pageUrl = document.location.href;
// Set image url // Set image url
@ -62,7 +56,6 @@ export default class Danbooru extends ScrapeEngineBase {
case "safe": case "safe":
post.rating = "safe"; post.rating = "safe";
break; break;
case "sensitive":
case "questionable": case "questionable":
post.rating = "sketchy"; post.rating = "sketchy";
break; break;
@ -83,8 +76,7 @@ export default class Danbooru extends ScrapeEngineBase {
} }
let category: TagCategory | undefined; let category: TagCategory | undefined;
for (const className of tagElement.classList) { switch (tagElement.className) {
switch (className) {
case "tag-type-3": case "tag-type-3":
category = "copyright"; category = "copyright";
break; break;
@ -98,10 +90,9 @@ export default class Danbooru extends ScrapeEngineBase {
category = "meta"; category = "meta";
break; break;
} }
}
if (tagName) { if (tagName) {
const tag = new ScrapedTag(tagName, category); let tag = new ScrapedTag(tagName, category);
post.tags.push(tag); post.tags.push(tag);
} }
} }
@ -111,7 +102,7 @@ export default class Danbooru extends ScrapeEngineBase {
if (sourceEl && sourceEl.children.length > 0) { if (sourceEl && sourceEl.children.length > 0) {
const sourceLink = sourceEl.children[0] as HTMLAnchorElement; const sourceLink = sourceEl.children[0] as HTMLAnchorElement;
if (sourceLink) { if (sourceLink) {
post.sources = [sourceLink.href]; post.source = sourceLink.href;
} }
} }

49
src/engines/Ehentai.ts
View File

@ -1,49 +0,0 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapeEngineFeature } from "../ScrapeEngine.js";
import { parseResolutionString } from "../Utility.js";
import Cookies from "js-cookie";
export default class Ehentai extends ScrapeEngineBase {
name = "E-hentai";
features: ScrapeEngineFeature[] = ["content", "resolution", "extra_content", "cookies"];
notes = ["Rating is assumed to be unsafe."];
supportedHosts = ["e-hentai.org", "exhentai.org"];
scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name);
const post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url
const imgUrl = document.getElementById("img") as HTMLImageElement;
if (imgUrl) {
post.contentUrl = imgUrl.src;
}
const originalSourceUrl = document.querySelector("a[href*='fullimg']") as HTMLAnchorElement;
if (originalSourceUrl) {
// Set "extra" image url. This URL only gives a good response if the cookies are included.
// This might also only work on certain accounts.
post.extraContentUrl = originalSourceUrl.href;
// Set resolution
const m = originalSourceUrl.innerText.match(/\d+ x \d+/);
if (m?.length == 1) {
post.resolution = parseResolutionString(m[0]);
}
}
// Set rating hardcoded unsafe
post.rating = "unsafe";
// No tags available on this page.
// On the parent page you could probably get some tags, though these don't say anything about the individual image.
// The tags only say something about the gallery as a whole.
// Set cookies
post.cookies = Cookies.get();
result.tryAddPost(post);
return result;
}
}

37
src/engines/Fallback.ts
View File

@ -1,5 +1,4 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ContentType, ScrapeEngineFeature } from "../ScrapeEngine.js"; import { ScrapeEngine, ScrapeResult, ScrapedPost, ContentType } from "../ScrapeEngine";
import { guessContentType } from "../Utility.js";
class MediaElement { class MediaElement {
constructor( constructor(
@ -9,28 +8,14 @@ class MediaElement {
) {} ) {}
} }
export default class Fallback extends ScrapeEngineBase { export default class Fallback implements ScrapeEngine {
name = "fallback"; name = "fallback";
features: ScrapeEngineFeature[] = ["content"];
notes = ["Tries to find the largest image or video on the current page."];
supportedHosts = [];
scrapeDocument(document: Document): ScrapeResult { canImport(location: Location): boolean {
const result = new ScrapeResult(this.name); return false;
// Check if the current page is a link to a video file.
// If it is then use the document.location.href as media source and immediately return.
// This is needed because firefox does not add a <source> to a <video> element
// when going to a direct link. It looks like chromium-based browsers do not have this issue.
if (guessContentType(document.location.href) == "video") {
const post = new ScrapedPost();
post.pageUrl = document.location.href;
post.contentUrl = document.location.href;
post.contentType = "video";
result.tryAddPost(post);
return result;
} }
scrapeDocument(document: Document): ScrapeResult {
// Get all img elements. // Get all img elements.
const largestImgElement = Array.from(document.getElementsByTagName("img")).map( const largestImgElement = Array.from(document.getElementsByTagName("img")).map(
(x) => new MediaElement("image", x.src, x.width * x.height) (x) => new MediaElement("image", x.src, x.width * x.height)
@ -48,18 +33,14 @@ export default class Fallback extends ScrapeEngineBase {
a?.size > b?.size ? a : b a?.size > b?.size ? a : b
); );
let result = new ScrapeResult(this.name);
if (largestMediaElement) { if (largestMediaElement) {
const post = new ScrapedPost(); let post = new ScrapedPost();
post.name = "Upload as URL";
post.pageUrl = document.location.href; post.pageUrl = document.location.href;
post.contentUrl = largestMediaElement.contentUrl; post.contentUrl = largestMediaElement.contentUrl;
post.contentType = largestMediaElement.contentType; post.contentType = largestMediaElement.contentType;
result.tryAddPost(post); result.posts.push(post);
const clone = Object.assign({}, post);
clone.name = "Upload as content";
clone.uploadMode = "content";
result.tryAddPost(clone);
} }
return result; return result;

16
src/engines/FurAffinity.test.ts
View File

@ -1,16 +0,0 @@
import { scrapeUrl } from "../test/engine.js";
test("get furaffinity.net post 23281942", async () => {
const pageUrl = "https://www.furaffinity.net/view/23281942/";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("zac");
expect(res.posts[0]).toHaveTag("riven");
expect(res.posts[0]).toHaveTag("pool");
expect(res.posts[0]).toHaveTag("nielsdejong", "artist");
expect(res.posts[0]).toHaveResolution([1280, 1279]);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://d.furaffinity.net/art/nielsdejong/1492781239/1492781239.nielsdejong_pool_party_fun_with_riven_and_zac_by_polkinart.jpg"
);
});

35
src/engines/FurAffinity.ts
View File

@ -1,15 +1,17 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js"; import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { guessContentType, parseResolutionString } from "../Utility.js"; import { guessContentType } from "../Utility";
export default class FurAffinity extends ScrapeEngineBase { export default class e621 implements ScrapeEngine {
name = "furaffinity"; name = "furaffinity";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category"];
notes = []; canImport(url: Location): boolean {
supportedHosts = ["furaffinity.net"]; return url.host == "www.furaffinity.net";
}
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); let result = new ScrapeResult(this.name);
const post = new ScrapedPost();
let post = new ScrapedPost();
post.pageUrl = document.location.href; post.pageUrl = document.location.href;
// Set image url // Set image url
@ -20,14 +22,6 @@ export default class FurAffinity extends ScrapeEngineBase {
// Set content type // Set content type
post.contentType = guessContentType(post.contentUrl); post.contentType = guessContentType(post.contentUrl);
// Set resolution
const infoTexts = Array.from(document.querySelector("section.info.text")?.children ?? []);
const sizeDiv = infoTexts.find((x) => x.querySelector("strong")?.innerText == "Size");
if (sizeDiv) {
const resolution = parseResolutionString(sizeDiv.querySelector("span")?.innerText);
post.resolution = resolution;
}
// Set rating // Set rating
const ratingEls = Array.from(document.querySelectorAll("div.rating > span.rating-box")).map( const ratingEls = Array.from(document.querySelectorAll("div.rating > span.rating-box")).map(
(x) => x as HTMLSpanElement (x) => x as HTMLSpanElement
@ -47,15 +41,12 @@ export default class FurAffinity extends ScrapeEngineBase {
} }
// Set tags // Set tags
post.tags = Array.from(document.querySelectorAll("section.tags-row > span.tags a")) post.tags = Array.from(document.querySelectorAll("section.tags-row > span.tags"))
.map((x) => (x as HTMLSpanElement).innerText) .map((x) => (x as HTMLSpanElement).innerText)
.map((x) => new ScrapedTag(x)); .map((x) => new ScrapedTag(x));
// Set user who submitted the art as the artist // Set source
const submissionUser = document.querySelector<HTMLElement>(".submission-id-sub-container a")?.innerText; post.source = document.location.href;
if (submissionUser) {
post.tags.push(new ScrapedTag(submissionUser, "artist"));
}
result.tryAddPost(post); result.tryAddPost(post);

20
src/engines/Gelbooru.test.ts
View File

@ -1,4 +1,4 @@
import { scrapeUrl } from "../test/engine.js"; import { scrapeUrl } from "../test/engine";
test("get safebooru.org post 2871532", async () => { test("get safebooru.org post 2871532", async () => {
const pageUrl = "https://safebooru.org/index.php?page=post&s=view&id=2871532"; const pageUrl = "https://safebooru.org/index.php?page=post&s=view&id=2871532";
@ -13,8 +13,7 @@ test("get safebooru.org post 2871532", async () => {
expect(res.posts[0]).toHaveTag("long_hair"); expect(res.posts[0]).toHaveTag("long_hair");
expect(res.posts[0]).toHaveResolution([1142, 1600]); expect(res.posts[0]).toHaveResolution([1142, 1600]);
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("safe"); expect(res.posts[0].contentUrl).toBe(
expect(res.posts[0].contentUrl).toContain(
"https://safebooru.org//images/2757/5f19c7744db918c3d4c4c6035bc65dfa9e18866f.jpg" "https://safebooru.org//images/2757/5f19c7744db918c3d4c4c6035bc65dfa9e18866f.jpg"
); );
}); });
@ -26,16 +25,15 @@ test("get gelbooru.com post 4925055", async () => {
expect(res.posts[0]).toHaveTag("maosen", "artist"); expect(res.posts[0]).toHaveTag("maosen", "artist");
expect(res.posts[0]).toHaveTag("butterfly"); expect(res.posts[0]).toHaveTag("butterfly");
expect(res.posts[0]).toHaveResolution([720, 1103]); expect(res.posts[0]).toHaveResolution([720, 1103]);
expect(res.posts[0]).toHaveSource("https://www.pixiv.net/artworks/76853958");
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("sketchy"); expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
expect(res.posts[0].contentUrl).toContain("gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
}); });
test("get gelbooru.com post 6225495 with notes", async () => { test("get gelbooru.com post 6225495 with notes", async () => {
const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=6225495"; const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=6225495";
const res = await scrapeUrl(pageUrl); const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("bangs");
expect(res.posts[0]).toHaveTag("hair_up"); expect(res.posts[0]).toHaveTag("hair_up");
expect(res.posts[0]).toHaveTag("hair_up"); expect(res.posts[0]).toHaveTag("hair_up");
expect(res.posts[0]).toHaveTag("teeth"); expect(res.posts[0]).toHaveTag("teeth");
@ -47,10 +45,8 @@ test("get gelbooru.com post 6225495 with notes", async () => {
expect(res.posts[0]).toHaveNote("But no need to worry!"); expect(res.posts[0]).toHaveNote("But no need to worry!");
expect(res.posts[0]).toHaveNote("You're always the best just by being yourself-!!!"); expect(res.posts[0]).toHaveNote("You're always the best just by being yourself-!!!");
expect(res.posts[0]).toHaveResolution([1600, 2200]); expect(res.posts[0]).toHaveResolution([1600, 2200]);
expect(res.posts[0]).toHaveSource("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=88721816");
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("sketchy"); expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/af/1d/af1dd661fab6714dffc29cb38b6b2269.png");
expect(res.posts[0].contentUrl).toContain("gelbooru.com/images/af/1d/af1dd661fab6714dffc29cb38b6b2269.png");
}); });
test("get rule34.xxx post 2740994", async () => { test("get rule34.xxx post 2740994", async () => {
@ -58,13 +54,12 @@ test("get rule34.xxx post 2740994", async () => {
const res = await scrapeUrl(pageUrl); const res = await scrapeUrl(pageUrl);
expect(res.posts[0]).toHaveTag("fire_emblem", "copyright"); expect(res.posts[0]).toHaveTag("fire_emblem", "copyright");
expect(res.posts[0]).toHaveTag("nintendo", "copyright"); expect(res.posts[0]).toHaveTag("nintendo", "copyright");
expect(res.posts[0]).toHaveTag("lucina_(fire_emblem)", "character"); expect(res.posts[0]).toHaveTag("lucina", "character");
expect(res.posts[0]).toHaveTag("blue_hair"); expect(res.posts[0]).toHaveTag("blue_hair");
expect(res.posts[0]).toHaveTag("solo"); expect(res.posts[0]).toHaveTag("solo");
expect(res.posts[0]).toHaveResolution([600, 800]); expect(res.posts[0]).toHaveResolution([600, 800]);
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("safe");
expect(res.posts[0].contentUrl).toContain(".rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg"); expect(res.posts[0].contentUrl).toContain(".rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg");
}); });
@ -79,6 +74,5 @@ test("get xbooru.com post 834083", async () => {
expect(res.posts[0]).toHaveResolution([1024, 1878]); expect(res.posts[0]).toHaveResolution([1024, 1878]);
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("safe"); expect(res.posts[0].contentUrl).toBe("https://img.xbooru.com//images/584/040dc1625c094a7d5d7b0918aa0d8324.png");
expect(res.posts[0].contentUrl).toContain("https://img.xbooru.com//images/584/040dc1625c094a7d5d7b0918aa0d8324.png");
}); });

51
src/engines/Gelbooru.ts
View File

@ -1,7 +1,7 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js"; import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes.js"; import { TagCategory } from "../BooruTypes";
import { CategoryMap } from "./Common.js"; import { CategoryMap } from "./Common";
import { createNoteFromDanbooruArticle, guessContentType, parseResolutionString } from "../Utility.js"; import { createNoteFromDanbooruArticle, guessContentType, parseResolutionString } from "../Utility";
// Small hack to avoid screwing with semver // Small hack to avoid screwing with semver
enum Version { enum Version {
@ -11,17 +11,6 @@ enum Version {
export default class Gelbooru extends ScrapeEngineBase { export default class Gelbooru extends ScrapeEngineBase {
name = "gelbooru"; name = "gelbooru";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
notes = ["Supported features might vary between hosts."];
supportedHosts = [
"safebooru.org",
"gelbooru.com",
"rule34.xxx",
"tbib.org",
"xbooru.com",
"mspabooru.com",
"hypnohub.net",
];
private readonly classNameToCategoryMap: CategoryMap = { private readonly classNameToCategoryMap: CategoryMap = {
"tag-type-copyright": "copyright", "tag-type-copyright": "copyright",
@ -30,16 +19,24 @@ export default class Gelbooru extends ScrapeEngineBase {
"tag-type-metadata": "meta", "tag-type-metadata": "meta",
}; };
canImport(url: Location): boolean {
return (
url.host == "safebooru.org" ||
url.host == "gelbooru.com" ||
url.host == "rule34.xxx" ||
url.host == "tbib.org" ||
url.host == "xbooru.com"
);
}
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); let result = new ScrapeResult(this.name);
let version: Version | null = null; let version: Version | null = null;
switch (document.location.host) { switch (document.location.host) {
case "safebooru.org": case "safebooru.org":
case "rule34.xxx": case "rule34.xxx":
case "tbib.org": case "tbib.org":
case "xbooru.com": case "xbooru.com":
case "mspabooru.com":
case "hypnohub.net":
version = Version.v020; version = Version.v020;
break; break;
case "gelbooru.com": case "gelbooru.com":
@ -52,7 +49,7 @@ export default class Gelbooru extends ScrapeEngineBase {
this.log("Guessed version: " + version); this.log("Guessed version: " + version);
const post = new ScrapedPost(); let post = new ScrapedPost();
post.pageUrl = document.location.href; post.pageUrl = document.location.href;
// Set image url // Set image url
@ -89,12 +86,10 @@ export default class Gelbooru extends ScrapeEngineBase {
if (matches && matches.length == 3) { if (matches && matches.length == 3) {
// If Source element // If Source element
if (matches[1] == "Source") { if (matches[1] == "Source") {
const anchor = el.querySelector("a"); if (version == Version.v020) {
if (anchor && anchor.href) { post.source = matches[2];
post.sources = [anchor.href];
} else { } else {
// Fallback to plain text just in case post.source = (el.children[0] as HTMLAnchorElement).href;
post.sources = [matches[2]];
} }
} }
// If Rating element // If Rating element
@ -103,7 +98,6 @@ export default class Gelbooru extends ScrapeEngineBase {
case "safe": case "safe":
post.rating = "safe"; post.rating = "safe";
break; break;
case "sensitive":
case "questionable": case "questionable":
post.rating = "sketchy"; post.rating = "sketchy";
break; break;
@ -140,9 +134,7 @@ export default class Gelbooru extends ScrapeEngineBase {
// Some <li/> don't actually contain a name. // Some <li/> don't actually contain a name.
// E.g. the sub-headers ("Copyright", "Character", "General", "Meta") on rule34.xxx // E.g. the sub-headers ("Copyright", "Character", "General", "Meta") on rule34.xxx
if (nameElements.length > 0) { if (nameElements.length > 0) {
// Get the last element, because rule34.xxx also adds a "?" link in front. tagName = el.getElementsByTagName("a")[0].innerText;
const linkEls = Array.from(el.getElementsByTagName("a"));
tagName = linkEls[linkEls.length - 1].innerText;
} else { } else {
continue; continue;
} }
@ -162,7 +154,7 @@ export default class Gelbooru extends ScrapeEngineBase {
} }
} }
const tag = new ScrapedTag(tagName, category); let tag = new ScrapedTag(tagName, category);
post.tags.push(tag); post.tags.push(tag);
} }
} }
@ -181,7 +173,6 @@ export default class Gelbooru extends ScrapeEngineBase {
} }
// End of code duplication with Danbooru engine. // End of code duplication with Danbooru engine.
post.referrer = document.location.origin;
result.tryAddPost(post); result.tryAddPost(post);
return result; return result;

5
src/engines/Inkbunny.test.ts
View File

@ -1,4 +1,4 @@
import { scrapeUrl } from "../test/engine.js"; import { scrapeUrl } from "../test/engine";
test("get inkbunny.net post 2430530", async () => { test("get inkbunny.net post 2430530", async () => {
const pageUrl = "https://inkbunny.net/s/2430530"; const pageUrl = "https://inkbunny.net/s/2430530";
@ -7,7 +7,6 @@ test("get inkbunny.net post 2430530", async () => {
expect(res.posts[0]).toHaveTag("comic"); expect(res.posts[0]).toHaveTag("comic");
expect(res.posts[0]).toHaveTag("flower"); expect(res.posts[0]).toHaveTag("flower");
expect(res.posts[0]).toHaveTag("medusa"); expect(res.posts[0]).toHaveTag("medusa");
expect(res.posts[0]).toHaveResolution([1062, 1521]);
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toContain("metapix.net/files/full/3562/3562592_vavacung_medusa.png"); expect(res.posts[0].contentUrl).toBe("https://nl.ib.metapix.net/files/full/3562/3562592_vavacung_medusa.png");
}); });

27
src/engines/Inkbunny.ts
View File

@ -1,14 +1,16 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js"; import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
export default class Inkbunny extends ScrapeEngineBase { export default class Inkbunny implements ScrapeEngine {
name = "inkbunny"; name = "inkbunny";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags"];
notes = []; canImport(url: Location): boolean {
supportedHosts = ["inkbunny.net"]; return url.host == "inkbunny.net";
}
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); let result = new ScrapeResult(this.name);
const post = new ScrapedPost();
let post = new ScrapedPost();
post.pageUrl = document.location.href; post.pageUrl = document.location.href;
// Set image url // Set image url
@ -31,20 +33,13 @@ export default class Inkbunny extends ScrapeEngineBase {
return result; return result;
} }
// Set post resolution
const width = parseInt((<HTMLInputElement>document.getElementById("submission-origwidth"))?.value);
const height = parseInt((<HTMLInputElement>document.getElementById("submission-origheight"))?.value);
if (!isNaN(width) && !isNaN(height)) {
post.resolution = [width, height];
}
// Set rating // Set rating
const ratingExp = new RegExp("Rating: (.*)"); const ratingExp = new RegExp("Rating: (.*)");
const spanEls = Array.from(document.querySelectorAll("div > span")); const spanEls = Array.from(document.querySelectorAll("div > span"));
const ratingEls = spanEls.map((x) => x as HTMLDivElement).filter((x) => x.innerText.indexOf("Rating:") != -1); const ratingEls = spanEls.map((x) => x as HTMLDivElement).filter((x) => x.innerText.indexOf("Rating:") != -1);
if (ratingEls.length > 0) { if (ratingEls.length > 0) {
const matches = ratingEls[0].parentElement?.innerText.match(ratingExp); const matches = ratingEls[0].parentElement!.innerText.match(ratingExp);
if (matches && matches.length > 0) { if (matches && matches.length > 0) {
switch (matches[1].toLowerCase()) { switch (matches[1].toLowerCase()) {
case "general": case "general":
@ -66,7 +61,7 @@ export default class Inkbunny extends ScrapeEngineBase {
const keywords = keywordsContent.split(",").map((x) => x.trim()); const keywords = keywordsContent.split(",").map((x) => x.trim());
for (const keyword of keywords) { for (const keyword of keywords) {
const tag = new ScrapedTag(keyword); let tag = new ScrapedTag(keyword);
post.tags.push(tag); post.tags.push(tag);
} }
} }

145
src/engines/Mastodon.ts
View File

@ -1,145 +0,0 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapeEngineFeature, ContentType} from "../ScrapeEngine.js";
interface Attachment {
url: string; // original media URL
contentType: ContentType;
}
export default class Mastodon extends ScrapeEngineBase {
name = "mastodon";
features: ScrapeEngineFeature[] = ["content"];
notes = [];
supportedHosts = [
"aethy.com",
"baraag.net",
"mastodon.online",
"mastodon.social",
"mastodon.world",
"mstdn.jp",
"mstdn.social",
"pawoo.net"
];
/**
* Enabled for any other instances matching the pattern: .../@user/<postID>
*/
canImport(url: Location): boolean {
return this.supportedHosts.indexOf(url.host.replace("www.", "")) !== -1
|| /^\/@[^/]+\/\d+\/?$/.test(url.pathname);
}
scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name);
const attachments = this.findAttachments(document);
for (const att of attachments) {
const post = new ScrapedPost();
post.pageUrl = document.location.href;
post.contentUrl = att.url;
post.contentType = att.contentType;
post.referrer = document.location.origin;
result.tryAddPost(post);
}
return result;
}
private findAttachments(document: Document): Attachment[] {
const out: Attachment[] = [];
const seen = new Set<string>();
const add = (att: Attachment) => {
if (seen.has(att.url)) return;
seen.add(att.url);
out.push(att);
};
/**
* Images -> a[href]
*
* <a class="media-gallery__item-thumbnail" href=".../original/....png">
* <img src=".../small/....png">
* </a>
*/
let anchors = Array.from(
document.querySelectorAll<HTMLAnchorElement>("a.media-gallery__item-thumbnail[href]")
);
if (anchors.length === 0) {
anchors = Array.from(
document.querySelectorAll<HTMLAnchorElement>("a[href*='/media_attachments/']")
);
}
for (const a of anchors) {
const rawHref = a.getAttribute("href");
if (!rawHref) continue;
const img = a.querySelector("img");
const url = this.resolveUrl(rawHref) || this.largestFromSrcset(img);
if (!url) continue;
add({
url,
contentType: this.contentTypeFor(url),
});
}
/**
* GIFs and Videos -> video[src]
*
* <div class="media-gallery__gifv autoplay">
* <video class="media-gallery__item-gifv-thumbnail" src=".../original/....mp4" autoplay loop></video>
* </div>
*
* <div role="menuitem" class="video-player">
* <video src=".../original/....mp4" poster=".../small/....png"></video>
* </div>
*/
const videos = Array.from(document.querySelectorAll<HTMLVideoElement>("video[src]"));
for (const v of videos) {
const rawSrc = v.getAttribute("src") || v.currentSrc;
if (!rawSrc) continue;
const url = this.resolveUrl(rawSrc);
if (!url) continue;
add({
url,
contentType: "video",
});
}
return out;
}
/**
* Resolve possible relative media URLs against current page.
*/
private resolveUrl(href: string): string {
try {
return new URL(href, document.location.href).toString();
} catch {
return href;
}
}
private largestFromSrcset(img: HTMLImageElement | null): string | undefined {
if (!img) return undefined;
const srcset = img.getAttribute("srcset");
if (!srcset) return img.src || undefined;
let best: { url: string; w: number } | undefined;
for (const part of srcset.split(",")) {
const [urlPart, descriptor] = part.trim().split(/\s+/);
if (!urlPart) continue;
const w = descriptor?.endsWith("w") ? parseInt(descriptor, 10) : 0;
if (!best || w > best.w) best = { url: urlPart, w };
}
return best?.url;
}
private contentTypeFor(url: string): ContentType {
const path = url.split("?")[0].toLowerCase();
if (/\.(mp4|webm|mov|m4v)$/.test(path)) return "video";
return "image";
}
}

24
src/engines/Moebooru.test.ts
View File

@ -1,4 +1,4 @@
import { scrapeUrl, scrapeEvalUrl } from "../test/engine.js"; import { scrapeUrl, scrapeEvalUrl } from "../test/engine";
test("get yande.re post 438510", async () => { test("get yande.re post 438510", async () => {
const pageUrl = "https://yande.re/post/show/438510"; const pageUrl = "https://yande.re/post/show/438510";
@ -21,7 +21,6 @@ test("get yande.re post 684478 with notes", async () => {
expect(res.posts[0]).toHaveTag("gym_uniform"); expect(res.posts[0]).toHaveTag("gym_uniform");
expect(res.posts[0]).toHaveNote("post #671879"); expect(res.posts[0]).toHaveNote("post #671879");
expect(res.posts[0]).toHaveResolution([2480, 3508]); expect(res.posts[0]).toHaveResolution([2480, 3508]);
expect(res.posts[0]).toHaveSource("https://www.pixiv.net/artworks/84228323");
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe( expect(res.posts[0].contentUrl).toBe(
"https://files.yande.re/image/8bdf7c54b0a459acee080f0aeb2a259c/yande.re%20684478%20gym_uniform%20pal_lett.jpg" "https://files.yande.re/image/8bdf7c54b0a459acee080f0aeb2a259c/yande.re%20684478%20gym_uniform%20pal_lett.jpg"
@ -34,7 +33,6 @@ test("get konachan.com post 289968", async () => {
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("league_of_legends", "copyright"); expect(res.posts[0]).toHaveTag("league_of_legends", "copyright");
expect(res.posts[0]).toHaveResolution([1200, 783]); expect(res.posts[0]).toHaveResolution([1200, 783]);
expect(res.posts[0]).toHaveSource("https://www.pixiv.net/member_illust.php?mode=medium&illust_id=72418059");
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe( expect(res.posts[0].contentUrl).toBe(
"https://konachan.com/image/e40b38fb704434e4da3eae41e26bca39/Konachan.com%20-%20289968%20akali%20blue_eyes%20breasts%20choker%20cleavage%20evelynn%20foxgirl%20gloves%20group%20hat%20kai%27sa%20long_hair%20microphone%20ponytail%20saphirya%20short_hair%20signed%20watermark.jpg" "https://konachan.com/image/e40b38fb704434e4da3eae41e26bca39/Konachan.com%20-%20289968%20akali%20blue_eyes%20breasts%20choker%20cleavage%20evelynn%20foxgirl%20gloves%20group%20hat%20kai%27sa%20long_hair%20microphone%20ponytail%20saphirya%20short_hair%20signed%20watermark.jpg"
@ -51,7 +49,6 @@ test("get konachan.com post 327454 with notes", async () => {
expect(res.posts[0]).toHaveTag("water"); expect(res.posts[0]).toHaveTag("water");
expect(res.posts[0]).toHaveNote("Under construction. \nBe careful of your safety"); expect(res.posts[0]).toHaveNote("Under construction. \nBe careful of your safety");
expect(res.posts[0]).toHaveResolution([5629, 2505]); expect(res.posts[0]).toHaveResolution([5629, 2505]);
expect(res.posts[0]).toHaveSource("https://www.pixiv.net/en/artworks/86644963");
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe( expect(res.posts[0].contentUrl).toBe(
"https://konachan.com/image/94a13c8a1fcb9c0f6c9a6c6c5c316259/Konachan.com%20-%20327454%20apron%20brown_hair%20building%20car%20gloves%20kexue%20long_hair%20original%20rain%20see_through%20short_hair%20twintails%20uniform%20water.jpg" "https://konachan.com/image/94a13c8a1fcb9c0f6c9a6c6c5c316259/Konachan.com%20-%20327454%20apron%20brown_hair%20building%20car%20gloves%20kexue%20long_hair%20original%20rain%20see_through%20short_hair%20twintails%20uniform%20water.jpg"
@ -74,23 +71,4 @@ test("get konachan.com post 285782 with notes", async () => {
expect(res.posts[0].contentUrl).toBe( expect(res.posts[0].contentUrl).toBe(
"https://konachan.com/image/24f5974ad8ed725292ccb3f682181bc1/Konachan.com%20-%20285782%20anthropomorphism%20bed%20black_eyes%20blonde_hair%20cropped%20fang%20kantai_collection%20long_hair%20momoniku%20pajamas%20white%20yuudachi_%28kancolle%29.jpg" "https://konachan.com/image/24f5974ad8ed725292ccb3f682181bc1/Konachan.com%20-%20285782%20anthropomorphism%20bed%20black_eyes%20blonde_hair%20cropped%20fang%20kantai_collection%20long_hair%20momoniku%20pajamas%20white%20yuudachi_%28kancolle%29.jpg"
); );
expect(res.posts[0]).toHaveSource("https://www.pixiv.net/member_illust.php?mode=medium&illust_id=75747354");
});
test("get konachan.com post 358247 jpeg and png", async () => {
const pageUrl = "https://konachan.com/post/show/358247/3d-animal_ears-arknights-black_hair-blonde_hair-br";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(2);
expect(res.posts[0]).toHaveTag("texas_(arknights)", "character");
expect(res.posts[0]).toHaveResolution([6144, 3456]);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://konachan.com/image/56983a2166a13f4af963774ccd424f0e/Konachan.com%20-%20358247%203d%20arknights%20computer%20couch%20dress%20drink%20gloves%20gun%20halo%20hood%20necklace%20pantyhose%20red_eyes%20red_hair%20shirt%20shorts%20skirt%20tie%20twintails%20weapon%20wolfgirl.png"
);
expect(res.posts[0]).toHaveSource("https://www.pixiv.net/en/artworks/108925945");
// JPEG variant
expect(res.posts[1].contentUrl).toBe(
"https://konachan.com/jpeg/56983a2166a13f4af963774ccd424f0e/Konachan.com%20-%20358247%203d%20arknights%20computer%20couch%20dress%20drink%20gloves%20gun%20halo%20hood%20necklace%20pantyhose%20red_eyes%20red_hair%20shirt%20shorts%20skirt%20tie%20twintails%20weapon%20wolfgirl.jpg"
);
}); });

47
src/engines/Moebooru.ts
View File

@ -1,22 +1,19 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js"; import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapedNote } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes.js"; import { TagCategory } from "../BooruTypes";
import { createNotesFromMoebooruBoxes, guessContentType, parseResolutionString } from "../Utility.js"; import { createNotesFromMoebooruBoxes, guessContentType, parseResolutionString } from "../Utility";
export default class Moebooru extends ScrapeEngineBase { export default class Moebooru extends ScrapeEngineBase {
name = "moebooru"; name = "moebooru";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
notes = []; canImport(url: Location): boolean {
supportedHosts = ["yande.re", "konachan.com", "konachan.net", "sakugabooru.com", "img.genshiken-itb.org"]; return url.host == "yande.re" || url.host == "konachan.com";
}
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); let result = new ScrapeResult(this.name);
const post = new ScrapedPost(); let post = new ScrapedPost();
post.pageUrl = document.location.href; post.pageUrl = document.location.href;
if (document.location.host == "yande.re") {
post.uploadMode = "content";
}
// Set image url // Set image url
const originalImageElement = document.querySelector("#highres") as HTMLAnchorElement; const originalImageElement = document.querySelector("#highres") as HTMLAnchorElement;
if (originalImageElement) { if (originalImageElement) {
@ -38,8 +35,8 @@ export default class Moebooru extends ScrapeEngineBase {
const matches = el.innerText.match(regex); const matches = el.innerText.match(regex);
if (matches && matches.length == 3) { if (matches && matches.length == 3) {
// If Source element // If Source element
if (matches[1] == "Source" && el.children.length > 0) { if (matches[1] == "Source") {
post.sources = [(el.children[0] as HTMLAnchorElement).href]; post.source = (el.children[0] as HTMLAnchorElement).href;
} }
// If Rating element // If Rating element
else if (matches[1] == "Rating") { else if (matches[1] == "Rating") {
@ -67,14 +64,12 @@ export default class Moebooru extends ScrapeEngineBase {
for (const el of tagElements) { for (const el of tagElements) {
let tagName: string; let tagName: string;
const tagNameElements = el.getElementsByTagName("a"); let tagNameElements = el.getElementsByTagName("a");
if (tagNameElements.length > 1) { if (tagNameElements.length > 1) {
// First <a> is "?" // First <a> is "?"
// (opt) Second <a> is "+" (when logged in) // Second <a> is "actual_tag_name"
// (opt) Third <a> is "-" (when logged in) tagName = tagNameElements[1].innerText;
// Last <a> is the actual tag name
tagName = tagNameElements[tagNameElements.length - 1].innerText;
} else { } else {
// Skip if no tag name is found // Skip if no tag name is found
continue; continue;
@ -101,7 +96,7 @@ export default class Moebooru extends ScrapeEngineBase {
} }
} }
const tag = new ScrapedTag(tagName, category); let tag = new ScrapedTag(tagName, category);
post.tags.push(tag); post.tags.push(tag);
} }
@ -114,18 +109,6 @@ export default class Moebooru extends ScrapeEngineBase {
post.notes = createNotesFromMoebooruBoxes(document, boxSize); post.notes = createNotesFromMoebooruBoxes(document, boxSize);
} }
// Sometimes there is both a JPEG and a PNG version of a post.
// If this is the case, copy all the information from the JPEG
// version and only replace the contentUrl with the link to the PNG version.
// We want to add this PNG version before the other version, so that it appears on top.
const unchangedImgEl = document.querySelector<HTMLAnchorElement>("a.original-file-unchanged");
if (unchangedImgEl && unchangedImgEl.href != post.contentUrl) {
const copy = Object.assign(new ScrapedPost(), post);
copy.contentUrl = unchangedImgEl.href;
result.tryAddPost(copy);
}
// Add normal/non-png post.
result.tryAddPost(post); result.tryAddPost(post);
return result; return result;

49
src/engines/Nozomi.ts
View File

@ -1,49 +0,0 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
import { TagCategory } from "../BooruTypes.js";
export default class e621 extends ScrapeEngineBase {
name = "nozomi";
features: ScrapeEngineFeature[] = ["content", "tags", "tag_category"];
notes = ["Image preview might not work due to hotlink protection."];
supportedHosts = ["nozomi.la"];
scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name);
const post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url
const downloadEl = document.querySelector(".post a") as HTMLAnchorElement;
post.contentUrl = downloadEl?.href;
// Set tags
const tagEls = Array.from(document.querySelectorAll(".sidebar li")).map((x) => x as HTMLLIElement);
for (const el of tagEls) {
const tagName = el.innerText;
let category: TagCategory | undefined;
switch (el.querySelector("a")?.className) {
// https://e621.net/help/api#tags_listing
case "copyright":
category = "copyright";
break;
case "character":
category = "character";
break;
case "artist":
category = "artist";
break;
}
if (tagName) {
const tag = new ScrapedTag(tagName, category);
post.tags.push(tag);
}
}
result.tryAddPost(post);
return result;
}
}

43
src/engines/Philomena.test.ts
View File

@ -1,43 +0,0 @@
import { scrapeUrl } from "../test/engine.js";
test("get derpibooru.org post 3153524", async () => {
const pageUrl = "https://derpibooru.org/images/3153524";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].rating).toBe("safe");
expect(res.posts[0]).not.toHaveTag("safe");
expect(res.posts[0]).toHaveTag("yay");
expect(res.posts[0]).toHaveTag("flutteryay");
expect(res.posts[0]).toHaveSource("https://twitter.com/Confetti_Cakez/status/1674145455666544640");
expect(res.posts[0]).toHaveResolution([4000, 4000]);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe("https://derpicdn.net/img/download/2023/6/29/3153524.jpg");
});
test("get derpibooru.org post 3157356", async () => {
const pageUrl = "https://derpibooru.org/images/3157356?q=oc";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].rating).toBe("safe");
expect(res.posts[0]).not.toHaveTag("safe");
expect(res.posts[0]).toHaveTag("oc");
expect(res.posts[0]).toHaveTag("serra20", "artist");
expect(res.posts[0]).toHaveTag("pony", "species");
expect(res.posts[0]).toHaveTag("eris_(oc)", "character");
expect(res.posts[0]).toHaveResolution([960, 960]);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe("https://derpicdn.net/img/download/2023/7/5/3157356.png");
});
test("get derpibooru.org video post 3161674", async () => {
const pageUrl = "https://derpibooru.org/images/3161674?q=video";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].rating).toBe("safe");
expect(res.posts[0]).not.toHaveTag("safe");
expect(res.posts[0]).toHaveTag("oc");
expect(res.posts[0]).toHaveTag("rainbow_dash", "character");
expect(res.posts[0]).toHaveResolution([1920, 1080]);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe("https://derpicdn.net/img/download/2023/7/12/3161674.webm");
});

127
src/engines/Philomena.ts
View File

@ -1,127 +0,0 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js";
import { SafetyRating, TagCategory } from "../BooruTypes.js";
import { parseResolutionString } from "../Utility.js";
export default class Philomena extends ScrapeEngineBase {
name = "Philomena";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source"];
notes = [];
supportedHosts = [
"derpibooru.org",
"trixiebooru.org",
"ponybooru.org",
"furbooru.org",
"ponerpics.org",
"manebooru.art",
"twibooru.org",
];
removePrefixes = ["artist", "editor", "character", "species", "oc", "ship"];
removePrefixesSuffix = new Map([["oc", "_(oc)"]]);
artistPrefixes = ["artist", "editor"];
safetyMap = new Map<string, SafetyRating>([
["safe", "safe"],
["suggestive", "sketchy"],
["questionable", "sketchy"],
["explicit", "unsafe"],
]);
categoryMap = new Map<string, TagCategory>([
["character", "character"],
["species", "species"],
["oc", "character"],
["content-official", "copyright"],
]);
scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name);
const post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url
const downloadEl = document.querySelector("a[title='Download (no tags in filename)']") as HTMLAnchorElement;
post.contentUrl = downloadEl?.href;
// Set content type
if (document.querySelector("video#image-display") != null) {
post.contentType = "video";
}
// Set default rating (Will be determined later)
post.rating = "unsafe";
// Set resolution
const resStr = (<HTMLElement>document.querySelector("span[class='image-size']"))?.innerText?.match(
/(\d+x\d+)/
)?.[0];
if (resStr) {
const res = parseResolutionString(resStr);
if (res) {
post.resolution = res;
}
}
// Set tags
const tagEls = Array.from(document.querySelectorAll(".tag.dropdown")).map((x) => x as HTMLSpanElement);
for (const el of tagEls) {
let tagName = el.getAttribute("data-tag-name");
if (!tagName) continue;
// Set rating (philomena uses tags for this)
// These tags also have `data-tag-category="rating"` which we could use.
const safety = this.safetyMap.get(tagName);
if (safety != undefined) {
post.rating = safety;
// Don't add this as a tag.
continue;
}
let category: TagCategory | undefined;
// Categorize tags using data-tag-category
const dataTagCategory = el.getAttribute("data-tag-category");
if (dataTagCategory) {
const mappedCategory = this.categoryMap.get(dataTagCategory);
if (mappedCategory) {
category = mappedCategory;
}
}
// Artist tag categorize using prefixes
for (const prefix of this.artistPrefixes) {
if (tagName.includes(prefix + ":")) {
category = "artist";
break;
}
}
// Remove listed prefixes (removePrefixes) from tagName
for (const prefix of this.removePrefixes) {
if (tagName.includes(prefix + ":")) {
tagName = tagName.replace(prefix + ":", "").trim();
// Add suffix if configured
const suffix = this.removePrefixesSuffix.get(prefix);
if (suffix) {
tagName = tagName + suffix;
}
break;
}
}
// Push tag
if (tagName) {
const tag = new ScrapedTag(tagName, category);
post.tags.push(tag);
}
}
// Set source
const sourceEls = Array.from(document.querySelectorAll("a.js-source-link")).map((x) => x as HTMLAnchorElement);
post.sources = sourceEls.map((x) => x.href);
result.tryAddPost(post);
return result;
}
}

89
src/engines/Pixiv.test.ts
View File

@ -1,18 +1,5 @@
import { ScrapeResults } from "../ScrapeEngine.js"; import { ScrapeResults } from "../ScrapeEngine";
import { MobileUserAgent, scrapeEvalUrl, scrapePage } from "../test/engine.js"; import { scrapeEvalUrl } from "../test/engine";
import { upgradeUrlToOriginalQuality } from "./Pixiv.js";
test("upgradeUrlToOriginalQuality", async () => {
expect(
upgradeUrlToOriginalQuality("https://i.pximg.net/img-master/img/2022/02/15/06/50/28/96268450_p2_master1200.jpg")
).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p2.jpg");
expect(
upgradeUrlToOriginalQuality(
"https://i.pximg.net/c/600x1200_90_webp/img-master/img/2022/02/15/06/50/28/96268450_p2_master1200.jpg"
)
).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p2.jpg");
});
describe("get pixiv post 93964654", () => { describe("get pixiv post 93964654", () => {
const pageUrl = "https://www.pixiv.net/en/artworks/93964654"; const pageUrl = "https://www.pixiv.net/en/artworks/93964654";
@ -22,72 +9,10 @@ describe("get pixiv post 93964654", () => {
res = await scrapeEvalUrl(pageUrl, ".gtm-expand-full-size-illust"); res = await scrapeEvalUrl(pageUrl, ".gtm-expand-full-size-illust");
}); });
it("should find exactly one post", () => { it("should grab original image url", async () => {
expect(res.posts.length).toBe(1); pending("Test does not work.");
}); // expect(res.posts.length).toBe(1);
// expect(res.posts[0].pageUrl).toBe(pageUrl);
it("should grab original image url", () => { // expect(res.posts[0].contentUrl).toBe("https://i.pximg.net/img-original/img/2021/11/07/00/01/02/93964654_p0.jpg");
expect(res.posts[0].contentUrl).toBe("https://i.pximg.net/img-original/img/2021/11/07/00/01/02/93964654_p0.jpg");
});
});
describe("get pixiv post 93964654 (mobile)", () => {
const pageUrl = "https://www.pixiv.net/en/artworks/93964654";
let res: ScrapeResults;
beforeAll(async () => {
res = await scrapeEvalUrl(pageUrl, ".work-thumb", MobileUserAgent);
});
it("should find exactly one post", () => {
expect(res.posts.length).toBe(1);
});
it("should grab original image url", () => {
expect(res.posts[0].contentUrl).toBe("https://i.pximg.net/img-original/img/2021/11/07/00/01/02/93964654_p0.jpg");
});
});
describe("get pixiv post 96268450 (multipage, not expanded)", () => {
const pageUrl = "https://www.pixiv.net/en/artworks/96268450";
let res: ScrapeResults;
beforeAll(async () => {
res = await scrapeEvalUrl(pageUrl, "div[role='presentation'] img");
});
it("should find exactly one post", () => {
expect(res.posts.length).toBe(1);
});
it("should grab original image url", () => {
expect(res.posts[0].contentUrl).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p0.jpg");
});
});
describe("get pixiv post 96268450 (multipage, expanded)", () => {
const pageUrl = "https://www.pixiv.net/en/artworks/96268450";
let res: ScrapeResults;
beforeAll(async () => {
await jestPuppeteer.resetPage();
await page.goto(pageUrl);
await page.waitForSelector("div[role='presentation'] img", { visible: true });
await expect(page).toClick("button", { text: "Show all" });
await page.waitForSelector(".gtm-expand-full-size-illust", { visible: true });
// await jestPuppeteer.debug();
await page.waitForTimeout(2000);
res = await scrapePage(pageUrl, page);
});
it("should find exactly four posts", () => {
expect(res.posts.length).toBe(4);
});
it("should grab original image url", () => {
expect(res.posts[0].contentUrl).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p0.jpg");
expect(res.posts[1].contentUrl).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p1.jpg");
expect(res.posts[2].contentUrl).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p2.jpg");
expect(res.posts[3].contentUrl).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p3.jpg");
}); });
}); });

76
src/engines/Pixiv.ts
View File

@ -1,78 +1,28 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapeEngineFeature } from "../ScrapeEngine.js"; import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { guessContentType } from "../Utility.js"; import { TagCategory } from "../BooruTypes";
import { guessContentType } from "../Utility";
export function upgradeUrlToOriginalQuality(imgLink: string) { export default class Pixiv implements ScrapeEngine {
if (imgLink.indexOf("img-master") != -1) {
// Example:
// Input: https://i.pximg.net/img-master/img/2022/02/15/06/50/28/96268450_p2_master1200.jpg
// Input: https://i.pximg.net/c/600x1200_90_webp/img-master/img/2022/02/15/06/50/28/96268450_p2_master1200.jpg
// Output: https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p2.jpg
//
// Warning:
// This will NOT work for a srcset like this:
// https://i.pximg.net/c/540x540_70/img-master/img/2022/02/15/06/50/28/96268450_p0_master1200.jpg 540w,https://i.pximg.net/img-master/img/2022/02/15/06/50/28/96268450_p0_master1200.jpg
//
// See regexr.com/6rcag for an example.
// TODO: Add a check to verify the newly constructed/guessed url?
return imgLink.replace(/(https?:\/\/i\.pximg\.net).*\/img-master\/(.*)_master\d+(.*)/, "$1/img-original/$2$3");
}
return imgLink;
}
export default class Pixiv extends ScrapeEngineBase {
name = "pixiv"; name = "pixiv";
features: ScrapeEngineFeature[] = ["content"];
notes = []; canImport(url: Location): boolean {
supportedHosts = ["pixiv.net"]; return url.host == "pixiv.net" || url.host == "www.pixiv.net";
}
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); let result = new ScrapeResult(this.name);
const imgLinks = [] as string[];
// Query for when the user is on the desktop site, is logged in, and has optionally clicked on "Show all". const imgLinks = Array.from(document.querySelectorAll("div[role='presentation'] > a")).map(
// If the user has not clicked on "Show all" then this will only grab the first post. (x) => x as HTMLAnchorElement
imgLinks.push(
...Array.from(document.querySelectorAll<HTMLAnchorElement>("div[role='presentation'] > a")).map((x) => x.href)
); );
if (imgLinks.length == 0) {
// Query for when the user is on the desktop site and not logged in.
imgLinks.push(
...Array.from(document.querySelectorAll<HTMLImageElement>("div[role='presentation'] img")).map((x) => x.src)
);
}
if (imgLinks.length == 0) {
// Query for when the user is on the mobile site, and has clicked on "Show all".
// This only works on pages with multiple posts. The user can optionally be logged in.
imgLinks.push(
...Array.from(document.querySelectorAll<HTMLImageElement>(".manga-page > a > img[data-big]")).map(
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
(x) => x.getAttribute("data-big")!
)
);
}
if (imgLinks.length == 0) {
// Query for when the user is on the mobile site, and has not clicked on "See all" -or- there is only one post.
const imgLink = document.querySelector<HTMLImageElement>(".work-main-image img")?.src;
if (imgLink) {
imgLinks.push(imgLink);
}
}
for (const imgLink of imgLinks) { for (const imgLink of imgLinks) {
const post = new ScrapedPost(); let post = new ScrapedPost();
post.pageUrl = document.location.href; post.pageUrl = document.location.href;
post.contentUrl = upgradeUrlToOriginalQuality(imgLink); post.contentUrl = imgLink.href;
post.contentType = guessContentType(post.contentUrl); post.contentType = guessContentType(post.contentUrl);
post.referrer = document.location.origin; post.referrer = document.location.origin;
result.tryAddPost(post); result.tryAddPost(post);
if (document.querySelector('a[href*="/tags/R-18/artworks"]')) {
post.rating = "unsafe";
}
} }
return result; return result;

2
src/engines/Reddit.test.ts
View File

@ -1,4 +1,4 @@
import { scrapeUrl } from "../test/engine.js"; import { scrapeUrl } from "../test/engine";
test("get old.reddit.com post", async () => { test("get old.reddit.com post", async () => {
const pageUrl = "https://old.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/"; const pageUrl = "https://old.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/";

17
src/engines/Reddit.ts
View File

@ -1,15 +1,16 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapeEngineFeature } from "../ScrapeEngine.js"; import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine";
import { guessContentType } from "../Utility.js"; import { guessContentType } from "../Utility";
export default class Reddit extends ScrapeEngineBase { export default class Reddit implements ScrapeEngine {
name = "reddit"; name = "reddit";
features: ScrapeEngineFeature[] = ["content"];
notes = []; canImport(url: Location): boolean {
supportedHosts = ["reddit.com", "new.reddit.com", "old.reddit.com"]; return url.host.endsWith("reddit.com");
}
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); let result = new ScrapeResult(this.name);
const post = new ScrapedPost(); let post = new ScrapedPost();
post.pageUrl = document.location.href; post.pageUrl = document.location.href;
// Set image url // Set image url

2
src/engines/Rule34us.test.ts
View File

@ -1,4 +1,4 @@
import { scrapeUrl } from "../test/engine.js"; import { scrapeUrl } from "../test/engine";
test("get rule34.us post 4646579", async () => { test("get rule34.us post 4646579", async () => {
// Kinda sketchy. // Kinda sketchy.

22
src/engines/Rule34us.ts
View File

@ -1,16 +1,13 @@
// Technically speaking rule34.us also uses gelbooru, albeit a (heavily?) modified one. // Technically speaking rule34.us also uses gelbooru, albeit a (heavily?) modified one.
// Because pretty much no scraper code can be shared with other gelbooru-likes it makes more sense to create a new engine for rule34.us. // Because pretty much no scraper code can be shared with other gelbooru-likes it makes more sense to create a new engine for rule34.us.
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js"; import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes.js"; import { TagCategory } from "../BooruTypes";
import { CategoryMap } from "./Common.js"; import { CategoryMap } from "./Common";
import { guessContentType, parseResolutionString } from "../Utility.js"; import { guessContentType, parseResolutionString } from "../Utility";
export default class Gelbooru extends ScrapeEngineBase { export default class Gelbooru implements ScrapeEngine {
name = "rule34us"; name = "rule34us";
features: ScrapeEngineFeature[] = ["content", "resolution", "tags", "tag_category"];
notes = ["Rating is assumed to be unsafe."];
supportedHosts = ["rule34.us"];
private readonly classNameToCategoryMap: CategoryMap = { private readonly classNameToCategoryMap: CategoryMap = {
"character-tag": "character", "character-tag": "character",
@ -19,9 +16,14 @@ export default class Gelbooru extends ScrapeEngineBase {
"metadata-tag": "meta", "metadata-tag": "meta",
}; };
canImport(url: Location): boolean {
return url.host == "rule34.us";
}
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); let result = new ScrapeResult(this.name);
const post = new ScrapedPost();
let post = new ScrapedPost();
post.pageUrl = document.location.href; post.pageUrl = document.location.href;
// Set image url // Set image url

32
src/engines/SankakuComplex.test.ts
View File

@ -1,15 +1,37 @@
import { scrapeUrl } from "../test/engine.js"; import { scrapeUrl } from "../test/engine";
test("get chan.sankakucomplex.com post 19614481 with notes", async () => { test("get chan.sankakucomplex.com post 19614481 with notes", async () => {
const pageUrl = "https://chan.sankakucomplex.com/en/posts/porqpNvZBa7"; const pageUrl = "https://chan.sankakucomplex.com/post/show/19614481";
const res = await scrapeUrl(pageUrl); const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("bangs"); expect(res.posts[0]).toHaveTag("bangs");
expect(res.posts[0]).toHaveTag("bare_shoulders"); expect(res.posts[0]).toHaveTag("bare_shoulders");
expect(res.posts[0]).toHaveTag("solo");
expect(res.posts[0]).toHaveTag("translated");
expect(res.posts[0]).toHaveTag("text");
expect(res.posts[0]).toHaveTag("idolmaster"); expect(res.posts[0]).toHaveTag("idolmaster");
expect(res.posts[0]).toHaveTag("mayuzumi_fuyuko"); expect(res.posts[0]).toHaveTag("mayuzumi_fuyuko");
// expect(res.posts[0]).toHaveNote("What are you looking at..."); expect(res.posts[0]).toHaveNote("What are you looking at...");
// expect(res.posts[0]).toHaveNote("What...?"); expect(res.posts[0]).toHaveNote("What...?");
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toContain("sankakucomplex.com/data/50/7e/507e986a62cc5b866d92ad491dcfb1a4.jpg"); expect(res.posts[0].contentUrl).toContain(
"https://s.sankakucomplex.com/data/50/7e/507e986a62cc5b866d92ad491dcfb1a4.jpg"
);
});
test("get chan.sankakucomplex.com post 29354061 with notes", async () => {
// Kinda sketchy
const pageUrl = "https://chan.sankakucomplex.com/post/show/29354061";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("bangs");
expect(res.posts[0]).toHaveTag("braid");
expect(res.posts[0]).toHaveTag("sankuro_(agoitei)");
expect(res.posts[0]).toHaveTag("translated");
expect(res.posts[0]).toHaveTag("note_translated");
expect(res.posts[0]).toHaveNote("No. \nI already told you...");
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toContain(
"https://s.sankakucomplex.com/data/c8/37/c8376f02608548365d394a658e85cb3f.jpg"
);
}); });

40
src/engines/SankakuComplex.ts
View File

@ -1,16 +1,17 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js"; import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapedNote } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes.js"; import { TagCategory } from "../BooruTypes";
import { createNotesFromMoebooruBoxes, guessContentType } from "../Utility.js"; import { createNotesFromMoebooruBoxes, guessContentType } from "../Utility";
export default class SankakuComplex extends ScrapeEngineBase { export default class SankakuComplex extends ScrapeEngineBase {
name = "sankakucomplex"; name = "sankakucomplex";
features: ScrapeEngineFeature[] = ["content", "rating", "tags", "tag_category", "notes"];
notes = []; canImport(url: Location): boolean {
supportedHosts = ["chan.sankakucomplex.com"]; return url.host == "chan.sankakucomplex.com";
}
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); let result = new ScrapeResult(this.name);
const post = new ScrapedPost(); let post = new ScrapedPost();
post.pageUrl = document.location.href; post.pageUrl = document.location.href;
// Set image url // Set image url
@ -27,14 +28,25 @@ export default class SankakuComplex extends ScrapeEngineBase {
post.contentType = guessContentType(post.contentUrl); post.contentType = guessContentType(post.contentUrl);
// Set rating // Set rating
const ratingElement = document.querySelector("#stats .rating-s, #stats .rating-q, #stats .rating-e"); const safetyExp = new RegExp("Rating: (.*)");
if (ratingElement) { const safetyElements = Array.from(document.querySelectorAll("#stats > ul > li"))
if (ratingElement.classList.contains("rating-s")) { .map((x) => x as HTMLLIElement)
.filter((x) => x.innerText.startsWith("Rating"));
if (safetyElements.length > 0) {
const matches = safetyElements[0].innerText.match(safetyExp);
if (matches && matches.length > 0) {
switch (matches[1].toLowerCase()) {
case "safe":
post.rating = "safe"; post.rating = "safe";
} else if (ratingElement.classList.contains("rating-q")) { break;
case "questionable":
post.rating = "sketchy"; post.rating = "sketchy";
} else if (ratingElement.classList.contains("rating-e")) { break;
case "explicit":
post.rating = "unsafe"; post.rating = "unsafe";
break;
}
} }
} }
@ -64,7 +76,7 @@ export default class SankakuComplex extends ScrapeEngineBase {
} }
if (tagName) { if (tagName) {
const tag = new ScrapedTag(tagName, category); let tag = new ScrapedTag(tagName, category);
post.tags.push(tag); post.tags.push(tag);
} }
} }

8
src/engines/Shimmie2.test.ts
View File

@ -1,13 +1,11 @@
import { scrapeUrl } from "../test/engine.js"; import { scrapeUrl } from "../test/engine";
test("get rule34.paheal.net post 2505982", async () => { test("get rule34.paheal.net post 2505982", async () => {
const pageUrl = "https://rule34.paheal.net/post/view/2505982"; // Somewhat sketchy const pageUrl = "https://rule34.paheal.net/post/view/2505982"; // Somewhat sketchy
const res = await scrapeUrl(pageUrl); const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("kai'sa");
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0]).toHaveSource( expect(res.posts[0].contentUrl).toContain(
"http://law-zilla.tumblr.com/post/171249285285/finished-subdraw-37-kaisa-from-league-of-legends" ".paheal.net/_images/15427ab5ed2cddbeef947e30f5c9cab6/2505982%20-%20Kai%27Sa%20League_of_Legends%20lawzilla.png"
); );
expect(res.posts[0].contentUrl).toContain("https://r34i.paheal-cdn.net");
}); });

44
src/engines/Shimmie2.ts
View File

@ -1,33 +1,31 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js"; import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { guessContentType } from "../Utility.js"; import { guessContentType } from "../Utility";
export default class Shimmie2 extends ScrapeEngineBase { export default class Shimmie2 implements ScrapeEngine {
name = "shimmie2"; name = "shimmie2";
features: ScrapeEngineFeature[] = ["content", "tags", "source"];
notes = ["Rating is assumed to be unsafe."]; canImport(url: Location): boolean {
supportedHosts = ["rule34.paheal.net", "rule34hentai.net"]; return url.host == "rule34.paheal.net" || url.host == "rule34hentai.net";
}
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); let result = new ScrapeResult(this.name);
const post = new ScrapedPost(); let post = new ScrapedPost();
post.pageUrl = document.location.href; post.pageUrl = document.location.href;
// Set image url // Set image url
// const originalImageElements = Array.from(document.querySelectorAll(".image_info > tbody > tr > td > a")) const originalImageElements = Array.from(document.querySelectorAll(".image_info > tbody > tr > td > a"))
// .map((x) => x as HTMLAnchorElement) .map((x) => x as HTMLAnchorElement)
// .filter((x) => x.innerText == "Image Only" || x.innerText == "File Only"); .filter((x) => x.innerText == "Image Only" || x.innerText == "File Only");
// if (originalImageElements.length > 0) { if (originalImageElements.length > 0) {
// post.contentUrl = originalImageElements[0].href; post.contentUrl = originalImageElements[0].href;
// } else { } else {
// const dowloadBtnEl = document.querySelector("a[download]") as HTMLAnchorElement; const dowloadBtnEl = document.querySelector("a[download]") as HTMLAnchorElement;
// if (dowloadBtnEl) { if (dowloadBtnEl) {
// post.contentUrl = dowloadBtnEl.href; post.contentUrl = dowloadBtnEl.href;
// } }
// } }
// @ts-expect-error
post.contentUrl = document.getElementById("main_image").src;
// Set content type // Set content type
post.contentType = guessContentType(post.contentUrl); post.contentType = guessContentType(post.contentUrl);
@ -47,7 +45,7 @@ export default class Shimmie2 extends ScrapeEngineBase {
// Set source // Set source
const sourceEl = document.querySelector(".image_info > tbody > tr > td > div > a") as HTMLAnchorElement; const sourceEl = document.querySelector(".image_info > tbody > tr > td > div > a") as HTMLAnchorElement;
if (sourceEl) { if (sourceEl) {
post.sources = [sourceEl.href]; post.source = sourceEl.href;
} }
result.tryAddPost(post); result.tryAddPost(post);

2
src/engines/Shuushuu.test.ts
View File

@ -1,4 +1,4 @@
import { scrapeUrl } from "../test/engine.js"; import { scrapeUrl } from "../test/engine";
test("get e-shuushuu.net post 1063586", async () => { test("get e-shuushuu.net post 1063586", async () => {
const pageUrl = "https://e-shuushuu.net/image/1063586/"; const pageUrl = "https://e-shuushuu.net/image/1063586/";

19
src/engines/Shuushuu.ts
View File

@ -1,16 +1,17 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js"; import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes.js"; import { TagCategory } from "../BooruTypes";
import { guessContentType, parseResolutionString } from "../Utility.js"; import { guessContentType, parseResolutionString } from "../Utility";
export default class Shuushuu extends ScrapeEngineBase { export default class Shuushuu implements ScrapeEngine {
name = "shuushuu"; name = "shuushuu";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category"];
notes = ["Rating is assumed to be safe."]; canImport(url: Location): boolean {
supportedHosts = ["e-shuushuu.net"]; return url.host == "e-shuushuu.net";
}
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); let result = new ScrapeResult(this.name);
const post = new ScrapedPost(); let post = new ScrapedPost();
post.pageUrl = document.location.href; post.pageUrl = document.location.href;
// Set image url // Set image url

4
src/engines/Twitter.test.ts
View File

@ -1,5 +1,5 @@
import { ScrapeResults } from "../ScrapeEngine.js"; import { ScrapeResults } from "../ScrapeEngine";
import { scrapeEvalUrl } from "../test/engine.js"; import { scrapeEvalUrl } from "../test/engine";
describe("get twitter.com post", () => { describe("get twitter.com post", () => {
const pageUrl = "https://twitter.com/Dishwasher1910/status/1171162651461746688/photo/1"; const pageUrl = "https://twitter.com/Dishwasher1910/status/1171162651461746688/photo/1";

19
src/engines/Twitter.ts
View File

@ -1,29 +1,30 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapeEngineFeature } from "../ScrapeEngine.js"; import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine";
export function getOriginalImageUrl(image: HTMLImageElement) { export function getOriginalImageUrl(image: HTMLImageElement) {
// if the link includes a resolution, change it to orig: // if the link includes a resolution, change it to orig:
// https://pbs.twimg.com/media/stuff?format=jpg&name=900x900 // https://pbs.twimg.com/media/stuff?format=jpg&name=900x900
// -> https://pbs.twimg.com/media/stuff?format=jpg&name=orig // -> https://pbs.twimg.com/media/stuff?format=jpg&name=orig
const img_url = new URL((image as HTMLImageElement).src); let img_url = new URL((image as HTMLImageElement).src);
if (img_url.searchParams.has("name") && img_url.searchParams.get("name") != "orig") { if (img_url.searchParams.has("name") && img_url.searchParams.get("name") != "orig") {
img_url.searchParams.set("name", "orig"); img_url.searchParams.set("name", "orig");
} }
return img_url; return img_url;
} }
export default class Twitter extends ScrapeEngineBase { export default class Twitter implements ScrapeEngine {
name = "twitter"; name = "twitter";
features: ScrapeEngineFeature[] = ["content"];
notes = []; canImport(url: Location): boolean {
supportedHosts = ["twitter.com", "mobile.twitter.com", "x.com"]; return url.host == "twitter.com" || url.host == "mobile.twitter.com";
}
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); let result = new ScrapeResult(this.name);
// first look for a focused image on desktop: // first look for a focused image on desktop:
const center_img = document.querySelector("div[role=dialog] div[role=dialog] div[aria-label=Image] img"); const center_img = document.querySelector("div[role=dialog] div[role=dialog] div[aria-label=Image] img");
if (center_img) { if (center_img) {
const post = new ScrapedPost(); let post = new ScrapedPost();
post.pageUrl = document.location.href; post.pageUrl = document.location.href;
post.contentUrl = getOriginalImageUrl(center_img as HTMLImageElement).href; post.contentUrl = getOriginalImageUrl(center_img as HTMLImageElement).href;
result.tryAddPost(post); result.tryAddPost(post);
@ -33,7 +34,7 @@ export default class Twitter extends ScrapeEngineBase {
const imgs = [...document.querySelectorAll("article a[role='link'] div[data-testid='tweetPhoto'] img")]; const imgs = [...document.querySelectorAll("article a[role='link'] div[data-testid='tweetPhoto'] img")];
for (const img of imgs) { for (const img of imgs) {
const post = new ScrapedPost(); let post = new ScrapedPost();
post.pageUrl = document.location.href; post.pageUrl = document.location.href;
post.contentUrl = getOriginalImageUrl(img as HTMLImageElement).href; post.contentUrl = getOriginalImageUrl(img as HTMLImageElement).href;
result.tryAddPost(post); result.tryAddPost(post);

10
src/engines/Zerochan.test.ts
View File

@ -1,4 +1,4 @@
import { scrapeUrl } from "../test/engine.js"; import { scrapeUrl } from "../test/engine";
test("get zerochan.net post 2934998", async () => { test("get zerochan.net post 2934998", async () => {
const pageUrl = "https://www.zerochan.net/2934998"; const pageUrl = "https://www.zerochan.net/2934998";
@ -13,11 +13,11 @@ test("get zerochan.net post 2934998", async () => {
}); });
test("get zerochan.net gif post 3511809", async () => { test("get zerochan.net gif post 3511809", async () => {
const pageUrl = "https://www.zerochan.net/4356451"; const pageUrl = "https://www.zerochan.net/3511809";
const res = await scrapeUrl(pageUrl); const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("honkai_star_rail"); expect(res.posts[0]).toHaveTag("rwby");
expect(res.posts[0]).toHaveResolution([2192, 1644]); expect(res.posts[0]).toHaveResolution([500, 150]);
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe("https://static.zerochan.net/Honkai.Star.Rail.full.4356451.jpg"); expect(res.posts[0].contentUrl).toBe("https://static.zerochan.net/Ruby.Rose.full.3511809.gif");
}); });

33
src/engines/Zerochan.ts
View File

@ -1,21 +1,22 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js"; import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes.js"; import { TagCategory } from "../BooruTypes";
import { guessContentType, parseResolutionString } from "../Utility.js"; import { guessContentType, parseResolutionString } from "../Utility";
export default class Zerochan extends ScrapeEngineBase { export default class Zerochan implements ScrapeEngine {
name = "zerochan"; name = "zerochan";
features: ScrapeEngineFeature[] = ["content", "resolution", "tags", "tag_category"];
notes = ["Rating is assumed to be safe."]; canImport(url: Location): boolean {
supportedHosts = ["zerochan.net"]; return url.host.endsWith("zerochan.net");
}
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); let result = new ScrapeResult(this.name);
const post = new ScrapedPost(); let post = new ScrapedPost();
post.pageUrl = document.location.href; post.pageUrl = document.location.href;
// Set image url // Set image url
const previewElements = document.getElementsByClassName("preview"); const previewElements = document.getElementsByClassName("preview");
if (previewElements.length > 0 && (previewElements[0] as HTMLAnchorElement).href) { if (previewElements.length > 0) {
// The preview element only exists when there is a larger size available. Preview's img element contains a downscaled version. // The preview element only exists when there is a larger size available. Preview's img element contains a downscaled version.
post.contentUrl = (previewElements[0] as HTMLAnchorElement).href; post.contentUrl = (previewElements[0] as HTMLAnchorElement).href;
} else { } else {
@ -30,9 +31,9 @@ export default class Zerochan extends ScrapeEngineBase {
post.contentType = guessContentType(post.contentUrl); post.contentType = guessContentType(post.contentUrl);
// Set resolution // Set resolution
const resSizeEl = document.querySelector("#image-info")?.firstElementChild; const resSizeEl = document.querySelector("#large > p > br")?.parentNode as HTMLParagraphElement;
if (resSizeEl) { if (resSizeEl) {
post.resolution = parseResolutionString(resSizeEl.textContent?.replace("×", "x")); post.resolution = parseResolutionString(resSizeEl.childNodes[0].textContent);
} }
// Set rating // Set rating
@ -52,26 +53,26 @@ export default class Zerochan extends ScrapeEngineBase {
} }
let category: TagCategory | undefined; let category: TagCategory | undefined;
const className = tagElement.classList[0]; switch (tagElement.className) {
switch (className) {
case "game": case "game":
case "series": case "series":
case "studio": case "studio":
category = "copyright"; category = "copyright";
break; break;
case "character": case "character":
case "character group":
category = "character"; category = "character";
break; break;
case "mangaka": case "mangaka":
category = "artist"; category = "artist";
break; break;
case "source": case "source":
// Skip these type of tags, because they are not 'booru-like'. // Skip these type of tags, because they are not 'booru-like'
continue; continue;
} }
if (tagName) { if (tagName) {
const tag = new ScrapedTag(tagName, category); let tag = new ScrapedTag(tagName, category);
post.tags.push(tag); post.tags.push(tag);
} }
} }

15
src/engines/e621.test.ts
View File

@ -1,15 +0,0 @@
import { scrapeUrl } from "../test/engine.js";
test("get e621.net post 380315", async () => {
const pageUrl = "https://e621.net/posts/380315?q=league_of_legends+rating%3Asafe+order%3Ascore";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("inkinesss", "artist");
expect(res.posts[0]).toHaveTag("nasus_(lol)", "character");
expect(res.posts[0]).toHaveTag("ambiguous_gender");
expect(res.posts[0]).toHaveTag("low_res", "meta");
expect(res.posts[0]).toHaveResolution([500, 500]);
expect(res.posts[0]).toHaveSource("https://inkinesss.deviantart.com/art/Nasus-388347996");
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe("https://static1.e621.net/data/92/e7/92e71bd73ab219d80516c4d9adad3db5.jpg");
});

52
src/engines/e621.ts
View File

@ -1,19 +1,21 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine.js"; import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes.js"; import { TagCategory } from "../BooruTypes";
export default class e621 extends ScrapeEngineBase { export default class e621 implements ScrapeEngine {
name = "e621"; name = "e621";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source"];
notes = []; canImport(url: Location): boolean {
supportedHosts = ["e621.net", "e926.net", "e6ai.net"]; return url.host == "e621.net" || url.host == "e926.net";
}
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
const result = new ScrapeResult(this.name); let result = new ScrapeResult(this.name);
const post = new ScrapedPost();
let post = new ScrapedPost();
post.pageUrl = document.location.href; post.pageUrl = document.location.href;
// Set image url // Set image url
const downloadEl = document.querySelector(".ptbr-fullscreen-button") as HTMLAnchorElement; const downloadEl = document.querySelector("#image-download-link > a") as HTMLAnchorElement;
post.contentUrl = downloadEl?.href; post.contentUrl = downloadEl?.href;
// Set content type // Set content type
@ -35,54 +37,50 @@ export default class e621 extends ScrapeEngineBase {
break; break;
} }
// Set resolution
const width = parseInt((<HTMLSpanElement>document.querySelector("span[itemprop='width']"))?.innerText);
const height = parseInt((<HTMLSpanElement>document.querySelector("span[itemprop='height']"))?.innerText);
if (!isNaN(width) && !isNaN(height)) {
post.resolution = [width, height];
}
// Set tags // Set tags
const tagEls = Array.from(document.querySelectorAll("#tag-list > ul > li")).map((x) => x as HTMLLIElement); const tagEls = Array.from(document.querySelectorAll("#tag-list > ul > li")).map((x) => x as HTMLLIElement);
for (const el of tagEls) { for (const el of tagEls) {
const tagName = decodeURIComponent(el.getAttribute("data-name") as string); const tagName = (el.querySelector(".search-tag") as HTMLAnchorElement)?.innerText;
let category: TagCategory | undefined; let category: TagCategory | undefined;
switch (el.getAttribute("data-category")) { switch (el.className) {
// https://e621.net/help/api#tags_listing // https://e621.net/help/api#tags_listing
case "copyright": case "category-3":
category = "copyright"; category = "copyright";
break; break;
case "character": case "category-4":
category = "character"; category = "character";
break; break;
case "artist": case "category-1":
category = "artist"; category = "artist";
break; break;
case "meta": case "category-7":
category = "meta"; category = "meta";
break; break;
case "species": case "category-5":
category = "species"; category = "species";
break; break;
// case "invalid": // case "category-6":
// category = "invalid"; // category = "invalid";
// break; // break;
case "lore": case "category-8":
category = "lore"; category = "lore";
break; break;
} }
if (tagName) { if (tagName) {
const tag = new ScrapedTag(tagName, category); let tag = new ScrapedTag(tagName, category);
post.tags.push(tag); post.tags.push(tag);
} }
} }
// Set source // Set source
const sourceEls = Array.from(document.querySelectorAll(".source-link > a")).map((x) => x as HTMLAnchorElement); const sourceEls = Array.from(document.querySelectorAll(".source-link > a")).map((x) => x as HTMLAnchorElement);
post.sources = sourceEls.map((x) => x.href); // We currently only support one source.
if (sourceEls.length > 0) {
post.source = sourceEls[0].href;
}
result.tryAddPost(post); result.tryAddPost(post);

37
src/engines/index.ts
View File

@ -1,21 +1,16 @@
export { default as AnimePictures } from "./AnimePictures.js"; export { default as AnimePictures } from "./AnimePictures";
export { default as Bluesky } from "./Bluesky.js"; export { default as Danbooru } from "./Danbooru";
export { default as Danbooru } from "./Danbooru.js"; export { default as e621 } from "./e621";
export { default as e621 } from "./e621.js"; export { default as Fallback } from "./Fallback";
export { default as Ehentai } from "./Ehentai.js"; export { default as FurAffinity } from "./FurAffinity";
export { default as Fallback } from "./Fallback.js"; export { default as Gelbooru } from "./Gelbooru";
export { default as FurAffinity } from "./FurAffinity.js"; export { default as Inkbunny } from "./Inkbunny";
export { default as Gelbooru } from "./Gelbooru.js"; export { default as Moebooru } from "./Moebooru";
export { default as Inkbunny } from "./Inkbunny.js"; export { default as Pixiv } from "./Pixiv";
export { default as Mastodon } from "./Mastodon.js"; export { default as Reddit } from "./Reddit";
export { default as Moebooru } from "./Moebooru.js"; export { default as Rule34us } from "./Rule34us";
export { default as Nozomi } from "./Nozomi.js"; export { default as SankakuComplex } from "./SankakuComplex";
export { default as Philomena } from "./Philomena.js"; export { default as Shimmie2 } from "./Shimmie2";
export { default as Pixiv } from "./Pixiv.js"; export { default as Shuushuu } from "./Shuushuu";
export { default as Reddit } from "./Reddit.js"; export { default as Twitter } from "./Twitter";
export { default as Rule34us } from "./Rule34us.js"; export { default as Zerochan } from "./Zerochan";
export { default as SankakuComplex } from "./SankakuComplex.js";
export { default as Shimmie2 } from "./Shimmie2.js";
export { default as Shuushuu } from "./Shuushuu.js";
export { default as Twitter } from "./Twitter.js";
export { default as Zerochan } from "./Zerochan.js";

8
src/index.ts
View File

@ -1,8 +1,8 @@
export { default as NeoScraper } from "./NeoScraper.js"; export { default as NeoScraper } from "./NeoScraper";
export * from "./ScrapeEngine.js"; export * from "./ScrapeEngine";
import * as Engines from "./ScrapeEngine.js"; import * as Engines from "./engines";
export { Engines }; export { Engines };
import * as BooruTypes from "./BooruTypes.js"; import * as BooruTypes from "./BooruTypes";
export { BooruTypes }; export { BooruTypes };

29
src/test/engine.ts
View File

@ -1,7 +1,5 @@
import { JSDOM, ConstructorOptions } from "jsdom"; import { JSDOM, ConstructorOptions } from "jsdom";
import NeoScraper from "../NeoScraper.js"; import NeoScraper from "../NeoScraper";
export const MobileUserAgent = "Mozilla/5.0 (Android 12; Mobile; rv:68.0) Gecko/68.0 Firefox/103.0";
/** /**
* Go to an URL and parse it's raw HTML without executing any client-side code. * Go to an URL and parse it's raw HTML without executing any client-side code.
@ -9,9 +7,7 @@ export const MobileUserAgent = "Mozilla/5.0 (Android 12; Mobile; rv:68.0) Gecko/
* @returns * @returns
*/ */
export async function scrapeUrl(url: string) { export async function scrapeUrl(url: string) {
const dom = await JSDOM.fromURL(url, { const dom = await JSDOM.fromURL(url);
userAgent: "PostmanRuntime/7.37.3",
});
return scrapeDom(dom.window.document); return scrapeDom(dom.window.document);
} }
@ -21,26 +17,9 @@ export async function scrapeUrl(url: string) {
* @param waitForSelector * @param waitForSelector
* @returns * @returns
*/ */
export async function scrapeEvalUrl( export async function scrapeEvalUrl(url: string, waitForSelector: string) {
url: string,
waitForSelector: string,
userAgent: string | undefined = "PostmanRuntime/7.37.3"
) {
await jestPuppeteer.resetPage();
if (userAgent) {
page.setUserAgent(userAgent);
}
await page.goto(url); await page.goto(url);
await page.waitForSelector(waitForSelector, { visible: true });
const res = await page.waitForSelector(waitForSelector, { visible: true, timeout: 10_000 });
if (res == null) throw new Error("waitForSelector timed out.");
return scrapePage(url, page);
}
export async function scrapePage(url: string, page: any) {
const document = await page.content(); const document = await page.content();
return scrapeHtml(url, document); return scrapeHtml(url, document);
} }

8
src/test/jest.d.ts vendored
View File

@ -1,15 +1,13 @@
import "jest-extended"; import "jest-extended";
import { TagCategory } from "../BooruTypes.js"; import { TagCategory } from "../BooruTypes";
declare global { declare global {
namespace jest { namespace jest {
interface Matchers<R> { interface Matchers<R> {
toHaveTag(tagName: string, tagCategory?: TagCategory): CustomMatcherResult; // The `= undefined` part is incorrect according to VSCode, but it is actually required.
toHaveTag(tagName: string, tagCategory: TagCategory | undefined = undefined): CustomMatcherResult;
toHaveNote(noteText: string): CustomMatcherResult; toHaveNote(noteText: string): CustomMatcherResult;
toHaveResolution(resolution: [number, number]): CustomMatcherResult; toHaveResolution(resolution: [number, number]): CustomMatcherResult;
toHaveSource(source: string): CustomMatcherResult;
} }
} }
} }
export {};

19
src/test/jest.setup.ts
View File

@ -1,6 +1,6 @@
import { JSDOM } from "jsdom"; import { JSDOM } from "jsdom";
import { TagCategory } from "../BooruTypes.js"; import { TagCategory } from "../BooruTypes";
import { ScrapedPost } from "../ScrapeEngine.js"; import { ScrapedPost } from "../ScrapeEngine";
// Expose JSDOM Element constructor // Expose JSDOM Element constructor
global.Element = new JSDOM().window.Element; global.Element = new JSDOM().window.Element;
@ -19,7 +19,7 @@ expect.extend({
tagName: string, tagName: string,
tagCategory: TagCategory | undefined = undefined tagCategory: TagCategory | undefined = undefined
): jest.CustomMatcherResult { ): jest.CustomMatcherResult {
let pass = false; let pass: boolean = false;
const message = () => (pass ? "" : `Post does not have the '${tagName}' tag, or the category is incorrect.`); const message = () => (pass ? "" : `Post does not have the '${tagName}' tag, or the category is incorrect.`);
if (received instanceof ScrapedPost) { if (received instanceof ScrapedPost) {
@ -35,7 +35,7 @@ expect.extend({
return { pass, message }; return { pass, message };
}, },
toHaveNote(received: any, noteText: string): jest.CustomMatcherResult { toHaveNote(received: any, noteText: string): jest.CustomMatcherResult {
let pass = false; let pass: boolean = false;
const message = () => (pass ? "" : `Post does not have a note with '${noteText}'.`); const message = () => (pass ? "" : `Post does not have a note with '${noteText}'.`);
if (received instanceof ScrapedPost) { if (received instanceof ScrapedPost) {
@ -44,7 +44,7 @@ expect.extend({
return { pass, message }; return { pass, message };
}, },
toHaveResolution(received: any, resolution: [number, number]): jest.CustomMatcherResult { toHaveResolution(received: any, resolution: [number, number]): jest.CustomMatcherResult {
let pass = false; let pass: boolean = false;
const message = () => (pass ? "" : `Expected: ${resolution}\nReceived: ${received?.resolution}`); const message = () => (pass ? "" : `Expected: ${resolution}\nReceived: ${received?.resolution}`);
if (received instanceof ScrapedPost && received.resolution?.length == 2) { if (received instanceof ScrapedPost && received.resolution?.length == 2) {
@ -52,13 +52,4 @@ expect.extend({
} }
return { pass, message }; return { pass, message };
}, },
toHaveSource(received: any, source: string): jest.CustomMatcherResult {
let pass = false;
const message = () => (pass ? "" : `Post does not have source '${source}'.`);
if (received instanceof ScrapedPost) {
pass = received.sources.find((x) => x == source) != undefined;
}
return { pass, message };
},
}); });

13
tsconfig.json
View File

@ -1,17 +1,16 @@
{ {
"compilerOptions": { "compilerOptions": {
"lib": [ "lib": [
"ES2019", "es2019",
"DOM", "dom",
"DOM.Iterable" "dom.iterable"
], ],
"rootDir": "./src/",
"outDir": "./dist/", "outDir": "./dist/",
"sourceMap": true, "sourceMap": true,
"strict": true, "strict": true,
"module": "node16", "module": "es2015",
"moduleResolution": "node16", "moduleResolution": "node",
"target": "es2020", "target": "es2015",
"skipLibCheck": true, "skipLibCheck": true,
"esModuleInterop": true, "esModuleInterop": true,
"removeComments": true, "removeComments": true,

12
tsconfig.test.json
View File

@ -1,16 +1,16 @@
{ {
"compilerOptions": { "compilerOptions": {
"lib": [ "lib": [
"ES2019", "es2019",
"DOM", "dom",
"DOM.Iterable" "dom.iterable"
], ],
"outDir": "./dist/", "outDir": "./dist/",
"sourceMap": true, "sourceMap": true,
"strict": true, "strict": true,
"module": "ESNext", "module": "es2015",
"moduleResolution": "NodeNext", "moduleResolution": "node",
"target": "ESNext", "target": "es2015",
"skipLibCheck": true, "skipLibCheck": true,
"esModuleInterop": true, "esModuleInterop": true,
"removeComments": true, "removeComments": true,

4613
yarn.lock Normal file
View File

File diff suppressed because it is too large Load Diff