Nes
/
neo-scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add feature info to engines + project cleanup

pull/16/head
neobooru 2022-08-07 20:13:59 +02:00
parent 198da89bc4
commit a599710276
31 changed files with 1370 additions and 210 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
.eslintrc.cjs Normal file
View File

@ -0,0 +1,17 @@
module.exports = {
env: {
browser: true,
commonjs: true,
es2021: true,
node: true,
},
extends: ["eslint:recommended", "plugin:@typescript-eslint/recommended"],
parser: "@typescript-eslint/parser",
parserOptions: {
ecmaVersion: "latest",
},
plugins: ["@typescript-eslint"],
rules: {
semi: ["error", "always"],
},
};

21
README.md
View File

@ -1,3 +1,24 @@
# neo-scraper
Scrape a booru-like post object from the DOM.
## Supported sites
| Engine | Sites | Features | Notes |
| -------------- | ------------------------------------------------------------- | -------------------------------------------------------------- | ------------------------------------------------------------- |
| animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
| danbooru | danbooru.donmai.us, safebooru.donmai.us | content, rating, resolution, tags, tag_category, source, notes | |
| e621 | e621.net, e926.net | content, rating, resolution, tags, tag_category, source | |
| furaffinity | www.furaffinity.net | content, rating, resolution, tags, tag_category | |
| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. |
| inkbunny | inkbunny.net | content, rating, resolution, tags | |
| moebooru | yande.re, konachan.com | content, rating, resolution, tags, tag_category, source, notes | |
| pixiv | pixiv.net, www.pixiv.net | content | |
| reddit | reddit.com, new.reddit.com, old.reddit.com | content | |
| rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. |
| sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes | |
| shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. |
| shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. |
| twitter | twitter.com, mobile.twitter.com | content | |
| zerochan | www.zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. |
| fallback | | content | Tries to find the largest image or video on the current page. |

0
jest-puppeteer.config.js → jest-puppeteer.config.cjs
View File

0
jest.preset.js → jest.preset.cjs
View File

17
package.json
View File

@ -1,7 +1,8 @@
{
"name": "neo-scraper",
"version": "0.5.4",
"version": "0.5.5",
"description": "Scrape a booru-like post object from the DOM.",
"type": "module",
"main": "dist/index.js",
"types": "dist/index.d.js",
"scripts": {
@ -10,7 +11,9 @@
"test:debug": "cross-env HEADLESS=false jest --testTimeout=100000000",
"build": "tsc",
"watch": "tsc -w",
"prepare": "npm run build"
"prepare": "npm run build",
"lint": "eslint src scripts",
"summary-table": "npm run build && node --experimental-specifier-resolution=node --loader ts-node/esm scripts/summary-table.ts"
},
"author": "neobooru",
"license": "MIT",
@ -24,15 +27,23 @@
"@types/jsdom": "^12.2.4",
"@types/puppeteer": "^5.4.4",
"@types/turndown": "^5.0.1",
"@typescript-eslint/eslint-plugin": "^5.32.0",
"@typescript-eslint/parser": "^5.32.0",
"eslint": "8.7.0",
"eslint-plugin-import": "^2.25.2",
"eslint-plugin-n": "^15.0.0",
"eslint-plugin-promise": "^6.0.0",
"jest": "^26.6.3",
"jest-puppeteer": "^6.0.0",
"jsdom": "^15.2.1",
"prettier": "2.5.1",
"puppeteer": "^13.1.2",
"ts-jest": "^26.5.6",
"ts-node": "^10.9.1",
"typescript": "^4.5.2"
},
"jest": {
"preset": "./jest.preset.js",
"preset": "./jest.preset.cjs",
"transform": {
"^.+\\.tsx?$": "ts-jest"
},

15
scripts/summary-table.ts Normal file
View File

@ -0,0 +1,15 @@
import NeoScraper from "../src/NeoScraper";
import FallbackEngine from "../src/engines/Fallback";
const engines = [...new NeoScraper().engines, new FallbackEngine()];
const txt = ["| Engine | Sites | Features | Notes |", "|---|---|---|---|---|"];
for (const engine of engines) {
txt.push(
`| ${engine.name} | ${engine.supportedHosts.join(", ")} | ${engine.features.join(", ")} | ${engine.notes.join(
" "
)} |`
);
}
console.log(txt.join("\n"));

4
src/NeoScraper.test.ts
View File

@ -1,12 +1,12 @@
import NeoScraper from "./NeoScraper";
test("create scraper with default engines", async () => {
const scraper = new NeoScraper();
new NeoScraper();
});
test("all default scrapers have unique names", () => {
const scraper = new NeoScraper();
let alreadySeen: string[] = [];
const alreadySeen: string[] = [];
for (const engine of scraper.engines) {
if (alreadySeen.indexOf(engine.name) != -1) {

9
src/NeoScraper.ts
View File

@ -5,7 +5,7 @@ export default class NeoScraper {
engines: ScrapeEngine[] = [];
fallbackEngine: ScrapeEngine | null = null;
constructor(useDefaultEngines: boolean = true) {
constructor(useDefaultEngines = true) {
if (useDefaultEngines) {
this.engines.push(
new engines.AnimePictures(),
@ -28,12 +28,7 @@ export default class NeoScraper {
}
}
scrapeDocument(
document: Document,
engines: string[] | null = null,
allowFallbackEngine: boolean = false,
ignoreCanImport: boolean = false
): ScrapeResults {
scrapeDocument(document: Document, allowFallbackEngine = false, ignoreCanImport = false): ScrapeResults {
const res = new ScrapeResults();
for (const engine of this.engines) {

25
src/ScrapeEngine.ts
View File

@ -3,8 +3,8 @@ import { TagCategory, SafetyRating } from "./BooruTypes";
export type ContentType = "image" | "video";
export class ScrapedPost {
contentUrl: string = "";
pageUrl: string = "";
contentUrl = "";
pageUrl = "";
contentType: ContentType = "image";
resolution: [number, number] | undefined;
rating: SafetyRating = "safe";
@ -30,7 +30,7 @@ export class ScrapedNote {
export class ScrapeResult {
engine: string;
description: string = "";
description = "";
posts: ScrapedPost[] = [];
constructor(engine: string) {
@ -50,9 +50,9 @@ export class ScrapeResults {
results: ScrapeResult[] = [];
get posts(): ScrapedPost[] {
var posts: ScrapedPost[] = [];
let posts: ScrapedPost[] = [];
for (var res of this.results) {
for (const res of this.results) {
posts = posts.concat(res.posts);
}
@ -60,15 +60,28 @@ export class ScrapeResults {
}
}
export type ScrapeEngineFeature = "content" | "rating" | "resolution" | "tags" | "tag_category" | "source" | "notes";
export interface ScrapeEngine {
name: string;
features: ScrapeEngineFeature[];
notes: string[];
supportedHosts: string[];
canImport(url: Location): boolean;
scrapeDocument(document: Document): ScrapeResult;
}
export abstract class ScrapeEngineBase implements ScrapeEngine {
abstract name: string;
abstract canImport(url: Location): boolean;
abstract features: ScrapeEngineFeature[];
abstract notes: string[];
abstract supportedHosts: string[];
canImport(url: Location): boolean {
return this.supportedHosts.indexOf(url.host) != -1;
}
abstract scrapeDocument(document: Document): ScrapeResult;
protected log(str: string) {

14
src/Utility.ts
View File

@ -50,7 +50,7 @@ export function getUrlExtension(url: string) {
* @param url
* @returns "image" | "video"
*/
export function guessContentType(url: string): ContentType {
export function guessContentType(url: string | undefined): ContentType {
if (url != undefined) {
const ext = getUrlExtension(url);
if (videoExtensions.indexOf(ext) != -1) return "video";
@ -109,7 +109,7 @@ export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement
return undefined;
}
const num = parseInt(value);
if (num == NaN) {
if (isNaN(num)) {
logFail(`parseInt on '${value}' returned NaN.`);
}
return num;
@ -132,10 +132,10 @@ export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement
text = htmlNoteToMarkdownNote(text);
}
let x = getInt(data, "x");
let y = getInt(data, "y");
let w = getInt(data, "width");
let h = getInt(data, "height");
const x = getInt(data, "x");
const y = getInt(data, "y");
const w = getInt(data, "width");
const h = getInt(data, "height");
if (!text || !x || !y || !w || !h) {
logFail("A required data field could not be parsed.");
@ -159,7 +159,7 @@ export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement
* @returns
*/
export function createNotesFromMoebooruBoxes(document: Document, boxSize: [number, number]) {
let notes: ScrapedNote[] = [];
const notes: ScrapedNote[] = [];
const noteBoxes = Array.from(document.querySelectorAll(".note-box")).map((x) => x as HTMLLIElement);
const noteBodies = Array.from(document.querySelectorAll(".note-body")).map((x) => x as HTMLLIElement);

21
src/engines/AnimePictures.ts
View File

@ -1,19 +1,18 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes";
import { guessContentType, parseResolutionString } from "../Utility";
export default class AnimePictures implements ScrapeEngine {
export default class AnimePictures extends ScrapeEngineBase {
name = "animepictures";
canImport(url: Location): boolean {
return url.host == "anime-pictures.net";
}
features: ScrapeEngineFeature[] = ["content", "resolution", "tags", "tag_category"];
notes = ["Rating is assumed to be safe."];
supportedHosts = ["anime-pictures.net"];
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
let post = new ScrapedPost();
const result = new ScrapeResult(this.name);
const post = new ScrapedPost();
post.pageUrl = document.location.href;
post.source = document.location.href;
// Set image url
const originalImageElements = Array.from(document.getElementsByClassName("download_icon"));
@ -40,7 +39,7 @@ export default class AnimePictures implements ScrapeEngine {
// Set tags
const tagElements = Array.from(document.querySelectorAll<HTMLAnchorElement>("ul.tags > li > a"));
for (const tagElement of tagElements) {
let tagName = tagElement.innerText;
const tagName = tagElement.innerText;
let category: TagCategory | undefined;
for (const className of tagElement.classList) {
@ -58,7 +57,7 @@ export default class AnimePictures implements ScrapeEngine {
}
if (tagName) {
let tag = new ScrapedTag(tagName, category);
const tag = new ScrapedTag(tagName, category);
post.tags.push(tag);
}
}

15
src/engines/Danbooru.ts
View File

@ -1,17 +1,16 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapedNote } from "../ScrapeEngine";
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes";
import { createNoteFromDanbooruArticle, guessContentType } from "../Utility";
export default class Danbooru extends ScrapeEngineBase {
name = "danbooru";
canImport(url: Location): boolean {
return url.host == "danbooru.donmai.us" || url.host == "safebooru.donmai.us";
}
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
notes = [];
supportedHosts = ["danbooru.donmai.us", "safebooru.donmai.us"];
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
let post = new ScrapedPost();
const result = new ScrapeResult(this.name);
const post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url
@ -92,7 +91,7 @@ export default class Danbooru extends ScrapeEngineBase {
}
if (tagName) {
let tag = new ScrapedTag(tagName, category);
const tag = new ScrapedTag(tagName, category);
post.tags.push(tag);
}
}

17
src/engines/Fallback.ts
View File

@ -1,4 +1,4 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost, ContentType } from "../ScrapeEngine";
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ContentType, ScrapeEngineFeature } from "../ScrapeEngine";
import { guessContentType } from "../Utility";
class MediaElement {
@ -9,22 +9,21 @@ class MediaElement {
) {}
}
export default class Fallback implements ScrapeEngine {
export default class Fallback extends ScrapeEngineBase {
name = "fallback";
canImport(location: Location): boolean {
return false;
}
features: ScrapeEngineFeature[] = ["content"];
notes = ["Tries to find the largest image or video on the current page."];
supportedHosts = [];
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
const result = new ScrapeResult(this.name);
// Check if the current page is a link to a video file.
// If it is then use the document.location.href as media source and immediately return.
// This is needed because firefox does not add a <source> to a <video> element
// when going to a direct link. It looks like chromium-based browsers do not have this issue.
if (guessContentType(document.location.href) == "video") {
let post = new ScrapedPost();
const post = new ScrapedPost();
post.pageUrl = document.location.href;
post.contentUrl = document.location.href;
post.contentType = "video";
@ -50,7 +49,7 @@ export default class Fallback implements ScrapeEngine {
);
if (largestMediaElement) {
let post = new ScrapedPost();
const post = new ScrapedPost();
post.pageUrl = document.location.href;
post.contentUrl = largestMediaElement.contentUrl;
post.contentType = largestMediaElement.contentType;

19
src/engines/FurAffinity.ts
View File

@ -1,17 +1,15 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine";
import { guessContentType, parseResolutionString } from "../Utility";
export default class FurAffinity implements ScrapeEngine {
export default class FurAffinity extends ScrapeEngineBase {
name = "furaffinity";
canImport(url: Location): boolean {
return url.host == "www.furaffinity.net";
}
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category"];
notes = [];
supportedHosts = ["www.furaffinity.net"];
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
let post = new ScrapedPost();
const result = new ScrapeResult(this.name);
const post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url
@ -53,9 +51,6 @@ export default class FurAffinity implements ScrapeEngine {
.map((x) => (x as HTMLSpanElement).innerText)
.map((x) => new ScrapedTag(x));
// Set source
post.source = document.location.href;
result.tryAddPost(post);
return result;

21
src/engines/Gelbooru.ts
View File

@ -1,4 +1,4 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes";
import { CategoryMap } from "./Common";
import { createNoteFromDanbooruArticle, guessContentType, parseResolutionString } from "../Utility";
@ -11,6 +11,9 @@ enum Version {
export default class Gelbooru extends ScrapeEngineBase {
name = "gelbooru";
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
notes = ["Supported features might vary between hosts."];
supportedHosts = ["safebooru.org", "gelbooru.com", "rule34.xxx", "tbib.org", "xbooru.com"];
private readonly classNameToCategoryMap: CategoryMap = {
"tag-type-copyright": "copyright",
@ -19,18 +22,8 @@ export default class Gelbooru extends ScrapeEngineBase {
"tag-type-metadata": "meta",
};
canImport(url: Location): boolean {
return (
url.host == "safebooru.org" ||
url.host == "gelbooru.com" ||
url.host == "rule34.xxx" ||
url.host == "tbib.org" ||
url.host == "xbooru.com"
);
}
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
const result = new ScrapeResult(this.name);
let version: Version | null = null;
switch (document.location.host) {
case "safebooru.org":
@ -49,7 +42,7 @@ export default class Gelbooru extends ScrapeEngineBase {
this.log("Guessed version: " + version);
let post = new ScrapedPost();
const post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url
@ -154,7 +147,7 @@ export default class Gelbooru extends ScrapeEngineBase {
}
}
let tag = new ScrapedTag(tagName, category);
const tag = new ScrapedTag(tagName, category);
post.tags.push(tag);
}
}

22
src/engines/Inkbunny.ts
View File

@ -1,16 +1,14 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine";
export default class Inkbunny implements ScrapeEngine {
export default class Inkbunny extends ScrapeEngineBase {
name = "inkbunny";
canImport(url: Location): boolean {
return url.host == "inkbunny.net";
}
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags"];
notes = [];
supportedHosts = ["inkbunny.net"];
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
let post = new ScrapedPost();
const result = new ScrapeResult(this.name);
const post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url
@ -36,7 +34,7 @@ export default class Inkbunny implements ScrapeEngine {
// Set post resolution
const width = parseInt((<HTMLInputElement>document.getElementById("submission-origwidth"))?.value);
const height = parseInt((<HTMLInputElement>document.getElementById("submission-origheight"))?.value);
if (width != NaN && height != NaN) {
if (!isNaN(width) && !isNaN(height)) {
post.resolution = [width, height];
}
@ -46,7 +44,7 @@ export default class Inkbunny implements ScrapeEngine {
const ratingEls = spanEls.map((x) => x as HTMLDivElement).filter((x) => x.innerText.indexOf("Rating:") != -1);
if (ratingEls.length > 0) {
const matches = ratingEls[0].parentElement!.innerText.match(ratingExp);
const matches = ratingEls[0].parentElement?.innerText.match(ratingExp);
if (matches && matches.length > 0) {
switch (matches[1].toLowerCase()) {
case "general":
@ -68,7 +66,7 @@ export default class Inkbunny implements ScrapeEngine {
const keywords = keywordsContent.split(",").map((x) => x.trim());
for (const keyword of keywords) {
let tag = new ScrapedTag(keyword);
const tag = new ScrapedTag(keyword);
post.tags.push(tag);
}
}

17
src/engines/Moebooru.ts
View File

@ -1,17 +1,16 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapedNote } from "../ScrapeEngine";
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes";
import { createNotesFromMoebooruBoxes, guessContentType, parseResolutionString } from "../Utility";
export default class Moebooru extends ScrapeEngineBase {
name = "moebooru";
canImport(url: Location): boolean {
return url.host == "yande.re" || url.host == "konachan.com";
}
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"];
notes = [];
supportedHosts = ["yande.re", "konachan.com"];
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
let post = new ScrapedPost();
const result = new ScrapeResult(this.name);
const post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url
@ -64,7 +63,7 @@ export default class Moebooru extends ScrapeEngineBase {
for (const el of tagElements) {
let tagName: string;
let tagNameElements = el.getElementsByTagName("a");
const tagNameElements = el.getElementsByTagName("a");
if (tagNameElements.length > 1) {
// First <a> is "?"
@ -96,7 +95,7 @@ export default class Moebooru extends ScrapeEngineBase {
}
}
let tag = new ScrapedTag(tagName, category);
const tag = new ScrapedTag(tagName, category);
post.tags.push(tag);
}

19
src/engines/Pixiv.ts
View File

@ -1,4 +1,4 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine";
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapeEngineFeature } from "../ScrapeEngine";
import { guessContentType } from "../Utility";
export function upgradeUrlToOriginalQuality(imgLink: string) {
@ -21,17 +21,15 @@ export function upgradeUrlToOriginalQuality(imgLink: string) {
return imgLink;
}
export default class Pixiv implements ScrapeEngine {
export default class Pixiv extends ScrapeEngineBase {
name = "pixiv";
canImport(url: Location): boolean {
return url.host == "pixiv.net" || url.host == "www.pixiv.net";
}
features: ScrapeEngineFeature[] = ["content"];
notes = [];
supportedHosts = ["pixiv.net", "www.pixiv.net"];
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
let imgLinks = [] as string[];
const result = new ScrapeResult(this.name);
const imgLinks = [] as string[];
// Query for when the user is on the desktop site, is logged in, and has optionally clicked on "See all".
// If the user has not clicked on "See all" then this will only grab the first post.
@ -49,6 +47,7 @@ export default class Pixiv implements ScrapeEngine {
// This only works on pages with multiple posts. The user can optionally be logged in.
imgLinks.push(
...Array.from(document.querySelectorAll<HTMLImageElement>(".manga-page > a > img[data-big]")).map(
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
(x) => x.getAttribute("data-big")!
)
);
@ -63,7 +62,7 @@ export default class Pixiv implements ScrapeEngine {
}
for (const imgLink of imgLinks) {
let post = new ScrapedPost();
const post = new ScrapedPost();
post.pageUrl = document.location.href;
post.contentUrl = upgradeUrlToOriginalQuality(imgLink);
post.contentType = guessContentType(post.contentUrl);

15
src/engines/Reddit.ts
View File

@ -1,16 +1,15 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine";
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapeEngineFeature } from "../ScrapeEngine";
import { guessContentType } from "../Utility";
export default class Reddit implements ScrapeEngine {
export default class Reddit extends ScrapeEngineBase {
name = "reddit";
canImport(url: Location): boolean {
return url.host.endsWith("reddit.com");
}
features: ScrapeEngineFeature[] = ["content"];
notes = [];
supportedHosts = ["reddit.com", "new.reddit.com", "old.reddit.com"];
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
let post = new ScrapedPost();
const result = new ScrapeResult(this.name);
const post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url

16
src/engines/Rule34us.ts
View File

@ -1,13 +1,16 @@
// Technically speaking rule34.us also uses gelbooru, albeit a (heavily?) modified one.
// Because pretty much no scraper code can be shared with other gelbooru-likes it makes more sense to create a new engine for rule34.us.
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes";
import { CategoryMap } from "./Common";
import { guessContentType, parseResolutionString } from "../Utility";
export default class Gelbooru implements ScrapeEngine {
export default class Gelbooru extends ScrapeEngineBase {
name = "rule34us";
features: ScrapeEngineFeature[] = ["content", "resolution", "tags", "tag_category"];
notes = ["Rating is assumed to be unsafe."];
supportedHosts = ["rule34.us"];
private readonly classNameToCategoryMap: CategoryMap = {
"character-tag": "character",
@ -16,14 +19,9 @@ export default class Gelbooru implements ScrapeEngine {
"metadata-tag": "meta",
};
canImport(url: Location): boolean {
return url.host == "rule34.us";
}
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
let post = new ScrapedPost();
const result = new ScrapeResult(this.name);
const post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url

15
src/engines/SankakuComplex.ts
View File

@ -1,17 +1,16 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapedNote } from "../ScrapeEngine";
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes";
import { createNotesFromMoebooruBoxes, guessContentType } from "../Utility";
export default class SankakuComplex extends ScrapeEngineBase {
name = "sankakucomplex";
canImport(url: Location): boolean {
return url.host == "chan.sankakucomplex.com";
}
features: ScrapeEngineFeature[] = ["content", "rating", "tags", "tag_category", "notes"];
notes = [];
supportedHosts = ["chan.sankakucomplex.com"];
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
let post = new ScrapedPost();
const result = new ScrapeResult(this.name);
const post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url
@ -76,7 +75,7 @@ export default class SankakuComplex extends ScrapeEngineBase {
}
if (tagName) {
let tag = new ScrapedTag(tagName, category);
const tag = new ScrapedTag(tagName, category);
post.tags.push(tag);
}
}

15
src/engines/Shimmie2.ts
View File

@ -1,16 +1,15 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine";
import { guessContentType } from "../Utility";
export default class Shimmie2 implements ScrapeEngine {
export default class Shimmie2 extends ScrapeEngineBase {
name = "shimmie2";
canImport(url: Location): boolean {
return url.host == "rule34.paheal.net" || url.host == "rule34hentai.net";
}
features: ScrapeEngineFeature[] = ["content", "tags", "source"];
notes = ["Rating is assumed to be unsafe."];
supportedHosts = ["rule34.paheal.net", "rule34hentai.net"];
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
let post = new ScrapedPost();
const result = new ScrapeResult(this.name);
const post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url

15
src/engines/Shuushuu.ts
View File

@ -1,17 +1,16 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes";
import { guessContentType, parseResolutionString } from "../Utility";
export default class Shuushuu implements ScrapeEngine {
export default class Shuushuu extends ScrapeEngineBase {
name = "shuushuu";
canImport(url: Location): boolean {
return url.host == "e-shuushuu.net";
}
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category"];
notes = ["Rating is assumed to be safe."];
supportedHosts = ["e-shuushuu.net"];
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
let post = new ScrapedPost();
const result = new ScrapeResult(this.name);
const post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url

19
src/engines/Twitter.ts
View File

@ -1,30 +1,29 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine";
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapeEngineFeature } from "../ScrapeEngine";
export function getOriginalImageUrl(image: HTMLImageElement) {
// if the link includes a resolution, change it to orig:
// https://pbs.twimg.com/media/stuff?format=jpg&name=900x900
// -> https://pbs.twimg.com/media/stuff?format=jpg&name=orig
let img_url = new URL((image as HTMLImageElement).src);
const img_url = new URL((image as HTMLImageElement).src);
if (img_url.searchParams.has("name") && img_url.searchParams.get("name") != "orig") {
img_url.searchParams.set("name", "orig");
}
return img_url;
}
export default class Twitter implements ScrapeEngine {
export default class Twitter extends ScrapeEngineBase {
name = "twitter";
canImport(url: Location): boolean {
return url.host == "twitter.com" || url.host == "mobile.twitter.com";
}
features: ScrapeEngineFeature[] = ["content"];
notes = [];
supportedHosts = ["twitter.com", "mobile.twitter.com"];
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
const result = new ScrapeResult(this.name);
// first look for a focused image on desktop:
const center_img = document.querySelector("div[role=dialog] div[role=dialog] div[aria-label=Image] img");
if (center_img) {
let post = new ScrapedPost();
const post = new ScrapedPost();
post.pageUrl = document.location.href;
post.contentUrl = getOriginalImageUrl(center_img as HTMLImageElement).href;
result.tryAddPost(post);
@ -34,7 +33,7 @@ export default class Twitter implements ScrapeEngine {
const imgs = [...document.querySelectorAll("article a[role='link'] div[data-testid='tweetPhoto'] img")];
for (const img of imgs) {
let post = new ScrapedPost();
const post = new ScrapedPost();
post.pageUrl = document.location.href;
post.contentUrl = getOriginalImageUrl(img as HTMLImageElement).href;
result.tryAddPost(post);

19
src/engines/Zerochan.ts
View File

@ -1,17 +1,16 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes";
import { guessContentType, parseResolutionString } from "../Utility";
export default class Zerochan implements ScrapeEngine {
export default class Zerochan extends ScrapeEngineBase {
name = "zerochan";
canImport(url: Location): boolean {
return url.host.endsWith("zerochan.net");
}
features: ScrapeEngineFeature[] = ["content", "resolution", "tags", "tag_category"];
notes = ["Rating is assumed to be safe."];
supportedHosts = ["www.zerochan.net"];
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
let post = new ScrapedPost();
const result = new ScrapeResult(this.name);
const post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url
@ -67,12 +66,12 @@ export default class Zerochan implements ScrapeEngine {
category = "artist";
break;
case "source":
// Skip these type of tags, because they are not 'booru-like'
// Skip these type of tags, because they are not 'booru-like'.
continue;
}
if (tagName) {
let tag = new ScrapedTag(tagName, category);
const tag = new ScrapedTag(tagName, category);
post.tags.push(tag);
}
}

20
src/engines/e621.ts
View File

@ -1,17 +1,15 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes";
export default class e621 implements ScrapeEngine {
export default class e621 extends ScrapeEngineBase {
name = "e621";
canImport(url: Location): boolean {
return url.host == "e621.net" || url.host == "e926.net";
}
features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source"];
notes = [];
supportedHosts = ["e621.net", "e926.net"];
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
let post = new ScrapedPost();
const result = new ScrapeResult(this.name);
const post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url
@ -40,7 +38,7 @@ export default class e621 implements ScrapeEngine {
// Set resolution
const width = parseInt((<HTMLSpanElement>document.querySelector("span[itemprop='width']"))?.innerText);
const height = parseInt((<HTMLSpanElement>document.querySelector("span[itemprop='height']"))?.innerText);
if (width != NaN && height != NaN) {
if (!isNaN(width) && !isNaN(height)) {
post.resolution = [width, height];
}
@ -77,7 +75,7 @@ export default class e621 implements ScrapeEngine {
}
if (tagName) {
let tag = new ScrapedTag(tagName, category);
const tag = new ScrapedTag(tagName, category);
post.tags.push(tag);
}
}

2
src/index.ts
View File

@ -1,7 +1,7 @@
export { default as NeoScraper } from "./NeoScraper";
export * from "./ScrapeEngine";
import * as Engines from "./engines";
import * as Engines from "./ScrapeEngine";
export { Engines };
import * as BooruTypes from "./BooruTypes";

6
src/test/jest.d.ts vendored
View File

@ -5,9 +5,9 @@ declare global {
namespace jest {
interface Matchers<R> {
// The `= undefined` part is incorrect according to VSCode, but it is actually required.
toHaveTag(tagName: string, tagCategory: TagCategory | undefined = undefined): CustomMatcherResult;
toHaveNote(noteText: string): CustomMatcherResult;
toHaveResolution(resolution: [number, number]): CustomMatcherResult;
toHaveTag(tagName: string, tagCategory: TagCategory | undefined): R;
toHaveNote(noteText: string): R;
toHaveResolution(resolution: [number, number]): R;
}
}
}

6
src/test/jest.setup.ts
View File

@ -19,7 +19,7 @@ expect.extend({
tagName: string,
tagCategory: TagCategory | undefined = undefined
): jest.CustomMatcherResult {
let pass: boolean = false;
let pass = false;
const message = () => (pass ? "" : `Post does not have the '${tagName}' tag, or the category is incorrect.`);
if (received instanceof ScrapedPost) {
@ -35,7 +35,7 @@ expect.extend({
return { pass, message };
},
toHaveNote(received: any, noteText: string): jest.CustomMatcherResult {
let pass: boolean = false;
let pass = false;
const message = () => (pass ? "" : `Post does not have a note with '${noteText}'.`);
if (received instanceof ScrapedPost) {
@ -44,7 +44,7 @@ expect.extend({
return { pass, message };
},
toHaveResolution(received: any, resolution: [number, number]): jest.CustomMatcherResult {
let pass: boolean = false;
let pass = false;
const message = () => (pass ? "" : `Expected: ${resolution}\nReceived: ${received?.resolution}`);
if (received instanceof ScrapedPost && received.resolution?.length == 2) {

12
tsconfig.json
View File

@ -1,16 +1,16 @@
{
"compilerOptions": {
"lib": [
"es2019",
"dom",
"dom.iterable"
"ES2019",
"DOM",
"DOM.Iterable"
],
"outDir": "./dist/",
"sourceMap": true,
"strict": true,
"module": "es2015",
"moduleResolution": "node",
"target": "es2015",
"module": "ESNext",
"moduleResolution": "Node",
"target": "ESNext",
"skipLibCheck": true,
"esModuleInterop": true,
"removeComments": true,

1147
yarn.lock
View File

File diff suppressed because it is too large Load Diff