From a5997102767edf57df7bef066420b728fe14aa85 Mon Sep 17 00:00:00 2001 From: neobooru <50623835+neobooru@users.noreply.github.com> Date: Sun, 7 Aug 2022 20:13:59 +0200 Subject: [PATCH] Add feature info to engines + project cleanup --- .eslintrc.cjs | 17 + README.md | 21 + ...eer.config.js => jest-puppeteer.config.cjs | 0 jest.preset.js => jest.preset.cjs | 0 package.json | 17 +- scripts/summary-table.ts | 15 + src/NeoScraper.test.ts | 4 +- src/NeoScraper.ts | 9 +- src/ScrapeEngine.ts | 25 +- src/Utility.ts | 14 +- src/engines/AnimePictures.ts | 21 +- src/engines/Danbooru.ts | 15 +- src/engines/Fallback.ts | 17 +- src/engines/FurAffinity.ts | 19 +- src/engines/Gelbooru.ts | 21 +- src/engines/Inkbunny.ts | 22 +- src/engines/Moebooru.ts | 17 +- src/engines/Pixiv.ts | 19 +- src/engines/Reddit.ts | 15 +- src/engines/Rule34us.ts | 16 +- src/engines/SankakuComplex.ts | 15 +- src/engines/Shimmie2.ts | 15 +- src/engines/Shuushuu.ts | 15 +- src/engines/Twitter.ts | 19 +- src/engines/Zerochan.ts | 19 +- src/engines/e621.ts | 20 +- src/index.ts | 2 +- src/test/jest.d.ts | 6 +- src/test/jest.setup.ts | 6 +- tsconfig.json | 12 +- yarn.lock | 1147 ++++++++++++++++- 31 files changed, 1370 insertions(+), 210 deletions(-) create mode 100644 .eslintrc.cjs rename jest-puppeteer.config.js => jest-puppeteer.config.cjs (100%) rename jest.preset.js => jest.preset.cjs (100%) create mode 100644 scripts/summary-table.ts diff --git a/.eslintrc.cjs b/.eslintrc.cjs new file mode 100644 index 0000000..21d2864 --- /dev/null +++ b/.eslintrc.cjs @@ -0,0 +1,17 @@ +module.exports = { + env: { + browser: true, + commonjs: true, + es2021: true, + node: true, + }, + extends: ["eslint:recommended", "plugin:@typescript-eslint/recommended"], + parser: "@typescript-eslint/parser", + parserOptions: { + ecmaVersion: "latest", + }, + plugins: ["@typescript-eslint"], + rules: { + semi: ["error", "always"], + }, +}; diff --git a/README.md b/README.md index ebcd535..8ef8009 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,24 @@ # neo-scraper Scrape a booru-like post object from the DOM. + +## Supported sites + +| Engine | Sites | Features | Notes | +| -------------- | ------------------------------------------------------------- | -------------------------------------------------------------- | ------------------------------------------------------------- | +| animepictures | anime-pictures.net | content, resolution, tags, tag_category | Rating is assumed to be safe. | +| danbooru | danbooru.donmai.us, safebooru.donmai.us | content, rating, resolution, tags, tag_category, source, notes | | +| e621 | e621.net, e926.net | content, rating, resolution, tags, tag_category, source | | +| furaffinity | www.furaffinity.net | content, rating, resolution, tags, tag_category | | +| gelbooru | safebooru.org, gelbooru.com, rule34.xxx, tbib.org, xbooru.com | content, rating, resolution, tags, tag_category, source, notes | Supported features might vary between hosts. | +| inkbunny | inkbunny.net | content, rating, resolution, tags | | +| moebooru | yande.re, konachan.com | content, rating, resolution, tags, tag_category, source, notes | | +| pixiv | pixiv.net, www.pixiv.net | content | | +| reddit | reddit.com, new.reddit.com, old.reddit.com | content | | +| rule34us | rule34.us | content, resolution, tags, tag_category | Rating is assumed to be unsafe. | +| sankakucomplex | chan.sankakucomplex.com | content, rating, tags, tag_category, notes | | +| shimmie2 | rule34.paheal.net, rule34hentai.net | content, tags, source | Rating is assumed to be unsafe. | +| shuushuu | e-shuushuu.net | content, rating, resolution, tags, tag_category | Rating is assumed to be safe. | +| twitter | twitter.com, mobile.twitter.com | content | | +| zerochan | www.zerochan.net | content, resolution, tags, tag_category | Rating is assumed to be safe. | +| fallback | | content | Tries to find the largest image or video on the current page. | diff --git a/jest-puppeteer.config.js b/jest-puppeteer.config.cjs similarity index 100% rename from jest-puppeteer.config.js rename to jest-puppeteer.config.cjs diff --git a/jest.preset.js b/jest.preset.cjs similarity index 100% rename from jest.preset.js rename to jest.preset.cjs diff --git a/package.json b/package.json index 5608c0e..3152dea 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,8 @@ { "name": "neo-scraper", - "version": "0.5.4", + "version": "0.5.5", "description": "Scrape a booru-like post object from the DOM.", + "type": "module", "main": "dist/index.js", "types": "dist/index.d.js", "scripts": { @@ -10,7 +11,9 @@ "test:debug": "cross-env HEADLESS=false jest --testTimeout=100000000", "build": "tsc", "watch": "tsc -w", - "prepare": "npm run build" + "prepare": "npm run build", + "lint": "eslint src scripts", + "summary-table": "npm run build && node --experimental-specifier-resolution=node --loader ts-node/esm scripts/summary-table.ts" }, "author": "neobooru", "license": "MIT", @@ -24,15 +27,23 @@ "@types/jsdom": "^12.2.4", "@types/puppeteer": "^5.4.4", "@types/turndown": "^5.0.1", + "@typescript-eslint/eslint-plugin": "^5.32.0", + "@typescript-eslint/parser": "^5.32.0", + "eslint": "8.7.0", + "eslint-plugin-import": "^2.25.2", + "eslint-plugin-n": "^15.0.0", + "eslint-plugin-promise": "^6.0.0", "jest": "^26.6.3", "jest-puppeteer": "^6.0.0", "jsdom": "^15.2.1", + "prettier": "2.5.1", "puppeteer": "^13.1.2", "ts-jest": "^26.5.6", + "ts-node": "^10.9.1", "typescript": "^4.5.2" }, "jest": { - "preset": "./jest.preset.js", + "preset": "./jest.preset.cjs", "transform": { "^.+\\.tsx?$": "ts-jest" }, diff --git a/scripts/summary-table.ts b/scripts/summary-table.ts new file mode 100644 index 0000000..f5d3c08 --- /dev/null +++ b/scripts/summary-table.ts @@ -0,0 +1,15 @@ +import NeoScraper from "../src/NeoScraper"; +import FallbackEngine from "../src/engines/Fallback"; + +const engines = [...new NeoScraper().engines, new FallbackEngine()]; +const txt = ["| Engine | Sites | Features | Notes |", "|---|---|---|---|---|"]; + +for (const engine of engines) { + txt.push( + `| ${engine.name} | ${engine.supportedHosts.join(", ")} | ${engine.features.join(", ")} | ${engine.notes.join( + " " + )} |` + ); +} + +console.log(txt.join("\n")); diff --git a/src/NeoScraper.test.ts b/src/NeoScraper.test.ts index 07ceab6..07e8ee4 100644 --- a/src/NeoScraper.test.ts +++ b/src/NeoScraper.test.ts @@ -1,12 +1,12 @@ import NeoScraper from "./NeoScraper"; test("create scraper with default engines", async () => { - const scraper = new NeoScraper(); + new NeoScraper(); }); test("all default scrapers have unique names", () => { const scraper = new NeoScraper(); - let alreadySeen: string[] = []; + const alreadySeen: string[] = []; for (const engine of scraper.engines) { if (alreadySeen.indexOf(engine.name) != -1) { diff --git a/src/NeoScraper.ts b/src/NeoScraper.ts index 519cd19..919f516 100644 --- a/src/NeoScraper.ts +++ b/src/NeoScraper.ts @@ -5,7 +5,7 @@ export default class NeoScraper { engines: ScrapeEngine[] = []; fallbackEngine: ScrapeEngine | null = null; - constructor(useDefaultEngines: boolean = true) { + constructor(useDefaultEngines = true) { if (useDefaultEngines) { this.engines.push( new engines.AnimePictures(), @@ -28,12 +28,7 @@ export default class NeoScraper { } } - scrapeDocument( - document: Document, - engines: string[] | null = null, - allowFallbackEngine: boolean = false, - ignoreCanImport: boolean = false - ): ScrapeResults { + scrapeDocument(document: Document, allowFallbackEngine = false, ignoreCanImport = false): ScrapeResults { const res = new ScrapeResults(); for (const engine of this.engines) { diff --git a/src/ScrapeEngine.ts b/src/ScrapeEngine.ts index e532e96..3c98051 100644 --- a/src/ScrapeEngine.ts +++ b/src/ScrapeEngine.ts @@ -3,8 +3,8 @@ import { TagCategory, SafetyRating } from "./BooruTypes"; export type ContentType = "image" | "video"; export class ScrapedPost { - contentUrl: string = ""; - pageUrl: string = ""; + contentUrl = ""; + pageUrl = ""; contentType: ContentType = "image"; resolution: [number, number] | undefined; rating: SafetyRating = "safe"; @@ -30,7 +30,7 @@ export class ScrapedNote { export class ScrapeResult { engine: string; - description: string = ""; + description = ""; posts: ScrapedPost[] = []; constructor(engine: string) { @@ -50,9 +50,9 @@ export class ScrapeResults { results: ScrapeResult[] = []; get posts(): ScrapedPost[] { - var posts: ScrapedPost[] = []; + let posts: ScrapedPost[] = []; - for (var res of this.results) { + for (const res of this.results) { posts = posts.concat(res.posts); } @@ -60,15 +60,28 @@ export class ScrapeResults { } } +export type ScrapeEngineFeature = "content" | "rating" | "resolution" | "tags" | "tag_category" | "source" | "notes"; + export interface ScrapeEngine { name: string; + features: ScrapeEngineFeature[]; + notes: string[]; + supportedHosts: string[]; + canImport(url: Location): boolean; scrapeDocument(document: Document): ScrapeResult; } export abstract class ScrapeEngineBase implements ScrapeEngine { abstract name: string; - abstract canImport(url: Location): boolean; + abstract features: ScrapeEngineFeature[]; + abstract notes: string[]; + abstract supportedHosts: string[]; + + canImport(url: Location): boolean { + return this.supportedHosts.indexOf(url.host) != -1; + } + abstract scrapeDocument(document: Document): ScrapeResult; protected log(str: string) { diff --git a/src/Utility.ts b/src/Utility.ts index 53a8efc..64f389f 100644 --- a/src/Utility.ts +++ b/src/Utility.ts @@ -50,7 +50,7 @@ export function getUrlExtension(url: string) { * @param url * @returns "image" | "video" */ -export function guessContentType(url: string): ContentType { +export function guessContentType(url: string | undefined): ContentType { if (url != undefined) { const ext = getUrlExtension(url); if (videoExtensions.indexOf(ext) != -1) return "video"; @@ -109,7 +109,7 @@ export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement return undefined; } const num = parseInt(value); - if (num == NaN) { + if (isNaN(num)) { logFail(`parseInt on '${value}' returned NaN.`); } return num; @@ -132,10 +132,10 @@ export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement text = htmlNoteToMarkdownNote(text); } - let x = getInt(data, "x"); - let y = getInt(data, "y"); - let w = getInt(data, "width"); - let h = getInt(data, "height"); + const x = getInt(data, "x"); + const y = getInt(data, "y"); + const w = getInt(data, "width"); + const h = getInt(data, "height"); if (!text || !x || !y || !w || !h) { logFail("A required data field could not be parsed."); @@ -159,7 +159,7 @@ export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement * @returns */ export function createNotesFromMoebooruBoxes(document: Document, boxSize: [number, number]) { - let notes: ScrapedNote[] = []; + const notes: ScrapedNote[] = []; const noteBoxes = Array.from(document.querySelectorAll(".note-box")).map((x) => x as HTMLLIElement); const noteBodies = Array.from(document.querySelectorAll(".note-body")).map((x) => x as HTMLLIElement); diff --git a/src/engines/AnimePictures.ts b/src/engines/AnimePictures.ts index 60a0ddd..97724f4 100644 --- a/src/engines/AnimePictures.ts +++ b/src/engines/AnimePictures.ts @@ -1,19 +1,18 @@ -import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; +import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine"; import { TagCategory } from "../BooruTypes"; import { guessContentType, parseResolutionString } from "../Utility"; -export default class AnimePictures implements ScrapeEngine { +export default class AnimePictures extends ScrapeEngineBase { name = "animepictures"; - - canImport(url: Location): boolean { - return url.host == "anime-pictures.net"; - } + features: ScrapeEngineFeature[] = ["content", "resolution", "tags", "tag_category"]; + notes = ["Rating is assumed to be safe."]; + supportedHosts = ["anime-pictures.net"]; scrapeDocument(document: Document): ScrapeResult { - let result = new ScrapeResult(this.name); - let post = new ScrapedPost(); + const result = new ScrapeResult(this.name); + const post = new ScrapedPost(); post.pageUrl = document.location.href; - post.source = document.location.href; + // Set image url const originalImageElements = Array.from(document.getElementsByClassName("download_icon")); @@ -40,7 +39,7 @@ export default class AnimePictures implements ScrapeEngine { // Set tags const tagElements = Array.from(document.querySelectorAll("ul.tags > li > a")); for (const tagElement of tagElements) { - let tagName = tagElement.innerText; + const tagName = tagElement.innerText; let category: TagCategory | undefined; for (const className of tagElement.classList) { @@ -58,7 +57,7 @@ export default class AnimePictures implements ScrapeEngine { } if (tagName) { - let tag = new ScrapedTag(tagName, category); + const tag = new ScrapedTag(tagName, category); post.tags.push(tag); } } diff --git a/src/engines/Danbooru.ts b/src/engines/Danbooru.ts index b3fce29..53767d0 100644 --- a/src/engines/Danbooru.ts +++ b/src/engines/Danbooru.ts @@ -1,17 +1,16 @@ -import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapedNote } from "../ScrapeEngine"; +import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapeEngineFeature } from "../ScrapeEngine"; import { TagCategory } from "../BooruTypes"; import { createNoteFromDanbooruArticle, guessContentType } from "../Utility"; export default class Danbooru extends ScrapeEngineBase { name = "danbooru"; - - canImport(url: Location): boolean { - return url.host == "danbooru.donmai.us" || url.host == "safebooru.donmai.us"; - } + features: ScrapeEngineFeature[] = ["content", "rating", "resolution", "tags", "tag_category", "source", "notes"]; + notes = []; + supportedHosts = ["danbooru.donmai.us", "safebooru.donmai.us"]; scrapeDocument(document: Document): ScrapeResult { - let result = new ScrapeResult(this.name); - let post = new ScrapedPost(); + const result = new ScrapeResult(this.name); + const post = new ScrapedPost(); post.pageUrl = document.location.href; // Set image url @@ -92,7 +91,7 @@ export default class Danbooru extends ScrapeEngineBase { } if (tagName) { - let tag = new ScrapedTag(tagName, category); + const tag = new ScrapedTag(tagName, category); post.tags.push(tag); } } diff --git a/src/engines/Fallback.ts b/src/engines/Fallback.ts index 76fddef..4f6f311 100644 --- a/src/engines/Fallback.ts +++ b/src/engines/Fallback.ts @@ -1,4 +1,4 @@ -import { ScrapeEngine, ScrapeResult, ScrapedPost, ContentType } from "../ScrapeEngine"; +import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ContentType, ScrapeEngineFeature } from "../ScrapeEngine"; import { guessContentType } from "../Utility"; class MediaElement { @@ -9,22 +9,21 @@ class MediaElement { ) {} } -export default class Fallback implements ScrapeEngine { +export default class Fallback extends ScrapeEngineBase { name = "fallback"; - - canImport(location: Location): boolean { - return false; - } + features: ScrapeEngineFeature[] = ["content"]; + notes = ["Tries to find the largest image or video on the current page."]; + supportedHosts = []; scrapeDocument(document: Document): ScrapeResult { - let result = new ScrapeResult(this.name); + const result = new ScrapeResult(this.name); // Check if the current page is a link to a video file. // If it is then use the document.location.href as media source and immediately return. // This is needed because firefox does not add a to a