From a06aea68f26c54c9b14ff8b9b11d0f454dc320a1 Mon Sep 17 00:00:00 2001 From: neobooru <50623835+neobooru@users.noreply.github.com> Date: Sat, 4 Jan 2020 18:55:09 +0100 Subject: [PATCH] Add e-shuushuu.net, update TypeScript, and some small fixes --- package-lock.json | 8 ++--- package.json | 4 +-- src/NeoScraper.ts | 1 + src/ScrapeEngine.ts | 2 +- src/engines/Shimmie2.ts | 2 +- src/engines/Shuushuu.ts | 65 +++++++++++++++++++++++++++++++++++++++++ src/engines/index.ts | 1 + 7 files changed, 75 insertions(+), 8 deletions(-) create mode 100644 src/engines/Shuushuu.ts diff --git a/package-lock.json b/package-lock.json index e129272..d3d8856 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "neo-scraper", - "version": "0.2.1", + "version": "0.2.2", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -4738,9 +4738,9 @@ } }, "typescript": { - "version": "3.6.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.6.3.tgz", - "integrity": "sha512-N7bceJL1CtRQ2RiG0AQME13ksR7DiuQh/QehubYcghzv20tnh+MQnQIuJddTmsbqYj+dztchykemz0zFzlvdQw==", + "version": "3.7.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.7.4.tgz", + "integrity": "sha512-A25xv5XCtarLwXpcDNZzCGvW2D1S3/bACratYBx2sax8PefsFhlYmkQicKHvpYflFS8if4zne5zT5kpJ7pzuvw==", "dev": true }, "uglify-js": { diff --git a/package.json b/package.json index bc80d4e..2e02f8d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "neo-scraper", - "version": "0.2.1", + "version": "0.2.2", "description": "Scrape a booru-like post object from the DOM.", "main": "dist/index.js", "types": "dist/index.d.js", @@ -17,7 +17,7 @@ "jest": "^24.9.0", "jsdom": "^15.1.1", "ts-jest": "^24.1.0", - "typescript": "^3.6.3" + "typescript": "^3.7.4" }, "jest": { "transform": { diff --git a/src/NeoScraper.ts b/src/NeoScraper.ts index 3ac6bd5..5b9f3f7 100644 --- a/src/NeoScraper.ts +++ b/src/NeoScraper.ts @@ -14,6 +14,7 @@ export default class NeoScraper { new engines.Reddit(), new engines.SankakuComplex(), new engines.Shimmie2(), + new engines.Shuushuu(), new engines.Twitter(), new engines.Zerochan(), ); diff --git a/src/ScrapeEngine.ts b/src/ScrapeEngine.ts index 4cb3175..2ce368a 100644 --- a/src/ScrapeEngine.ts +++ b/src/ScrapeEngine.ts @@ -12,7 +12,7 @@ export class ScrapedTag { category?: TagCategory; constructor(name: string, category?: TagCategory) { - this.name = name.replace(/\s/g, "_"); // Replace all spaces with underscores + this.name = name.replace(/\s/g, "_").toLowerCase(); // Replace all spaces with underscores and make everything lowercase this.category = category; } } diff --git a/src/engines/Shimmie2.ts b/src/engines/Shimmie2.ts index 77e212d..c0877c6 100644 --- a/src/engines/Shimmie2.ts +++ b/src/engines/Shimmie2.ts @@ -38,7 +38,7 @@ export default class Shimmie2 implements ScrapeEngine { // Tags might not match with other booru-likes, but that is for the user to figure out. // For example by deleting 'wrong' tags or setting up some aliases in szurubooru. post.tags = Array.from(document.querySelectorAll("a.tag_name")) - .map(x => (x as HTMLAnchorElement).innerText.toLowerCase()) + .map(x => (x as HTMLAnchorElement).innerText) .map(x => new ScrapedTag(x)); result.tryAddPost(post); diff --git a/src/engines/Shuushuu.ts b/src/engines/Shuushuu.ts new file mode 100644 index 0000000..3151a81 --- /dev/null +++ b/src/engines/Shuushuu.ts @@ -0,0 +1,65 @@ +import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; +import { TagCategory } from "../BooruTypes"; + +export default class Shuushuu implements ScrapeEngine { + name = "shuushuu"; + + canImport(url: Location): boolean { + return url.host == "e-shuushuu.net"; + } + + scrapeDocument(document: Document): ScrapeResult { + let result = new ScrapeResult(this.name); + let post = new ScrapedPost(); + post.source = document.location.href; + + // Set image url + const imgElement = document.querySelector(".thumb_image") as HTMLAnchorElement; + if (imgElement) { + post.imageUrl = imgElement.href; + } + + // Set rating + post.rating = "safe"; + + // Set tags + const exp = new RegExp(/(.*)_(\w+)/); + const tagElements = document.getElementsByClassName("quicktag"); + + for (const tagElement of tagElements) { + // id = quicktag1_1009628 + // match[0] = quicktag1_1009628 (aka input str) + // match[1] = quicktag1 + // match[2] = 1009628 + const match = exp.exec(tagElement.id); + if (!match || match.length != 3) { continue; } + + let tagCategory: TagCategory | undefined; + switch (match[1]) { + // Source, aka copyright + case "quicktag2": + tagCategory = "copyright"; + break; + // Artist + case "quicktag3": + tagCategory = "artist"; + break; + // Characters + case "quicktag4": + tagCategory = "character"; + break; + } + + for (const el of tagElement.children) { + const tagName = (el?.firstElementChild as HTMLAnchorElement)?.innerText; + if (tagName) { + post.tags.push(new ScrapedTag(tagName, tagCategory)); + } + } + } + + result.tryAddPost(post); + + return result; + } +} diff --git a/src/engines/index.ts b/src/engines/index.ts index 477a48f..e44505f 100644 --- a/src/engines/index.ts +++ b/src/engines/index.ts @@ -5,5 +5,6 @@ export { default as Moebooru } from "./Moebooru"; export { default as Reddit } from "./Reddit"; export { default as SankakuComplex } from "./SankakuComplex"; export { default as Shimmie2 } from "./Shimmie2"; +export { default as Shuushuu } from "./Shuushuu"; export { default as Twitter } from "./Twitter"; export { default as Zerochan } from "./Zerochan";