From b20db79a95dc9d9af9b1adb7eb514bf67847755a Mon Sep 17 00:00:00 2001 From: neobooru <50623835+neobooru@users.noreply.github.com> Date: Sun, 10 Nov 2019 01:23:32 +0100 Subject: [PATCH] Add Zerochan + small fixes --- package-lock.json | 20 ++++++------ package.json | 2 +- src/NeoScraper.ts | 5 +-- src/engines/Twitter.ts | 7 ++--- src/engines/Zerochan.ts | 68 +++++++++++++++++++++++++++++++++++++++++ src/engines/index.ts | 1 + 6 files changed, 85 insertions(+), 18 deletions(-) create mode 100644 src/engines/Zerochan.ts diff --git a/package-lock.json b/package-lock.json index 39ce4d5..05e26d7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1014,9 +1014,9 @@ } }, "commander": { - "version": "2.20.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.0.tgz", - "integrity": "sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==", + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", "dev": true, "optional": true }, @@ -2206,9 +2206,9 @@ "dev": true }, "handlebars": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.2.1.tgz", - "integrity": "sha512-bqPIlDk06UWbVEIFoYj+LVo42WhK96J+b25l7hbFDpxrOXMphFM3fNIm+cluwg4Pk2jiLjWU5nHQY7igGE75NQ==", + "version": "4.5.1", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.5.1.tgz", + "integrity": "sha512-C29UoFzHe9yM61lOsIlCE5/mQVGrnIOrOq7maQl76L7tYPCgC1og0Ajt6uWnX4ZTxBPnjw+CUvawphwCfJgUnA==", "dev": true, "requires": { "neo-async": "^2.6.0", @@ -4744,13 +4744,13 @@ "dev": true }, "uglify-js": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.6.0.tgz", - "integrity": "sha512-W+jrUHJr3DXKhrsS7NUVxn3zqMOFn0hL/Ei6v0anCIMoKC93TjcflTagwIHLW7SfMFfiQuktQyFVCFHGUE0+yg==", + "version": "3.6.8", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.6.8.tgz", + "integrity": "sha512-XhHJ3S3ZyMwP8kY1Gkugqx3CJh2C3O0y8NPiSxtm1tyD/pktLAkFZsFGpuNfTZddKDQ/bbDBLAd2YyA1pbi8HQ==", "dev": true, "optional": true, "requires": { - "commander": "~2.20.0", + "commander": "~2.20.3", "source-map": "~0.6.1" } }, diff --git a/package.json b/package.json index d733e51..7e1085d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "neo-scraper", - "version": "0.1.0", + "version": "0.2.0", "description": "Scrape a booru-like post object from the DOM.", "main": "dist/index.js", "types": "dist/index.d.js", diff --git a/src/NeoScraper.ts b/src/NeoScraper.ts index 6ab42e3..e51b18a 100644 --- a/src/NeoScraper.ts +++ b/src/NeoScraper.ts @@ -13,7 +13,8 @@ export default class NeoScraper { new engines.Moebooru(), new engines.Reddit(), new engines.Shimmie2(), - new engines.Twitter() + new engines.Twitter(), + new engines.Zerochan(), ); this.fallbackEngine = new engines.Fallback(); } @@ -30,7 +31,7 @@ export default class NeoScraper { if (allowFallbackEngine) { if (!this.fallbackEngine) { - console.error("this.fallbackEngine is unset!"); + console.error("NeoScraper.fallbackEngine is unset!"); } else { res.results.push(this.fallbackEngine.scrapeDocument(document)); } diff --git a/src/engines/Twitter.ts b/src/engines/Twitter.ts index fbd1d08..3d3bc36 100644 --- a/src/engines/Twitter.ts +++ b/src/engines/Twitter.ts @@ -9,10 +9,7 @@ export default class Twitter implements ScrapeEngine { name = "twitter"; canImport(url: Location): boolean { - return ( - url.host == "danbooru.donmai.us" || - url.host == "safebooru.donmai.us" - ); + return url.host == "twitter.com"; } scrapeDocument(document: Document): ScrapeResult { @@ -35,7 +32,7 @@ export default class Twitter implements ScrapeEngine { if (mediaImageElements.length > 0) { post.imageUrl = getOriginalImageUrl((mediaImageElements[0] as HTMLImageElement).src); } - + if (post.imageUrl) { result.posts.push(post); } diff --git a/src/engines/Zerochan.ts b/src/engines/Zerochan.ts new file mode 100644 index 0000000..ee677f4 --- /dev/null +++ b/src/engines/Zerochan.ts @@ -0,0 +1,68 @@ +import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; +import { TagCategory } from "../BooruTypes"; + +export default class Zerochan implements ScrapeEngine { + name = "zerochan"; + + canImport(url: Location): boolean { + return url.host.endsWith("zerochan.net"); + } + + scrapeDocument(document: Document): ScrapeResult { + let result = new ScrapeResult(this.name); + let post = new ScrapedPost(); + post.source = document.location.href; + + // Set image url + const previewElements = document.getElementsByClassName("preview"); + if (previewElements.length > 0) { + post.imageUrl = (previewElements[0] as HTMLAnchorElement).href; + } + + // Set rating + post.rating = "safe"; + + // Set tags + const tagElements = Array.from(document.getElementById("tags")!.children); + + for (const tagElement of tagElements) { + // Get tag name from url, because some longer tags are shortened. + // Eg. "Tate no Yuusha no Nariagari" -> "Tate no Yuusha no Nariaga..." + // (ironically this is longer, but you get the idea) + const tagUrl = (tagElement.children[0] as HTMLAnchorElement).pathname.substr(1); + const tagName = decodeURIComponent(tagUrl.replace(/\+/g, '_')).toLowerCase(); + + let tagType = ""; + if (tagElement.childNodes.length > 1 && tagElement.childNodes[1].textContent) + tagType = tagElement.childNodes[1].textContent.trim().toLowerCase(); + + let category: TagCategory | undefined; + switch (tagType) { + case "game": + case "series": + category = "copyright"; + break; + case "character": + category = "character"; + break; + case "mangaka": + category = "artist"; + break; + case "source": + // Skip these type of tags, because they are not 'booru-like' + continue; + } + + if (tagName) { + let tag = new ScrapedTag(tagName, category); + post.tags.push(tag); + } + } + + if (post.imageUrl) { + result.posts.push(post); + } + + return result; + } +} diff --git a/src/engines/index.ts b/src/engines/index.ts index 6127fb9..a8ce310 100644 --- a/src/engines/index.ts +++ b/src/engines/index.ts @@ -5,3 +5,4 @@ export { default as Moebooru } from "./Moebooru"; export { default as Reddit } from "./Reddit"; export { default as Shimmie2 } from "./Shimmie2"; export { default as Twitter } from "./Twitter"; +export { default as Zerochan } from "./Zerochan";