Nes
/
neo-scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add Zerochan + small fixes

pull/5/head
neobooru 2019-11-10 01:23:32 +01:00
parent 01022802c1
commit b20db79a95
6 changed files with 85 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
package-lock.json generated
View File

@ -1014,9 +1014,9 @@
}
},
"commander": {
"version": "2.20.0",
"resolved": "https://registry.npmjs.org/commander/-/commander-2.20.0.tgz",
"integrity": "sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==",
"version": "2.20.3",
"resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
"integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==",
"dev": true,
"optional": true
},
@ -2206,9 +2206,9 @@
"dev": true
},
"handlebars": {
"version": "4.2.1",
"resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.2.1.tgz",
"integrity": "sha512-bqPIlDk06UWbVEIFoYj+LVo42WhK96J+b25l7hbFDpxrOXMphFM3fNIm+cluwg4Pk2jiLjWU5nHQY7igGE75NQ==",
"version": "4.5.1",
"resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.5.1.tgz",
"integrity": "sha512-C29UoFzHe9yM61lOsIlCE5/mQVGrnIOrOq7maQl76L7tYPCgC1og0Ajt6uWnX4ZTxBPnjw+CUvawphwCfJgUnA==",
"dev": true,
"requires": {
"neo-async": "^2.6.0",
@ -4744,13 +4744,13 @@
"dev": true
},
"uglify-js": {
"version": "3.6.0",
"resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.6.0.tgz",
"integrity": "sha512-W+jrUHJr3DXKhrsS7NUVxn3zqMOFn0hL/Ei6v0anCIMoKC93TjcflTagwIHLW7SfMFfiQuktQyFVCFHGUE0+yg==",
"version": "3.6.8",
"resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.6.8.tgz",
"integrity": "sha512-XhHJ3S3ZyMwP8kY1Gkugqx3CJh2C3O0y8NPiSxtm1tyD/pktLAkFZsFGpuNfTZddKDQ/bbDBLAd2YyA1pbi8HQ==",
"dev": true,
"optional": true,
"requires": {
"commander": "~2.20.0",
"commander": "~2.20.3",
"source-map": "~0.6.1"
}
},

2
package.json
View File

@ -1,6 +1,6 @@
{
"name": "neo-scraper",
"version": "0.1.0",
"version": "0.2.0",
"description": "Scrape a booru-like post object from the DOM.",
"main": "dist/index.js",
"types": "dist/index.d.js",

5
src/NeoScraper.ts
View File

@ -13,7 +13,8 @@ export default class NeoScraper {
new engines.Moebooru(),
new engines.Reddit(),
new engines.Shimmie2(),
new engines.Twitter()
new engines.Twitter(),
new engines.Zerochan(),
);
this.fallbackEngine = new engines.Fallback();
}
@ -30,7 +31,7 @@ export default class NeoScraper {
if (allowFallbackEngine) {
if (!this.fallbackEngine) {
console.error("this.fallbackEngine is unset!");
console.error("NeoScraper.fallbackEngine is unset!");
} else {
res.results.push(this.fallbackEngine.scrapeDocument(document));
}

7
src/engines/Twitter.ts
View File

@ -9,10 +9,7 @@ export default class Twitter implements ScrapeEngine {
name = "twitter";
canImport(url: Location): boolean {
return (
url.host == "danbooru.donmai.us" ||
url.host == "safebooru.donmai.us"
);
return url.host == "twitter.com";
}
scrapeDocument(document: Document): ScrapeResult {
@ -35,7 +32,7 @@ export default class Twitter implements ScrapeEngine {
if (mediaImageElements.length > 0) {
post.imageUrl = getOriginalImageUrl((mediaImageElements[0] as HTMLImageElement).src);
}
if (post.imageUrl) {
result.posts.push(post);
}

68
src/engines/Zerochan.ts Normal file
View File

@ -0,0 +1,68 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes";
export default class Zerochan implements ScrapeEngine {
name = "zerochan";
canImport(url: Location): boolean {
return url.host.endsWith("zerochan.net");
}
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
let post = new ScrapedPost();
post.source = document.location.href;
// Set image url
const previewElements = document.getElementsByClassName("preview");
if (previewElements.length > 0) {
post.imageUrl = (previewElements[0] as HTMLAnchorElement).href;
}
// Set rating
post.rating = "safe";
// Set tags
const tagElements = Array.from(document.getElementById("tags")!.children);
for (const tagElement of tagElements) {
// Get tag name from url, because some longer tags are shortened.
// Eg. "Tate no Yuusha no Nariagari" -> "Tate no Yuusha no Nariaga..."
// (ironically this is longer, but you get the idea)
const tagUrl = (tagElement.children[0] as HTMLAnchorElement).pathname.substr(1);
const tagName = decodeURIComponent(tagUrl.replace(/\+/g, '_')).toLowerCase();
let tagType = "";
if (tagElement.childNodes.length > 1 && tagElement.childNodes[1].textContent)
tagType = tagElement.childNodes[1].textContent.trim().toLowerCase();
let category: TagCategory | undefined;
switch (tagType) {
case "game":
case "series":
category = "copyright";
break;
case "character":
category = "character";
break;
case "mangaka":
category = "artist";
break;
case "source":
// Skip these type of tags, because they are not 'booru-like'
continue;
}
if (tagName) {
let tag = new ScrapedTag(tagName, category);
post.tags.push(tag);
}
}
if (post.imageUrl) {
result.posts.push(post);
}
return result;
}
}

1
src/engines/index.ts
View File

@ -5,3 +5,4 @@ export { default as Moebooru } from "./Moebooru";
export { default as Reddit } from "./Reddit";
export { default as Shimmie2 } from "./Shimmie2";
export { default as Twitter } from "./Twitter";
export { default as Zerochan } from "./Zerochan";