Add SankakuComplex + npm audit fix
Though SankakuComplex doesn't work because of their hotlink protection, but that's an issue for later.pull/2/head
parent
63fc5a420c
commit
df5b1d9fc1
|
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
"name": "neo-scraper",
|
"name": "neo-scraper",
|
||||||
"version": "0.1.0",
|
"version": "0.2.1",
|
||||||
"lockfileVersion": 1,
|
"lockfileVersion": 1,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
|
|
@ -2206,9 +2206,9 @@
|
||||||
"dev": true
|
"dev": true
|
||||||
},
|
},
|
||||||
"handlebars": {
|
"handlebars": {
|
||||||
"version": "4.5.1",
|
"version": "4.5.3",
|
||||||
"resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.5.1.tgz",
|
"resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.5.3.tgz",
|
||||||
"integrity": "sha512-C29UoFzHe9yM61lOsIlCE5/mQVGrnIOrOq7maQl76L7tYPCgC1og0Ajt6uWnX4ZTxBPnjw+CUvawphwCfJgUnA==",
|
"integrity": "sha512-3yPecJoJHK/4c6aZhSvxOyG4vJKDshV36VHp0iVCDVh7o9w2vwi3NSnL2MMPj3YdduqaBcu7cGbggJQM0br9xA==",
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"requires": {
|
"requires": {
|
||||||
"neo-async": "^2.6.0",
|
"neo-async": "^2.6.0",
|
||||||
|
|
@ -4744,9 +4744,9 @@
|
||||||
"dev": true
|
"dev": true
|
||||||
},
|
},
|
||||||
"uglify-js": {
|
"uglify-js": {
|
||||||
"version": "3.6.8",
|
"version": "3.7.2",
|
||||||
"resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.6.8.tgz",
|
"resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.7.2.tgz",
|
||||||
"integrity": "sha512-XhHJ3S3ZyMwP8kY1Gkugqx3CJh2C3O0y8NPiSxtm1tyD/pktLAkFZsFGpuNfTZddKDQ/bbDBLAd2YyA1pbi8HQ==",
|
"integrity": "sha512-uhRwZcANNWVLrxLfNFEdltoPNhECUR3lc+UdJoG9CBpMcSnKyWA94tc3eAujB1GcMY5Uwq8ZMp4qWpxWYDQmaA==",
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"optional": true,
|
"optional": true,
|
||||||
"requires": {
|
"requires": {
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
"name": "neo-scraper",
|
"name": "neo-scraper",
|
||||||
"version": "0.2.0",
|
"version": "0.2.1",
|
||||||
"description": "Scrape a booru-like post object from the DOM.",
|
"description": "Scrape a booru-like post object from the DOM.",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
"types": "dist/index.d.js",
|
"types": "dist/index.d.js",
|
||||||
|
|
|
||||||
|
|
@ -12,6 +12,7 @@ export default class NeoScraper {
|
||||||
new engines.Gelbooru(),
|
new engines.Gelbooru(),
|
||||||
new engines.Moebooru(),
|
new engines.Moebooru(),
|
||||||
new engines.Reddit(),
|
new engines.Reddit(),
|
||||||
|
new engines.SankakuComplex(),
|
||||||
new engines.Shimmie2(),
|
new engines.Shimmie2(),
|
||||||
new engines.Twitter(),
|
new engines.Twitter(),
|
||||||
new engines.Zerochan(),
|
new engines.Zerochan(),
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,83 @@
|
||||||
|
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
|
||||||
|
import { TagCategory } from "../BooruTypes";
|
||||||
|
|
||||||
|
export default class SankakuComplex implements ScrapeEngine {
|
||||||
|
name = "sankakucomplex";
|
||||||
|
|
||||||
|
canImport(url: Location): boolean {
|
||||||
|
return url.host == "chan.sankakucomplex.com";
|
||||||
|
}
|
||||||
|
|
||||||
|
scrapeDocument(document: Document): ScrapeResult {
|
||||||
|
let result = new ScrapeResult(this.name);
|
||||||
|
let post = new ScrapedPost();
|
||||||
|
post.source = document.location.href;
|
||||||
|
|
||||||
|
// Set image url
|
||||||
|
const originalImageElement = document.getElementById("highres") as HTMLAnchorElement;
|
||||||
|
|
||||||
|
if (originalImageElement) {
|
||||||
|
post.imageUrl = originalImageElement.href;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set rating
|
||||||
|
const safetyExp = new RegExp("Rating: (.*)");
|
||||||
|
const safetyElements = Array.from(document.querySelectorAll("#stats > ul > li"))
|
||||||
|
.map(x => x as HTMLLIElement)
|
||||||
|
.filter(x => x.innerText.startsWith("Rating"));
|
||||||
|
|
||||||
|
if (safetyElements.length > 0) {
|
||||||
|
const matches = safetyElements[0].innerText.match(safetyExp);
|
||||||
|
if (matches && matches.length > 0) {
|
||||||
|
switch (matches[1].toLowerCase()) {
|
||||||
|
case "safe":
|
||||||
|
post.rating = "safe";
|
||||||
|
break;
|
||||||
|
case "questionable":
|
||||||
|
post.rating = "sketchy";
|
||||||
|
break;
|
||||||
|
case "explicit":
|
||||||
|
post.rating = "unsafe";
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set tags
|
||||||
|
const tagElements = Array.from(document.querySelectorAll("#tag-sidebar > li")).map(x => x as HTMLLIElement);
|
||||||
|
for (const tagElement of tagElements) {
|
||||||
|
let tagName: string | undefined;
|
||||||
|
const tagNameElement = tagElement.querySelector("a") as HTMLAnchorElement;
|
||||||
|
if (tagNameElement) {
|
||||||
|
tagName = tagNameElement.innerText;
|
||||||
|
}
|
||||||
|
|
||||||
|
let category: TagCategory | undefined;
|
||||||
|
switch (tagElement.className) {
|
||||||
|
case "tag-type-copyright":
|
||||||
|
category = "copyright";
|
||||||
|
break;
|
||||||
|
case "tag-type-character":
|
||||||
|
category = "character";
|
||||||
|
break;
|
||||||
|
case "tag-type-artist":
|
||||||
|
category = "artist";
|
||||||
|
break;
|
||||||
|
case "tag-type-medium":
|
||||||
|
category = "meta";
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (tagName) {
|
||||||
|
let tag = new ScrapedTag(tagName, category);
|
||||||
|
post.tags.push(tag);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (post.imageUrl) {
|
||||||
|
result.posts.push(post);
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -3,6 +3,7 @@ export { default as Fallback } from "./Fallback";
|
||||||
export { default as Gelbooru } from "./Gelbooru";
|
export { default as Gelbooru } from "./Gelbooru";
|
||||||
export { default as Moebooru } from "./Moebooru";
|
export { default as Moebooru } from "./Moebooru";
|
||||||
export { default as Reddit } from "./Reddit";
|
export { default as Reddit } from "./Reddit";
|
||||||
|
export { default as SankakuComplex } from "./SankakuComplex";
|
||||||
export { default as Shimmie2 } from "./Shimmie2";
|
export { default as Shimmie2 } from "./Shimmie2";
|
||||||
export { default as Twitter } from "./Twitter";
|
export { default as Twitter } from "./Twitter";
|
||||||
export { default as Zerochan } from "./Zerochan";
|
export { default as Zerochan } from "./Zerochan";
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue