Nes
/
neo-scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add direct video import and a lot more. 

- Add support for importing direct video urls
- Add contentType property (image/video)
- Add referrer property (for use with HTTP 'Referer' header)
pull/5/head
neobooru 2021-02-01 00:34:22 +01:00
parent 6f6ba711ce
commit e1df85b44a
20 changed files with 512 additions and 313 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
package.json
View File

@ -1,6 +1,6 @@
{
"name": "neo-scraper",
"version": "0.3.1",
"version": "0.4.0",
"description": "Scrape a booru-like post object from the DOM.",
"main": "dist/index.js",
"types": "dist/index.d.js",

158
src/NeoScraper.test.ts
View File

@ -2,132 +2,142 @@ import { JSDOM } from "jsdom";
import NeoScraper from "./NeoScraper";
async function scrapeUrl(url: string) {
const scraper = new NeoScraper();
const dom = await JSDOM.fromURL(url);
return scraper.scrapeDocument(dom.window.document);
const scraper = new NeoScraper();
const dom = await JSDOM.fromURL(url);
return scraper.scrapeDocument(dom.window.document);
}
beforeAll(() => {
// Expose JSDOM Element constructor
// @ts-ignore
global.Element = (new JSDOM()).window.Element;
// 'Implement' innerText in JSDOM: https://github.com/jsdom/jsdom/issues/1245
// @ts-ignore
Object.defineProperty(global.Element.prototype, 'innerText', {
get() {
return this.textContent.replace(/(\r\n|\n|\r|^\s+|\s+$)/gm, "");;
},
});
// Expose JSDOM Element constructor
// @ts-ignore
global.Element = new JSDOM().window.Element;
// 'Implement' innerText in JSDOM: https://github.com/jsdom/jsdom/issues/1245
// @ts-ignore
Object.defineProperty(global.Element.prototype, "innerText", {
get() {
return this.textContent.replace(/(\r\n|\n|\r|^\s+|\s+$)/gm, "");
},
});
});
test("create scraper with default engines", async () => {
const scraper = new NeoScraper();
const scraper = new NeoScraper();
});
test("all default scrapers have unique names", () => {
const scraper = new NeoScraper();
let alreadySeen: string[] = [];
const scraper = new NeoScraper();
let alreadySeen: string[] = [];
for (const engine of scraper.engines) {
if (alreadySeen.indexOf(engine.name) != -1) {
fail(engine.name + " is not a unique name!");
} else {
alreadySeen.push(engine.name);
}
for (const engine of scraper.engines) {
if (alreadySeen.indexOf(engine.name) != -1) {
fail(engine.name + " is not a unique name!");
} else {
alreadySeen.push(engine.name);
}
}
});
// Danbooru
// All links are SFW unless stated otherwise
test("get danbooru.donmai.us post 3555872", async () => {
var pageUrl = "https://danbooru.donmai.us/posts/3555872";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].imageUrl).toBe("https://danbooru.donmai.us/data/__flint_girls_und_panzer_drawn_by_omachi_slabco__90332299216db81b8793084e4cb7c15d.jpg");
var pageUrl = "https://danbooru.donmai.us/posts/3555872";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://danbooru.donmai.us/data/__flint_girls_und_panzer_drawn_by_omachi_slabco__90332299216db81b8793084e4cb7c15d.jpg"
);
});
// Gelbooru
test("get safebooru.org post 2871532", async () => {
var pageUrl = "https://safebooru.org/index.php?page=post&s=view&id=2871532";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].imageUrl).toBe("https://safebooru.org//images/2757/5f19c7744db918c3d4c4c6035bc65dfa9e18866f.jpg");
var pageUrl = "https://safebooru.org/index.php?page=post&s=view&id=2871532";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://safebooru.org//images/2757/5f19c7744db918c3d4c4c6035bc65dfa9e18866f.jpg"
);
});
test("get gelbooru.com post 4925055", async () => {
var pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=4925055&tags=kimetsu_no_yaiba";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].imageUrl).toBe("https://img2.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
var pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=4925055&tags=kimetsu_no_yaiba";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe("https://img2.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
});
test("get rule34.xxx post 2740994", async () => {
var pageUrl = "https://rule34.xxx/index.php?page=post&s=view&id=2740994";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].imageUrl).toContain(".rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg");
var pageUrl = "https://rule34.xxx/index.php?page=post&s=view&id=2740994";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toContain(".rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg");
});
// Moebooru
test("get yande.re post 438510", async () => {
var pageUrl = "https://yande.re/post/show/438510";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].imageUrl).toBe("https://files.yande.re/image/49b79bc372716a579d8525dcaccb56e2/yande.re%20438510%20landscape%20link%20nintendo%20princess_zelda%20sword%20the_legend_of_zelda%20the_legend_of_zelda%3A_breath_of_the_wild.jpg");
var pageUrl = "https://yande.re/post/show/438510";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://files.yande.re/image/49b79bc372716a579d8525dcaccb56e2/yande.re%20438510%20landscape%20link%20nintendo%20princess_zelda%20sword%20the_legend_of_zelda%20the_legend_of_zelda%3A_breath_of_the_wild.jpg"
);
});
test("get konachan.com post 289968", async () => {
var pageUrl = "https://konachan.com/post/show/289968/ahri_-league_of_legends-akali-animal_ears-blonde_h";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].imageUrl).toBe("https://konachan.com/image/e40b38fb704434e4da3eae41e26bca39/Konachan.com%20-%20289968%20akali%20blue_eyes%20breasts%20choker%20cleavage%20evelynn%20foxgirl%20gloves%20group%20hat%20kai%27sa%20long_hair%20microphone%20ponytail%20saphirya%20short_hair%20signed%20watermark.jpg");
var pageUrl = "https://konachan.com/post/show/289968/ahri_-league_of_legends-akali-animal_ears-blonde_h";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://konachan.com/image/e40b38fb704434e4da3eae41e26bca39/Konachan.com%20-%20289968%20akali%20blue_eyes%20breasts%20choker%20cleavage%20evelynn%20foxgirl%20gloves%20group%20hat%20kai%27sa%20long_hair%20microphone%20ponytail%20saphirya%20short_hair%20signed%20watermark.jpg"
);
});
// Reddit
test("get old.reddit.com post", async () => {
var pageUrl = "https://old.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].imageUrl).toBe("https://i.redd.it/ln8ahz17kpa31.jpg");
var pageUrl = "https://old.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe("https://i.redd.it/ln8ahz17kpa31.jpg");
});
test("get new.reddit.com post", async () => {
var pageUrl = "https://new.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].imageUrl).toBe("https://i.redd.it/ln8ahz17kpa31.jpg");
var pageUrl = "https://new.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe("https://i.redd.it/ln8ahz17kpa31.jpg");
});
// Shimmie2
test("get rule34.paheal.net post 2505982", async () => {
var pageUrl = "https://rule34.paheal.net/post/view/2505982"; // Somewhat sketchy
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].imageUrl).toContain(".paheal.net/_images/15427ab5ed2cddbeef947e30f5c9cab6/2505982%20-%20Kai%27Sa%20League_of_Legends%20lawzilla.png");
var pageUrl = "https://rule34.paheal.net/post/view/2505982"; // Somewhat sketchy
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toContain(
".paheal.net/_images/15427ab5ed2cddbeef947e30f5c9cab6/2505982%20-%20Kai%27Sa%20League_of_Legends%20lawzilla.png"
);
});
// Twitter
test("get twitter.com post", async () => {
pending("JSDOM can't parse this page");
// TODO: JSDOM sucks
// var pageUrl = "https://twitter.com/Dishwasher1910/status/1171162651461746688/photo/1";
// const res = await scrapeUrl(pageUrl);
// expect(res.posts.length).toBe(1);
// expect(res.posts[0].pageUrl).toBe(pageUrl);
// expect(res.posts[0].imageUrl).toBe("https://pbs.twimg.com/media/EEDOOCdU0AAHX00?format=jpg&name=orig");
pending("JSDOM can't parse this page");
// TODO: JSDOM sucks
// var pageUrl = "https://twitter.com/Dishwasher1910/status/1171162651461746688/photo/1";
// const res = await scrapeUrl(pageUrl);
// expect(res.posts.length).toBe(1);
// expect(res.posts[0].pageUrl).toBe(pageUrl);
// expect(res.posts[0].contentUrl).toBe("https://pbs.twimg.com/media/EEDOOCdU0AAHX00?format=jpg&name=orig");
});

91
src/NeoScraper.ts
View File

@ -1,51 +1,56 @@
import { ScrapeEngine, ScrapeResults, } from './ScrapeEngine';
import { ScrapeEngine, ScrapeResults } from "./ScrapeEngine";
import * as engines from "./engines";
export default class NeoScraper {
engines: ScrapeEngine[] = [];
fallbackEngine: ScrapeEngine | null = null;
engines: ScrapeEngine[] = [];
fallbackEngine: ScrapeEngine | null = null;
constructor(useDefaultEngines: boolean = true) {
if (useDefaultEngines) {
this.engines.push(
new engines.AnimePictures(),
new engines.Danbooru(),
new engines.e621(),
new engines.Gelbooru(),
new engines.Moebooru(),
new engines.Pixiv(),
new engines.Reddit(),
new engines.SankakuComplex(),
new engines.Shimmie2(),
new engines.Shuushuu(),
new engines.Twitter(),
new engines.Zerochan(),
);
this.fallbackEngine = new engines.Fallback();
}
constructor(useDefaultEngines: boolean = true) {
if (useDefaultEngines) {
this.engines.push(
new engines.AnimePictures(),
new engines.Danbooru(),
new engines.e621(),
new engines.Gelbooru(),
new engines.Moebooru(),
new engines.Pixiv(),
new engines.Reddit(),
new engines.SankakuComplex(),
new engines.Shimmie2(),
new engines.Shuushuu(),
new engines.Twitter(),
new engines.Zerochan()
);
this.fallbackEngine = new engines.Fallback();
}
}
scrapeDocument(
document: Document,
engines: string[] | null = null,
allowFallbackEngine: boolean = false,
ignoreCanImport: boolean = false
): ScrapeResults {
const res = new ScrapeResults();
for (const engine of this.engines) {
if (ignoreCanImport || engine.canImport(document.location)) {
console.log("Using engine: " + engine.name);
res.results.push(engine.scrapeDocument(document));
}
}
scrapeDocument(document: Document, engines: string[] | null = null, allowFallbackEngine: boolean = false, ignoreCanImport: boolean = false): ScrapeResults {
const res = new ScrapeResults();
for (const engine of this.engines) {
if (ignoreCanImport || engine.canImport(document.location)) {
console.log("Using engine: " + engine.name);
res.results.push(engine.scrapeDocument(document));
}
}
if (allowFallbackEngine) {
if (!this.fallbackEngine) {
console.error("NeoScraper.fallbackEngine is unset!");
} else {
console.log("Using fallback engine");
res.results.push(this.fallbackEngine.scrapeDocument(document));
}
}
console.dir(res);
return res;
if (allowFallbackEngine) {
if (!this.fallbackEngine) {
console.error("NeoScraper.fallbackEngine is unset!");
} else {
console.log("Using fallback engine");
res.results.push(this.fallbackEngine.scrapeDocument(document));
}
}
console.dir(res);
return res;
}
}

12
src/ScrapeEngine.ts
View File

@ -1,11 +1,15 @@
import { TagCategory, SafetyRating } from "./BooruTypes";
export type ContentType = "image" | "video";
export class ScrapedPost {
imageUrl: string = "";
contentUrl: string = "";
pageUrl: string = "";
contentType: ContentType = "image";
rating: SafetyRating = "safe";
tags: ScrapedTag[] = [];
source: string | undefined = undefined;
referrer: string | undefined = undefined;
}
export class ScrapedTag {
@ -28,10 +32,8 @@ export class ScrapeResult {
}
tryAddPost(post: ScrapedPost) {
if (!post.imageUrl) {
console.log(
`[${this.engine}] Not adding post because imageUrl is unset!`
);
if (!post.contentUrl) {
console.log(`[${this.engine}] Not adding post because imageUrl is unset!`);
} else {
this.posts.push(post);
}

21
src/Utility.test.ts Normal file
View File

@ -0,0 +1,21 @@
import { guessContentType } from "./Utility";
test("guessContentType", () => {
expect(guessContentType("video.mp4")).toBe("video");
expect(guessContentType("https://example.com/video.mp4")).toBe("video");
expect(guessContentType("https://example.com/video.mp4?q=r")).toBe("video");
expect(guessContentType("https://example.com/video.mp4?q=r&rrr=fss")).toBe("video");
expect(guessContentType("https://example.com/video.mp4?q=r&rrr=fss,ef./s.e")).toBe("video");
expect(guessContentType("file://example.com/video.mp4?q=r&rrr=fss,ef./s.e")).toBe("video");
expect(guessContentType("example.com/video.mp4?q=r&rrr=fss,ef./s.e")).toBe("video");
expect(guessContentType(".com/video.mp4?q=r&rrr=fss,ef./s.e")).toBe("video");
expect(guessContentType("c:\\my\\video.mp4?q=r&rrr=fss,ef./s.e")).toBe("video");
expect(guessContentType("")).toBe("image");
expect(guessContentType("08435n gf g =0324")).toBe("image");
expect(guessContentType("https://example.com/.mp4/image.png")).toBe("image");
expect(guessContentType("https://example.com/file.mp4.png")).toBe("image");
expect(guessContentType("https://example.com/.mp4.png")).toBe("image");
expect(guessContentType(undefined as any)).toBe("image");
expect(guessContentType(null as any)).toBe("image");
expect(() => guessContentType(984 as any)).toThrow();
});

50
src/Utility.ts Normal file
View File

@ -0,0 +1,50 @@
import { ContentType } from "./ScrapeEngine";
// https://github.com/sindresorhus/video-extensions/blob/main/video-extensions.json
export const videoExtensions = [
"3g2",
"3gp",
"aaf",
"asf",
"avchd",
"avi",
"drc",
"flv",
"m2v",
"m4p",
"m4v",
"mkv",
"mng",
"mov",
"mp2",
"mp4",
"mpe",
"mpeg",
"mpg",
"mpv",
"mxf",
"nsv",
"ogg",
"ogv",
"qt",
"rm",
"rmvb",
"roq",
"svi",
"vob",
"webm",
"wmv",
"yuv",
];
export function getUrlExtension(url: string) {
return url.split(/[#?]/)[0].split(".").pop()!.trim();
}
export function guessContentType(url: string): ContentType {
if (url != undefined) {
const ext = getUrlExtension(url);
if (videoExtensions.indexOf(ext) != -1) return "video";
}
return "image";
}

93
src/engines/AnimePictures.ts
View File

@ -1,56 +1,63 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes";
import { guessContentType } from "../Utility";
export default class AnimePictures implements ScrapeEngine {
name = "animepictures";
name = "animepictures";
canImport(url: Location): boolean {
return url.host == "anime-pictures.net";
canImport(url: Location): boolean {
return url.host == "anime-pictures.net";
}
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
let post = new ScrapedPost();
post.source = document.location.href;
// Set image url
const originalImageElements = Array.from(document.getElementsByClassName("download_icon"));
if (originalImageElements.length > 0) {
post.contentUrl = (originalImageElements[0] as HTMLAnchorElement).href;
} else {
// No point in continuing when we don't have an image.
return result;
}
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
let post = new ScrapedPost();
post.source = document.location.href;
// Set content type
post.contentType = guessContentType(post.contentUrl);
// Set image url
const originalImageElements = Array.from(document.getElementsByClassName("download_icon"));
// Set rating
post.rating = "safe"; // Probably
if (originalImageElements.length > 0) {
post.imageUrl = (originalImageElements[0] as HTMLAnchorElement).href;
// Set tags
const tagElements = Array.from(document.querySelectorAll("ul.tags > li > a")).map((x) => x as HTMLAnchorElement);
for (const tagElement of tagElements) {
let tagName = tagElement.innerText;
let category: TagCategory | undefined;
for (const className of tagElement.classList) {
switch (className) {
case "copyright":
category = "copyright";
break;
case "character":
category = "character";
break;
case "artist":
category = "artist";
break;
}
}
// Set rating
post.rating = "safe"; // Probably
// Set tags
const tagElements = Array.from(document.querySelectorAll("ul.tags > li > a")).map(x => x as HTMLAnchorElement);
for (const tagElement of tagElements) {
let tagName = tagElement.innerText;
let category: TagCategory | undefined;
for (const className of tagElement.classList) {
switch (className) {
case "copyright":
category = "copyright";
break;
case "character":
category = "character";
break;
case "artist":
category = "artist";
break;
}
}
if (tagName) {
let tag = new ScrapedTag(tagName, category);
post.tags.push(tag);
}
}
result.tryAddPost(post);
return result;
if (tagName) {
let tag = new ScrapedTag(tagName, category);
post.tags.push(tag);
}
}
result.tryAddPost(post);
return result;
}
}

13
src/engines/Danbooru.ts
View File

@ -1,5 +1,6 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes";
import { guessContentType } from "../Utility";
export default class Danbooru implements ScrapeEngine {
name = "danbooru";
@ -19,7 +20,17 @@ export default class Danbooru implements ScrapeEngine {
);
if (originalImageElements.length > 0) {
post.imageUrl = (originalImageElements[0] as HTMLAnchorElement).href;
post.contentUrl = (originalImageElements[0] as HTMLAnchorElement).href;
} else {
// No point in continuing when we don't have an image.
return result;
}
// Set content type
if (document.querySelector(".image-container > video") != null) {
post.contentType = "video";
} else {
post.contentType = guessContentType(post.contentUrl);
}
// Set rating

35
src/engines/Fallback.ts
View File

@ -1,4 +1,12 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine";
import { ScrapeEngine, ScrapeResult, ScrapedPost, ContentType } from "../ScrapeEngine";
class MediaElement {
constructor(
public readonly contentType: ContentType,
public readonly contentUrl: string,
public readonly size: number
) {}
}
export default class Fallback implements ScrapeEngine {
name = "fallback";
@ -8,21 +16,30 @@ export default class Fallback implements ScrapeEngine {
}
scrapeDocument(document: Document): ScrapeResult {
function getSize(el: HTMLImageElement) {
return el.width * el.height;
}
// Get all img elements.
const largestImgElement = Array.from(document.getElementsByTagName("img")).map(
(x) => new MediaElement("image", x.src, x.width * x.height)
);
// Get largest img element on this page
const largestImgElement = Array.from(document.getElementsByTagName("img")).reduce((a, b) =>
getSize(a) > getSize(b) ? a : b
// Get all video elements that have at least one source (child node).
const largestVideoElement = Array.from(document.getElementsByTagName("video")).flatMap((x) => {
const source = x.querySelector("source")?.src;
if (source != undefined) return new MediaElement("video", source, x.videoWidth * x.videoHeight);
return []; // Skip this item because it does not have a source.
});
// Get the largest media element.
const largestMediaElement = [...largestImgElement, ...largestVideoElement].reduce((a, b) =>
a?.size > b?.size ? a : b
);
let result = new ScrapeResult(this.name);
if (largestImgElement) {
if (largestMediaElement) {
let post = new ScrapedPost();
post.pageUrl = document.location.href;
post.imageUrl = largestImgElement.src;
post.contentUrl = largestMediaElement.contentUrl;
post.contentType = largestMediaElement.contentType;
result.posts.push(post);
}

14
src/engines/Gelbooru.ts
View File

@ -1,5 +1,6 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes";
import { guessContentType } from "../Utility";
// Small hack to avoid screwing with semver
enum Version {
@ -44,7 +45,18 @@ export default class Gelbooru implements ScrapeEngine {
Array.from(document.querySelectorAll("li > a"))
.map((x) => x as HTMLAnchorElement)
.filter((x) => x && x.innerText == "Original image")
.map((x) => (post.imageUrl = x.href));
.map((x) => (post.contentUrl = x.href));
if (post.contentUrl == undefined) {
return result;
}
// Set content type
if (document.getElementById("gelcomVideoPlayer") != null) {
post.contentType = "video";
} else {
post.contentType = guessContentType(post.contentUrl);
}
// Set rating and source
const regex = new RegExp("(.*): (.*)");

9
src/engines/Moebooru.ts
View File

@ -1,5 +1,6 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes";
import { guessContentType } from "../Utility";
export default class Moebooru implements ScrapeEngine {
name = "moebooru";
@ -16,9 +17,15 @@ export default class Moebooru implements ScrapeEngine {
// Set image url
const originalImageElement = document.querySelector("#highres") as HTMLAnchorElement;
if (originalImageElement) {
post.imageUrl = originalImageElement.href;
post.contentUrl = originalImageElement.href;
} else {
// No point in continuing when we don't have an image.
return result;
}
// Set content type
post.contentType = guessContentType(post.contentUrl);
// Set rating and source
// Same method as Gelbooru v025.
const regex = new RegExp("(.*): (.*)");

42
src/engines/Pixiv.ts
View File

@ -1,28 +1,30 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes";
import { guessContentType } from "../Utility";
export default class Pixiv implements ScrapeEngine {
name = "pixiv";
name = "pixiv";
canImport(url: Location): boolean {
return (
url.host == "pixiv.net" ||
url.host == "www.pixiv.net"
);
canImport(url: Location): boolean {
return url.host == "pixiv.net" || url.host == "www.pixiv.net";
}
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
const imgLinks = Array.from(document.querySelectorAll("div[role='presentation'] > a")).map(
(x) => x as HTMLAnchorElement
);
for (const imgLink of imgLinks) {
let post = new ScrapedPost();
post.pageUrl = document.location.href;
post.contentUrl = imgLink.href;
post.contentType = guessContentType(post.contentUrl);
post.referrer = document.location.origin;
result.tryAddPost(post);
}
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
const imgLinks = Array.from(document.querySelectorAll("div[role='presentation'] > a")).map(x => x as HTMLAnchorElement);
for (const imgLink of imgLinks) {
let post = new ScrapedPost();
post.pageUrl = document.location.href;
post.imageUrl = imgLink.href;
result.tryAddPost(post);
}
return result;
}
return result;
}
}

53
src/engines/Reddit.ts
View File

@ -1,32 +1,37 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine";
import { guessContentType } from "../Utility";
export default class Reddit implements ScrapeEngine {
name = "reddit";
name = "reddit";
canImport(url: Location): boolean {
return url.host.endsWith("reddit.com");
canImport(url: Location): boolean {
return url.host.endsWith("reddit.com");
}
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
let post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url
// For old.reddit.com
const oldLinkElements = document.querySelectorAll("a.title");
if (oldLinkElements.length > 0) {
post.contentUrl = (oldLinkElements[0] as HTMLAnchorElement).href;
}
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
let post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url
// For old.reddit.com
const oldLinkElements = document.querySelectorAll("a.title");
if (oldLinkElements.length > 0) {
post.imageUrl = (oldLinkElements[0] as HTMLAnchorElement).href;
}
// For new reddit
const newLinkElements = document.querySelectorAll("div[data-test-id='post-content'] > div > a");
if (newLinkElements.length > 0) {
post.imageUrl = (newLinkElements[0] as HTMLAnchorElement).href;
}
result.tryAddPost(post);
return result;
// For new reddit
const newLinkElements = document.querySelectorAll("div[data-test-id='post-content'] > div > a");
if (newLinkElements.length > 0) {
post.contentUrl = (newLinkElements[0] as HTMLAnchorElement).href;
}
if (post.contentUrl != undefined) {
post.contentType = guessContentType(post.contentUrl);
}
result.tryAddPost(post);
return result;
}
}

9
src/engines/SankakuComplex.ts
View File

@ -1,5 +1,6 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes";
import { guessContentType } from "../Utility";
export default class SankakuComplex implements ScrapeEngine {
name = "sankakucomplex";
@ -17,9 +18,15 @@ export default class SankakuComplex implements ScrapeEngine {
const originalImageElement = document.getElementById("highres") as HTMLAnchorElement;
if (originalImageElement) {
post.imageUrl = originalImageElement.href;
post.contentUrl = originalImageElement.href;
} else {
// No point in continuing when we don't have an image.
return result;
}
// Set content type
post.contentType = guessContentType(post.contentUrl);
// Set rating
const safetyExp = new RegExp("Rating: (.*)");
const safetyElements = Array.from(document.querySelectorAll("#stats > ul > li"))

12
src/engines/Shimmie2.ts
View File

@ -1,4 +1,5 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { guessContentType } from "../Utility";
export default class Shimmie2 implements ScrapeEngine {
name = "shimmie2";
@ -18,14 +19,21 @@ export default class Shimmie2 implements ScrapeEngine {
.filter((x) => x.innerText == "Image Only" || x.innerText == "File Only");
if (originalImageElements.length > 0) {
post.imageUrl = originalImageElements[0].href;
post.contentUrl = originalImageElements[0].href;
} else {
const dowloadBtnEl = document.querySelector("a[download]") as HTMLAnchorElement;
if (dowloadBtnEl) {
post.imageUrl = dowloadBtnEl.href;
post.contentUrl = dowloadBtnEl.href;
}
}
if (post.contentUrl == undefined) {
return result;
}
// Set content type
post.contentType = guessContentType(post.contentUrl);
// Set safety
post.rating = "unsafe"; // Usually it is unsafe

117
src/engines/Shuushuu.ts
View File

@ -1,65 +1,74 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes";
import { guessContentType } from "../Utility";
export default class Shuushuu implements ScrapeEngine {
name = "shuushuu";
name = "shuushuu";
canImport(url: Location): boolean {
return url.host == "e-shuushuu.net";
canImport(url: Location): boolean {
return url.host == "e-shuushuu.net";
}
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
let post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url
const imgElement = document.querySelector(".thumb_image") as HTMLAnchorElement;
if (imgElement) {
post.contentUrl = imgElement.href;
} else {
// No point in continuing when we don't have an image.
return result;
}
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
let post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set content type
post.contentType = guessContentType(post.contentUrl);
// Set image url
const imgElement = document.querySelector(".thumb_image") as HTMLAnchorElement;
if (imgElement) {
post.imageUrl = imgElement.href;
// Set rating
post.rating = "safe";
// Set tags
const exp = new RegExp(/(.*)_(\w+)/);
const tagElements = document.getElementsByClassName("quicktag");
for (const tagElement of tagElements) {
// id = quicktag1_1009628
// match[0] = quicktag1_1009628 (aka input str)
// match[1] = quicktag1
// match[2] = 1009628
const match = exp.exec(tagElement.id);
if (!match || match.length != 3) {
continue;
}
let tagCategory: TagCategory | undefined;
switch (match[1]) {
// Source, aka copyright
case "quicktag2":
tagCategory = "copyright";
break;
// Artist
case "quicktag3":
tagCategory = "artist";
break;
// Characters
case "quicktag4":
tagCategory = "character";
break;
}
for (const el of tagElement.children) {
const tagName = (el?.firstElementChild as HTMLAnchorElement)?.innerText;
if (tagName) {
post.tags.push(new ScrapedTag(tagName, tagCategory));
}
// Set rating
post.rating = "safe";
// Set tags
const exp = new RegExp(/(.*)_(\w+)/);
const tagElements = document.getElementsByClassName("quicktag");
for (const tagElement of tagElements) {
// id = quicktag1_1009628
// match[0] = quicktag1_1009628 (aka input str)
// match[1] = quicktag1
// match[2] = 1009628
const match = exp.exec(tagElement.id);
if (!match || match.length != 3) { continue; }
let tagCategory: TagCategory | undefined;
switch (match[1]) {
// Source, aka copyright
case "quicktag2":
tagCategory = "copyright";
break;
// Artist
case "quicktag3":
tagCategory = "artist";
break;
// Characters
case "quicktag4":
tagCategory = "character";
break;
}
for (const el of tagElement.children) {
const tagName = (el?.firstElementChild as HTMLAnchorElement)?.innerText;
if (tagName) {
post.tags.push(new ScrapedTag(tagName, tagCategory));
}
}
}
result.tryAddPost(post);
return result;
}
}
result.tryAddPost(post);
return result;
}
}

66
src/engines/Twitter.ts
View File

@ -1,40 +1,48 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine";
import { guessContentType } from "../Utility";
export function getOriginalImageUrl(imageUrl: string) {
const x = imageUrl.replace(/:large|:med|:small|:orig$/, ""); // Remove `:large`, `:small` etc from url
return x + ":orig"; // Add :orig to url (to get original image)
const x = imageUrl.replace(/:large|:med|:small|:orig$/, ""); // Remove `:large`, `:small` etc from url
return x + ":orig"; // Add :orig to url (to get original image)
}
export default class Twitter implements ScrapeEngine {
name = "twitter";
name = "twitter";
canImport(url: Location): boolean {
return url.host == "twitter.com";
canImport(url: Location): boolean {
return url.host == "twitter.com";
}
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
let post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url
// When clicked on post (not fullscreen, with post text etc)
const overlayElement = document.querySelector("#permalink-overlay-dialog");
if (overlayElement) {
const cardImageElement = overlayElement.querySelector("div[data-element-context='platform_photo_card'] > img");
if (cardImageElement) {
post.contentUrl = getOriginalImageUrl((cardImageElement as HTMLImageElement).src);
}
}
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
let post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url
// When clicked on post (not fullscreen, with post text etc)
const overlayElement = document.querySelector("#permalink-overlay-dialog");
if (overlayElement) {
const cardImageElement = overlayElement.querySelector("div[data-element-context='platform_photo_card'] > img");
if (cardImageElement) {
post.imageUrl = getOriginalImageUrl((cardImageElement as HTMLImageElement).src);
}
}
// When clicked on image (fullscreen)
const mediaImageElements = document.querySelectorAll("div.Gallery-media > img.media-image");
if (mediaImageElements.length > 0) {
post.imageUrl = getOriginalImageUrl((mediaImageElements[0] as HTMLImageElement).src);
}
result.tryAddPost(post);
return result;
// When clicked on image (fullscreen)
const mediaImageElements = document.querySelectorAll("div.Gallery-media > img.media-image");
if (mediaImageElements.length > 0) {
post.contentUrl = getOriginalImageUrl((mediaImageElements[0] as HTMLImageElement).src);
}
if (post.contentUrl == undefined) {
return result;
}
// Set content type
post.contentType = guessContentType(post.contentUrl);
result.tryAddPost(post);
return result;
}
}

13
src/engines/Zerochan.ts
View File

@ -1,5 +1,6 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes";
import { guessContentType } from "../Utility";
export default class Zerochan implements ScrapeEngine {
name = "zerochan";
@ -17,15 +18,23 @@ export default class Zerochan implements ScrapeEngine {
const previewElements = document.getElementsByClassName("preview");
if (previewElements.length > 0) {
// The preview element only exists when there is a larger size available. Preview's img element contains a downscaled version.
post.imageUrl = (previewElements[0] as HTMLAnchorElement).href;
post.contentUrl = (previewElements[0] as HTMLAnchorElement).href;
} else {
// When there is no larger version available just use the displayed version.
const imgElement = document.querySelector("img[title='No larger size available']") as HTMLImageElement;
if (imgElement) {
post.imageUrl = imgElement.src;
post.contentUrl = imgElement.src;
}
}
if (post.contentUrl == undefined) {
// No point in continuing when we don't have an image.
return result;
}
// Set content type
post.contentType = guessContentType(post.contentUrl);
// Set rating
post.rating = "safe";

11
src/engines/e621.ts
View File

@ -16,7 +16,16 @@ export default class e621 implements ScrapeEngine {
// Set image url
const downloadEl = document.querySelector("#image-download-link > a") as HTMLAnchorElement;
post.imageUrl = downloadEl?.href;
post.contentUrl = downloadEl?.href;
if (post.contentUrl == undefined) {
return result;
}
// Set content type
if (document.querySelector("video#image") != null) {
post.contentType = "video";
}
// Set rating
const ratingEl = document.getElementById("post-rating-text") as HTMLSpanElement;

2
tsconfig.json
View File

@ -1,7 +1,7 @@
{
"compilerOptions": {
"lib": [
"es6",
"es2019",
"dom",
"dom.iterable"
],