Nes
/
neo-scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add direct video import and a lot more. 

- Add support for importing direct video urls
- Add contentType property (image/video)
- Add referrer property (for use with HTTP 'Referer' header)
pull/5/head
neobooru 2021-02-01 00:34:22 +01:00
parent 6f6ba711ce
commit e1df85b44a
20 changed files with 512 additions and 313 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "neo-scraper", "name": "neo-scraper",
"version": "0.3.1", "version": "0.4.0",
"description": "Scrape a booru-like post object from the DOM.", "description": "Scrape a booru-like post object from the DOM.",
"main": "dist/index.js", "main": "dist/index.js",
"types": "dist/index.d.js", "types": "dist/index.d.js",

158
src/NeoScraper.test.ts
View File

@ -2,132 +2,142 @@ import { JSDOM } from "jsdom";
import NeoScraper from "./NeoScraper"; import NeoScraper from "./NeoScraper";
async function scrapeUrl(url: string) { async function scrapeUrl(url: string) {
const scraper = new NeoScraper(); const scraper = new NeoScraper();
const dom = await JSDOM.fromURL(url); const dom = await JSDOM.fromURL(url);
return scraper.scrapeDocument(dom.window.document); return scraper.scrapeDocument(dom.window.document);
} }
beforeAll(() => { beforeAll(() => {
// Expose JSDOM Element constructor // Expose JSDOM Element constructor
// @ts-ignore // @ts-ignore
global.Element = (new JSDOM()).window.Element; global.Element = new JSDOM().window.Element;
// 'Implement' innerText in JSDOM: https://github.com/jsdom/jsdom/issues/1245 // 'Implement' innerText in JSDOM: https://github.com/jsdom/jsdom/issues/1245
// @ts-ignore // @ts-ignore
Object.defineProperty(global.Element.prototype, 'innerText', { Object.defineProperty(global.Element.prototype, "innerText", {
get() { get() {
return this.textContent.replace(/(\r\n|\n|\r|^\s+|\s+$)/gm, "");; return this.textContent.replace(/(\r\n|\n|\r|^\s+|\s+$)/gm, "");
}, },
}); });
}); });
test("create scraper with default engines", async () => { test("create scraper with default engines", async () => {
const scraper = new NeoScraper(); const scraper = new NeoScraper();
}); });
test("all default scrapers have unique names", () => { test("all default scrapers have unique names", () => {
const scraper = new NeoScraper(); const scraper = new NeoScraper();
let alreadySeen: string[] = []; let alreadySeen: string[] = [];
for (const engine of scraper.engines) { for (const engine of scraper.engines) {
if (alreadySeen.indexOf(engine.name) != -1) { if (alreadySeen.indexOf(engine.name) != -1) {
fail(engine.name + " is not a unique name!"); fail(engine.name + " is not a unique name!");
} else { } else {
alreadySeen.push(engine.name); alreadySeen.push(engine.name);
}
} }
}
}); });
// Danbooru // Danbooru
// All links are SFW unless stated otherwise // All links are SFW unless stated otherwise
test("get danbooru.donmai.us post 3555872", async () => { test("get danbooru.donmai.us post 3555872", async () => {
var pageUrl = "https://danbooru.donmai.us/posts/3555872"; var pageUrl = "https://danbooru.donmai.us/posts/3555872";
const res = await scrapeUrl(pageUrl); const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].imageUrl).toBe("https://danbooru.donmai.us/data/__flint_girls_und_panzer_drawn_by_omachi_slabco__90332299216db81b8793084e4cb7c15d.jpg"); expect(res.posts[0].contentUrl).toBe(
"https://danbooru.donmai.us/data/__flint_girls_und_panzer_drawn_by_omachi_slabco__90332299216db81b8793084e4cb7c15d.jpg"
);
}); });
// Gelbooru // Gelbooru
test("get safebooru.org post 2871532", async () => { test("get safebooru.org post 2871532", async () => {
var pageUrl = "https://safebooru.org/index.php?page=post&s=view&id=2871532"; var pageUrl = "https://safebooru.org/index.php?page=post&s=view&id=2871532";
const res = await scrapeUrl(pageUrl); const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].imageUrl).toBe("https://safebooru.org//images/2757/5f19c7744db918c3d4c4c6035bc65dfa9e18866f.jpg"); expect(res.posts[0].contentUrl).toBe(
"https://safebooru.org//images/2757/5f19c7744db918c3d4c4c6035bc65dfa9e18866f.jpg"
);
}); });
test("get gelbooru.com post 4925055", async () => { test("get gelbooru.com post 4925055", async () => {
var pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=4925055&tags=kimetsu_no_yaiba"; var pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=4925055&tags=kimetsu_no_yaiba";
const res = await scrapeUrl(pageUrl); const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].imageUrl).toBe("https://img2.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg"); expect(res.posts[0].contentUrl).toBe("https://img2.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
}); });
test("get rule34.xxx post 2740994", async () => { test("get rule34.xxx post 2740994", async () => {
var pageUrl = "https://rule34.xxx/index.php?page=post&s=view&id=2740994"; var pageUrl = "https://rule34.xxx/index.php?page=post&s=view&id=2740994";
const res = await scrapeUrl(pageUrl); const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].imageUrl).toContain(".rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg"); expect(res.posts[0].contentUrl).toContain(".rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg");
}); });
// Moebooru // Moebooru
test("get yande.re post 438510", async () => { test("get yande.re post 438510", async () => {
var pageUrl = "https://yande.re/post/show/438510"; var pageUrl = "https://yande.re/post/show/438510";
const res = await scrapeUrl(pageUrl); const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].imageUrl).toBe("https://files.yande.re/image/49b79bc372716a579d8525dcaccb56e2/yande.re%20438510%20landscape%20link%20nintendo%20princess_zelda%20sword%20the_legend_of_zelda%20the_legend_of_zelda%3A_breath_of_the_wild.jpg"); expect(res.posts[0].contentUrl).toBe(
"https://files.yande.re/image/49b79bc372716a579d8525dcaccb56e2/yande.re%20438510%20landscape%20link%20nintendo%20princess_zelda%20sword%20the_legend_of_zelda%20the_legend_of_zelda%3A_breath_of_the_wild.jpg"
);
}); });
test("get konachan.com post 289968", async () => { test("get konachan.com post 289968", async () => {
var pageUrl = "https://konachan.com/post/show/289968/ahri_-league_of_legends-akali-animal_ears-blonde_h"; var pageUrl = "https://konachan.com/post/show/289968/ahri_-league_of_legends-akali-animal_ears-blonde_h";
const res = await scrapeUrl(pageUrl); const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].imageUrl).toBe("https://konachan.com/image/e40b38fb704434e4da3eae41e26bca39/Konachan.com%20-%20289968%20akali%20blue_eyes%20breasts%20choker%20cleavage%20evelynn%20foxgirl%20gloves%20group%20hat%20kai%27sa%20long_hair%20microphone%20ponytail%20saphirya%20short_hair%20signed%20watermark.jpg"); expect(res.posts[0].contentUrl).toBe(
"https://konachan.com/image/e40b38fb704434e4da3eae41e26bca39/Konachan.com%20-%20289968%20akali%20blue_eyes%20breasts%20choker%20cleavage%20evelynn%20foxgirl%20gloves%20group%20hat%20kai%27sa%20long_hair%20microphone%20ponytail%20saphirya%20short_hair%20signed%20watermark.jpg"
);
}); });
// Reddit // Reddit
test("get old.reddit.com post", async () => { test("get old.reddit.com post", async () => {
var pageUrl = "https://old.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/"; var pageUrl = "https://old.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/";
const res = await scrapeUrl(pageUrl); const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].imageUrl).toBe("https://i.redd.it/ln8ahz17kpa31.jpg"); expect(res.posts[0].contentUrl).toBe("https://i.redd.it/ln8ahz17kpa31.jpg");
}); });
test("get new.reddit.com post", async () => { test("get new.reddit.com post", async () => {
var pageUrl = "https://new.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/"; var pageUrl = "https://new.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/";
const res = await scrapeUrl(pageUrl); const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].imageUrl).toBe("https://i.redd.it/ln8ahz17kpa31.jpg"); expect(res.posts[0].contentUrl).toBe("https://i.redd.it/ln8ahz17kpa31.jpg");
}); });
// Shimmie2 // Shimmie2
test("get rule34.paheal.net post 2505982", async () => { test("get rule34.paheal.net post 2505982", async () => {
var pageUrl = "https://rule34.paheal.net/post/view/2505982"; // Somewhat sketchy var pageUrl = "https://rule34.paheal.net/post/view/2505982"; // Somewhat sketchy
const res = await scrapeUrl(pageUrl); const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].imageUrl).toContain(".paheal.net/_images/15427ab5ed2cddbeef947e30f5c9cab6/2505982%20-%20Kai%27Sa%20League_of_Legends%20lawzilla.png"); expect(res.posts[0].contentUrl).toContain(
".paheal.net/_images/15427ab5ed2cddbeef947e30f5c9cab6/2505982%20-%20Kai%27Sa%20League_of_Legends%20lawzilla.png"
);
}); });
// Twitter // Twitter
test("get twitter.com post", async () => { test("get twitter.com post", async () => {
pending("JSDOM can't parse this page"); pending("JSDOM can't parse this page");
// TODO: JSDOM sucks // TODO: JSDOM sucks
// var pageUrl = "https://twitter.com/Dishwasher1910/status/1171162651461746688/photo/1"; // var pageUrl = "https://twitter.com/Dishwasher1910/status/1171162651461746688/photo/1";
// const res = await scrapeUrl(pageUrl); // const res = await scrapeUrl(pageUrl);
// expect(res.posts.length).toBe(1); // expect(res.posts.length).toBe(1);
// expect(res.posts[0].pageUrl).toBe(pageUrl); // expect(res.posts[0].pageUrl).toBe(pageUrl);
// expect(res.posts[0].imageUrl).toBe("https://pbs.twimg.com/media/EEDOOCdU0AAHX00?format=jpg&name=orig"); // expect(res.posts[0].contentUrl).toBe("https://pbs.twimg.com/media/EEDOOCdU0AAHX00?format=jpg&name=orig");
}); });

91
src/NeoScraper.ts
View File

@ -1,51 +1,56 @@
import { ScrapeEngine, ScrapeResults, } from './ScrapeEngine'; import { ScrapeEngine, ScrapeResults } from "./ScrapeEngine";
import * as engines from "./engines"; import * as engines from "./engines";
export default class NeoScraper { export default class NeoScraper {
engines: ScrapeEngine[] = []; engines: ScrapeEngine[] = [];
fallbackEngine: ScrapeEngine | null = null; fallbackEngine: ScrapeEngine | null = null;
constructor(useDefaultEngines: boolean = true) { constructor(useDefaultEngines: boolean = true) {
if (useDefaultEngines) { if (useDefaultEngines) {
this.engines.push( this.engines.push(
new engines.AnimePictures(), new engines.AnimePictures(),
new engines.Danbooru(), new engines.Danbooru(),
new engines.e621(), new engines.e621(),
new engines.Gelbooru(), new engines.Gelbooru(),
new engines.Moebooru(), new engines.Moebooru(),
new engines.Pixiv(), new engines.Pixiv(),
new engines.Reddit(), new engines.Reddit(),
new engines.SankakuComplex(), new engines.SankakuComplex(),
new engines.Shimmie2(), new engines.Shimmie2(),
new engines.Shuushuu(), new engines.Shuushuu(),
new engines.Twitter(), new engines.Twitter(),
new engines.Zerochan(), new engines.Zerochan()
); );
this.fallbackEngine = new engines.Fallback(); this.fallbackEngine = new engines.Fallback();
} }
}
scrapeDocument(
document: Document,
engines: string[] | null = null,
allowFallbackEngine: boolean = false,
ignoreCanImport: boolean = false
): ScrapeResults {
const res = new ScrapeResults();
for (const engine of this.engines) {
if (ignoreCanImport || engine.canImport(document.location)) {
console.log("Using engine: " + engine.name);
res.results.push(engine.scrapeDocument(document));
}
} }
scrapeDocument(document: Document, engines: string[] | null = null, allowFallbackEngine: boolean = false, ignoreCanImport: boolean = false): ScrapeResults { if (allowFallbackEngine) {
const res = new ScrapeResults(); if (!this.fallbackEngine) {
console.error("NeoScraper.fallbackEngine is unset!");
for (const engine of this.engines) { } else {
if (ignoreCanImport || engine.canImport(document.location)) { console.log("Using fallback engine");
console.log("Using engine: " + engine.name); res.results.push(this.fallbackEngine.scrapeDocument(document));
res.results.push(engine.scrapeDocument(document)); }
}
}
if (allowFallbackEngine) {
if (!this.fallbackEngine) {
console.error("NeoScraper.fallbackEngine is unset!");
} else {
console.log("Using fallback engine");
res.results.push(this.fallbackEngine.scrapeDocument(document));
}
}
console.dir(res);
return res;
} }
console.dir(res);
return res;
}
} }

12
src/ScrapeEngine.ts
View File

@ -1,11 +1,15 @@
import { TagCategory, SafetyRating } from "./BooruTypes"; import { TagCategory, SafetyRating } from "./BooruTypes";
export type ContentType = "image" | "video";
export class ScrapedPost { export class ScrapedPost {
imageUrl: string = ""; contentUrl: string = "";
pageUrl: string = ""; pageUrl: string = "";
contentType: ContentType = "image";
rating: SafetyRating = "safe"; rating: SafetyRating = "safe";
tags: ScrapedTag[] = []; tags: ScrapedTag[] = [];
source: string | undefined = undefined; source: string | undefined = undefined;
referrer: string | undefined = undefined;
} }
export class ScrapedTag { export class ScrapedTag {
@ -28,10 +32,8 @@ export class ScrapeResult {
} }
tryAddPost(post: ScrapedPost) { tryAddPost(post: ScrapedPost) {
if (!post.imageUrl) { if (!post.contentUrl) {
console.log( console.log(`[${this.engine}] Not adding post because imageUrl is unset!`);
`[${this.engine}] Not adding post because imageUrl is unset!`
);
} else { } else {
this.posts.push(post); this.posts.push(post);
} }

21
src/Utility.test.ts Normal file
View File

@ -0,0 +1,21 @@
import { guessContentType } from "./Utility";
test("guessContentType", () => {
expect(guessContentType("video.mp4")).toBe("video");
expect(guessContentType("https://example.com/video.mp4")).toBe("video");
expect(guessContentType("https://example.com/video.mp4?q=r")).toBe("video");
expect(guessContentType("https://example.com/video.mp4?q=r&rrr=fss")).toBe("video");
expect(guessContentType("https://example.com/video.mp4?q=r&rrr=fss,ef./s.e")).toBe("video");
expect(guessContentType("file://example.com/video.mp4?q=r&rrr=fss,ef./s.e")).toBe("video");
expect(guessContentType("example.com/video.mp4?q=r&rrr=fss,ef./s.e")).toBe("video");
expect(guessContentType(".com/video.mp4?q=r&rrr=fss,ef./s.e")).toBe("video");
expect(guessContentType("c:\\my\\video.mp4?q=r&rrr=fss,ef./s.e")).toBe("video");
expect(guessContentType("")).toBe("image");
expect(guessContentType("08435n gf g =0324")).toBe("image");
expect(guessContentType("https://example.com/.mp4/image.png")).toBe("image");
expect(guessContentType("https://example.com/file.mp4.png")).toBe("image");
expect(guessContentType("https://example.com/.mp4.png")).toBe("image");
expect(guessContentType(undefined as any)).toBe("image");
expect(guessContentType(null as any)).toBe("image");
expect(() => guessContentType(984 as any)).toThrow();
});

50
src/Utility.ts Normal file
View File

@ -0,0 +1,50 @@
import { ContentType } from "./ScrapeEngine";
// https://github.com/sindresorhus/video-extensions/blob/main/video-extensions.json
export const videoExtensions = [
"3g2",
"3gp",
"aaf",
"asf",
"avchd",
"avi",
"drc",
"flv",
"m2v",
"m4p",
"m4v",
"mkv",
"mng",
"mov",
"mp2",
"mp4",
"mpe",
"mpeg",
"mpg",
"mpv",
"mxf",
"nsv",
"ogg",
"ogv",
"qt",
"rm",
"rmvb",
"roq",
"svi",
"vob",
"webm",
"wmv",
"yuv",
];
export function getUrlExtension(url: string) {
return url.split(/[#?]/)[0].split(".").pop()!.trim();
}
export function guessContentType(url: string): ContentType {
if (url != undefined) {
const ext = getUrlExtension(url);
if (videoExtensions.indexOf(ext) != -1) return "video";
}
return "image";
}

93
src/engines/AnimePictures.ts
View File

@ -1,56 +1,63 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes"; import { TagCategory } from "../BooruTypes";
import { guessContentType } from "../Utility";
export default class AnimePictures implements ScrapeEngine { export default class AnimePictures implements ScrapeEngine {
name = "animepictures"; name = "animepictures";
canImport(url: Location): boolean { canImport(url: Location): boolean {
return url.host == "anime-pictures.net"; return url.host == "anime-pictures.net";
}
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
let post = new ScrapedPost();
post.source = document.location.href;
// Set image url
const originalImageElements = Array.from(document.getElementsByClassName("download_icon"));
if (originalImageElements.length > 0) {
post.contentUrl = (originalImageElements[0] as HTMLAnchorElement).href;
} else {
// No point in continuing when we don't have an image.
return result;
} }
scrapeDocument(document: Document): ScrapeResult { // Set content type
let result = new ScrapeResult(this.name); post.contentType = guessContentType(post.contentUrl);
let post = new ScrapedPost();
post.source = document.location.href;
// Set image url // Set rating
const originalImageElements = Array.from(document.getElementsByClassName("download_icon")); post.rating = "safe"; // Probably
if (originalImageElements.length > 0) { // Set tags
post.imageUrl = (originalImageElements[0] as HTMLAnchorElement).href; const tagElements = Array.from(document.querySelectorAll("ul.tags > li > a")).map((x) => x as HTMLAnchorElement);
for (const tagElement of tagElements) {
let tagName = tagElement.innerText;
let category: TagCategory | undefined;
for (const className of tagElement.classList) {
switch (className) {
case "copyright":
category = "copyright";
break;
case "character":
category = "character";
break;
case "artist":
category = "artist";
break;
} }
}
// Set rating if (tagName) {
post.rating = "safe"; // Probably let tag = new ScrapedTag(tagName, category);
post.tags.push(tag);
// Set tags }
const tagElements = Array.from(document.querySelectorAll("ul.tags > li > a")).map(x => x as HTMLAnchorElement);
for (const tagElement of tagElements) {
let tagName = tagElement.innerText;
let category: TagCategory | undefined;
for (const className of tagElement.classList) {
switch (className) {
case "copyright":
category = "copyright";
break;
case "character":
category = "character";
break;
case "artist":
category = "artist";
break;
}
}
if (tagName) {
let tag = new ScrapedTag(tagName, category);
post.tags.push(tag);
}
}
result.tryAddPost(post);
return result;
} }
result.tryAddPost(post);
return result;
}
} }

13
src/engines/Danbooru.ts
View File

@ -1,5 +1,6 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes"; import { TagCategory } from "../BooruTypes";
import { guessContentType } from "../Utility";
export default class Danbooru implements ScrapeEngine { export default class Danbooru implements ScrapeEngine {
name = "danbooru"; name = "danbooru";
@ -19,7 +20,17 @@ export default class Danbooru implements ScrapeEngine {
); );
if (originalImageElements.length > 0) { if (originalImageElements.length > 0) {
post.imageUrl = (originalImageElements[0] as HTMLAnchorElement).href; post.contentUrl = (originalImageElements[0] as HTMLAnchorElement).href;
} else {
// No point in continuing when we don't have an image.
return result;
}
// Set content type
if (document.querySelector(".image-container > video") != null) {
post.contentType = "video";
} else {
post.contentType = guessContentType(post.contentUrl);
} }
// Set rating // Set rating

35
src/engines/Fallback.ts
View File

@ -1,4 +1,12 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine"; import { ScrapeEngine, ScrapeResult, ScrapedPost, ContentType } from "../ScrapeEngine";
class MediaElement {
constructor(
public readonly contentType: ContentType,
public readonly contentUrl: string,
public readonly size: number
) {}
}
export default class Fallback implements ScrapeEngine { export default class Fallback implements ScrapeEngine {
name = "fallback"; name = "fallback";
@ -8,21 +16,30 @@ export default class Fallback implements ScrapeEngine {
} }
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
function getSize(el: HTMLImageElement) { // Get all img elements.
return el.width * el.height; const largestImgElement = Array.from(document.getElementsByTagName("img")).map(
} (x) => new MediaElement("image", x.src, x.width * x.height)
);
// Get largest img element on this page // Get all video elements that have at least one source (child node).
const largestImgElement = Array.from(document.getElementsByTagName("img")).reduce((a, b) => const largestVideoElement = Array.from(document.getElementsByTagName("video")).flatMap((x) => {
getSize(a) > getSize(b) ? a : b const source = x.querySelector("source")?.src;
if (source != undefined) return new MediaElement("video", source, x.videoWidth * x.videoHeight);
return []; // Skip this item because it does not have a source.
});
// Get the largest media element.
const largestMediaElement = [...largestImgElement, ...largestVideoElement].reduce((a, b) =>
a?.size > b?.size ? a : b
); );
let result = new ScrapeResult(this.name); let result = new ScrapeResult(this.name);
if (largestImgElement) { if (largestMediaElement) {
let post = new ScrapedPost(); let post = new ScrapedPost();
post.pageUrl = document.location.href; post.pageUrl = document.location.href;
post.imageUrl = largestImgElement.src; post.contentUrl = largestMediaElement.contentUrl;
post.contentType = largestMediaElement.contentType;
result.posts.push(post); result.posts.push(post);
} }

14
src/engines/Gelbooru.ts
View File

@ -1,5 +1,6 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes"; import { TagCategory } from "../BooruTypes";
import { guessContentType } from "../Utility";
// Small hack to avoid screwing with semver // Small hack to avoid screwing with semver
enum Version { enum Version {
@ -44,7 +45,18 @@ export default class Gelbooru implements ScrapeEngine {
Array.from(document.querySelectorAll("li > a")) Array.from(document.querySelectorAll("li > a"))
.map((x) => x as HTMLAnchorElement) .map((x) => x as HTMLAnchorElement)
.filter((x) => x && x.innerText == "Original image") .filter((x) => x && x.innerText == "Original image")
.map((x) => (post.imageUrl = x.href)); .map((x) => (post.contentUrl = x.href));
if (post.contentUrl == undefined) {
return result;
}
// Set content type
if (document.getElementById("gelcomVideoPlayer") != null) {
post.contentType = "video";
} else {
post.contentType = guessContentType(post.contentUrl);
}
// Set rating and source // Set rating and source
const regex = new RegExp("(.*): (.*)"); const regex = new RegExp("(.*): (.*)");

9
src/engines/Moebooru.ts
View File

@ -1,5 +1,6 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes"; import { TagCategory } from "../BooruTypes";
import { guessContentType } from "../Utility";
export default class Moebooru implements ScrapeEngine { export default class Moebooru implements ScrapeEngine {
name = "moebooru"; name = "moebooru";
@ -16,9 +17,15 @@ export default class Moebooru implements ScrapeEngine {
// Set image url // Set image url
const originalImageElement = document.querySelector("#highres") as HTMLAnchorElement; const originalImageElement = document.querySelector("#highres") as HTMLAnchorElement;
if (originalImageElement) { if (originalImageElement) {
post.imageUrl = originalImageElement.href; post.contentUrl = originalImageElement.href;
} else {
// No point in continuing when we don't have an image.
return result;
} }
// Set content type
post.contentType = guessContentType(post.contentUrl);
// Set rating and source // Set rating and source
// Same method as Gelbooru v025. // Same method as Gelbooru v025.
const regex = new RegExp("(.*): (.*)"); const regex = new RegExp("(.*): (.*)");

42
src/engines/Pixiv.ts
View File

@ -1,28 +1,30 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes"; import { TagCategory } from "../BooruTypes";
import { guessContentType } from "../Utility";
export default class Pixiv implements ScrapeEngine { export default class Pixiv implements ScrapeEngine {
name = "pixiv"; name = "pixiv";
canImport(url: Location): boolean { canImport(url: Location): boolean {
return ( return url.host == "pixiv.net" || url.host == "www.pixiv.net";
url.host == "pixiv.net" || }
url.host == "www.pixiv.net"
); scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
const imgLinks = Array.from(document.querySelectorAll("div[role='presentation'] > a")).map(
(x) => x as HTMLAnchorElement
);
for (const imgLink of imgLinks) {
let post = new ScrapedPost();
post.pageUrl = document.location.href;
post.contentUrl = imgLink.href;
post.contentType = guessContentType(post.contentUrl);
post.referrer = document.location.origin;
result.tryAddPost(post);
} }
scrapeDocument(document: Document): ScrapeResult { return result;
let result = new ScrapeResult(this.name); }
const imgLinks = Array.from(document.querySelectorAll("div[role='presentation'] > a")).map(x => x as HTMLAnchorElement);
for (const imgLink of imgLinks) {
let post = new ScrapedPost();
post.pageUrl = document.location.href;
post.imageUrl = imgLink.href;
result.tryAddPost(post);
}
return result;
}
} }

53
src/engines/Reddit.ts
View File

@ -1,32 +1,37 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine"; import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine";
import { guessContentType } from "../Utility";
export default class Reddit implements ScrapeEngine { export default class Reddit implements ScrapeEngine {
name = "reddit"; name = "reddit";
canImport(url: Location): boolean { canImport(url: Location): boolean {
return url.host.endsWith("reddit.com"); return url.host.endsWith("reddit.com");
}
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
let post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url
// For old.reddit.com
const oldLinkElements = document.querySelectorAll("a.title");
if (oldLinkElements.length > 0) {
post.contentUrl = (oldLinkElements[0] as HTMLAnchorElement).href;
} }
scrapeDocument(document: Document): ScrapeResult { // For new reddit
let result = new ScrapeResult(this.name); const newLinkElements = document.querySelectorAll("div[data-test-id='post-content'] > div > a");
let post = new ScrapedPost(); if (newLinkElements.length > 0) {
post.pageUrl = document.location.href; post.contentUrl = (newLinkElements[0] as HTMLAnchorElement).href;
// Set image url
// For old.reddit.com
const oldLinkElements = document.querySelectorAll("a.title");
if (oldLinkElements.length > 0) {
post.imageUrl = (oldLinkElements[0] as HTMLAnchorElement).href;
}
// For new reddit
const newLinkElements = document.querySelectorAll("div[data-test-id='post-content'] > div > a");
if (newLinkElements.length > 0) {
post.imageUrl = (newLinkElements[0] as HTMLAnchorElement).href;
}
result.tryAddPost(post);
return result;
} }
if (post.contentUrl != undefined) {
post.contentType = guessContentType(post.contentUrl);
}
result.tryAddPost(post);
return result;
}
} }

9
src/engines/SankakuComplex.ts
View File

@ -1,5 +1,6 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes"; import { TagCategory } from "../BooruTypes";
import { guessContentType } from "../Utility";
export default class SankakuComplex implements ScrapeEngine { export default class SankakuComplex implements ScrapeEngine {
name = "sankakucomplex"; name = "sankakucomplex";
@ -17,9 +18,15 @@ export default class SankakuComplex implements ScrapeEngine {
const originalImageElement = document.getElementById("highres") as HTMLAnchorElement; const originalImageElement = document.getElementById("highres") as HTMLAnchorElement;
if (originalImageElement) { if (originalImageElement) {
post.imageUrl = originalImageElement.href; post.contentUrl = originalImageElement.href;
} else {
// No point in continuing when we don't have an image.
return result;
} }
// Set content type
post.contentType = guessContentType(post.contentUrl);
// Set rating // Set rating
const safetyExp = new RegExp("Rating: (.*)"); const safetyExp = new RegExp("Rating: (.*)");
const safetyElements = Array.from(document.querySelectorAll("#stats > ul > li")) const safetyElements = Array.from(document.querySelectorAll("#stats > ul > li"))

12
src/engines/Shimmie2.ts
View File

@ -1,4 +1,5 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { guessContentType } from "../Utility";
export default class Shimmie2 implements ScrapeEngine { export default class Shimmie2 implements ScrapeEngine {
name = "shimmie2"; name = "shimmie2";
@ -18,14 +19,21 @@ export default class Shimmie2 implements ScrapeEngine {
.filter((x) => x.innerText == "Image Only" || x.innerText == "File Only"); .filter((x) => x.innerText == "Image Only" || x.innerText == "File Only");
if (originalImageElements.length > 0) { if (originalImageElements.length > 0) {
post.imageUrl = originalImageElements[0].href; post.contentUrl = originalImageElements[0].href;
} else { } else {
const dowloadBtnEl = document.querySelector("a[download]") as HTMLAnchorElement; const dowloadBtnEl = document.querySelector("a[download]") as HTMLAnchorElement;
if (dowloadBtnEl) { if (dowloadBtnEl) {
post.imageUrl = dowloadBtnEl.href; post.contentUrl = dowloadBtnEl.href;
} }
} }
if (post.contentUrl == undefined) {
return result;
}
// Set content type
post.contentType = guessContentType(post.contentUrl);
// Set safety // Set safety
post.rating = "unsafe"; // Usually it is unsafe post.rating = "unsafe"; // Usually it is unsafe

117
src/engines/Shuushuu.ts
View File

@ -1,65 +1,74 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes"; import { TagCategory } from "../BooruTypes";
import { guessContentType } from "../Utility";
export default class Shuushuu implements ScrapeEngine { export default class Shuushuu implements ScrapeEngine {
name = "shuushuu"; name = "shuushuu";
canImport(url: Location): boolean { canImport(url: Location): boolean {
return url.host == "e-shuushuu.net"; return url.host == "e-shuushuu.net";
}
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
let post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url
const imgElement = document.querySelector(".thumb_image") as HTMLAnchorElement;
if (imgElement) {
post.contentUrl = imgElement.href;
} else {
// No point in continuing when we don't have an image.
return result;
} }
scrapeDocument(document: Document): ScrapeResult { // Set content type
let result = new ScrapeResult(this.name); post.contentType = guessContentType(post.contentUrl);
let post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url // Set rating
const imgElement = document.querySelector(".thumb_image") as HTMLAnchorElement; post.rating = "safe";
if (imgElement) {
post.imageUrl = imgElement.href; // Set tags
const exp = new RegExp(/(.*)_(\w+)/);
const tagElements = document.getElementsByClassName("quicktag");
for (const tagElement of tagElements) {
// id = quicktag1_1009628
// match[0] = quicktag1_1009628 (aka input str)
// match[1] = quicktag1
// match[2] = 1009628
const match = exp.exec(tagElement.id);
if (!match || match.length != 3) {
continue;
}
let tagCategory: TagCategory | undefined;
switch (match[1]) {
// Source, aka copyright
case "quicktag2":
tagCategory = "copyright";
break;
// Artist
case "quicktag3":
tagCategory = "artist";
break;
// Characters
case "quicktag4":
tagCategory = "character";
break;
}
for (const el of tagElement.children) {
const tagName = (el?.firstElementChild as HTMLAnchorElement)?.innerText;
if (tagName) {
post.tags.push(new ScrapedTag(tagName, tagCategory));
} }
}
// Set rating
post.rating = "safe";
// Set tags
const exp = new RegExp(/(.*)_(\w+)/);
const tagElements = document.getElementsByClassName("quicktag");
for (const tagElement of tagElements) {
// id = quicktag1_1009628
// match[0] = quicktag1_1009628 (aka input str)
// match[1] = quicktag1
// match[2] = 1009628
const match = exp.exec(tagElement.id);
if (!match || match.length != 3) { continue; }
let tagCategory: TagCategory | undefined;
switch (match[1]) {
// Source, aka copyright
case "quicktag2":
tagCategory = "copyright";
break;
// Artist
case "quicktag3":
tagCategory = "artist";
break;
// Characters
case "quicktag4":
tagCategory = "character";
break;
}
for (const el of tagElement.children) {
const tagName = (el?.firstElementChild as HTMLAnchorElement)?.innerText;
if (tagName) {
post.tags.push(new ScrapedTag(tagName, tagCategory));
}
}
}
result.tryAddPost(post);
return result;
} }
result.tryAddPost(post);
return result;
}
} }

66
src/engines/Twitter.ts
View File

@ -1,40 +1,48 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine"; import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine";
import { guessContentType } from "../Utility";
export function getOriginalImageUrl(imageUrl: string) { export function getOriginalImageUrl(imageUrl: string) {
const x = imageUrl.replace(/:large|:med|:small|:orig$/, ""); // Remove `:large`, `:small` etc from url const x = imageUrl.replace(/:large|:med|:small|:orig$/, ""); // Remove `:large`, `:small` etc from url
return x + ":orig"; // Add :orig to url (to get original image) return x + ":orig"; // Add :orig to url (to get original image)
} }
export default class Twitter implements ScrapeEngine { export default class Twitter implements ScrapeEngine {
name = "twitter"; name = "twitter";
canImport(url: Location): boolean { canImport(url: Location): boolean {
return url.host == "twitter.com"; return url.host == "twitter.com";
}
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
let post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url
// When clicked on post (not fullscreen, with post text etc)
const overlayElement = document.querySelector("#permalink-overlay-dialog");
if (overlayElement) {
const cardImageElement = overlayElement.querySelector("div[data-element-context='platform_photo_card'] > img");
if (cardImageElement) {
post.contentUrl = getOriginalImageUrl((cardImageElement as HTMLImageElement).src);
}
} }
scrapeDocument(document: Document): ScrapeResult { // When clicked on image (fullscreen)
let result = new ScrapeResult(this.name); const mediaImageElements = document.querySelectorAll("div.Gallery-media > img.media-image");
let post = new ScrapedPost(); if (mediaImageElements.length > 0) {
post.pageUrl = document.location.href; post.contentUrl = getOriginalImageUrl((mediaImageElements[0] as HTMLImageElement).src);
// Set image url
// When clicked on post (not fullscreen, with post text etc)
const overlayElement = document.querySelector("#permalink-overlay-dialog");
if (overlayElement) {
const cardImageElement = overlayElement.querySelector("div[data-element-context='platform_photo_card'] > img");
if (cardImageElement) {
post.imageUrl = getOriginalImageUrl((cardImageElement as HTMLImageElement).src);
}
}
// When clicked on image (fullscreen)
const mediaImageElements = document.querySelectorAll("div.Gallery-media > img.media-image");
if (mediaImageElements.length > 0) {
post.imageUrl = getOriginalImageUrl((mediaImageElements[0] as HTMLImageElement).src);
}
result.tryAddPost(post);
return result;
} }
if (post.contentUrl == undefined) {
return result;
}
// Set content type
post.contentType = guessContentType(post.contentUrl);
result.tryAddPost(post);
return result;
}
} }

13
src/engines/Zerochan.ts
View File

@ -1,5 +1,6 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes"; import { TagCategory } from "../BooruTypes";
import { guessContentType } from "../Utility";
export default class Zerochan implements ScrapeEngine { export default class Zerochan implements ScrapeEngine {
name = "zerochan"; name = "zerochan";
@ -17,15 +18,23 @@ export default class Zerochan implements ScrapeEngine {
const previewElements = document.getElementsByClassName("preview"); const previewElements = document.getElementsByClassName("preview");
if (previewElements.length > 0) { if (previewElements.length > 0) {
// The preview element only exists when there is a larger size available. Preview's img element contains a downscaled version. // The preview element only exists when there is a larger size available. Preview's img element contains a downscaled version.
post.imageUrl = (previewElements[0] as HTMLAnchorElement).href; post.contentUrl = (previewElements[0] as HTMLAnchorElement).href;
} else { } else {
// When there is no larger version available just use the displayed version. // When there is no larger version available just use the displayed version.
const imgElement = document.querySelector("img[title='No larger size available']") as HTMLImageElement; const imgElement = document.querySelector("img[title='No larger size available']") as HTMLImageElement;
if (imgElement) { if (imgElement) {
post.imageUrl = imgElement.src; post.contentUrl = imgElement.src;
} }
} }
if (post.contentUrl == undefined) {
// No point in continuing when we don't have an image.
return result;
}
// Set content type
post.contentType = guessContentType(post.contentUrl);
// Set rating // Set rating
post.rating = "safe"; post.rating = "safe";

11
src/engines/e621.ts
View File

@ -16,7 +16,16 @@ export default class e621 implements ScrapeEngine {
// Set image url // Set image url
const downloadEl = document.querySelector("#image-download-link > a") as HTMLAnchorElement; const downloadEl = document.querySelector("#image-download-link > a") as HTMLAnchorElement;
post.imageUrl = downloadEl?.href; post.contentUrl = downloadEl?.href;
if (post.contentUrl == undefined) {
return result;
}
// Set content type
if (document.querySelector("video#image") != null) {
post.contentType = "video";
}
// Set rating // Set rating
const ratingEl = document.getElementById("post-rating-text") as HTMLSpanElement; const ratingEl = document.getElementById("post-rating-text") as HTMLSpanElement;

4
tsconfig.json
View File

@ -1,7 +1,7 @@
{ {
"compilerOptions": { "compilerOptions": {
"lib": [ "lib": [
"es6", "es2019",
"dom", "dom",
"dom.iterable" "dom.iterable"
], ],
@ -19,4 +19,4 @@
"include": [ "include": [
"src/index.ts" "src/index.ts"
] ]
} }