diff --git a/package.json b/package.json index d98cd17..23fd76c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "neo-scraper", - "version": "0.3.1", + "version": "0.4.0", "description": "Scrape a booru-like post object from the DOM.", "main": "dist/index.js", "types": "dist/index.d.js", diff --git a/src/NeoScraper.test.ts b/src/NeoScraper.test.ts index cd93df8..5e8c420 100644 --- a/src/NeoScraper.test.ts +++ b/src/NeoScraper.test.ts @@ -2,132 +2,142 @@ import { JSDOM } from "jsdom"; import NeoScraper from "./NeoScraper"; async function scrapeUrl(url: string) { - const scraper = new NeoScraper(); - const dom = await JSDOM.fromURL(url); - return scraper.scrapeDocument(dom.window.document); + const scraper = new NeoScraper(); + const dom = await JSDOM.fromURL(url); + return scraper.scrapeDocument(dom.window.document); } beforeAll(() => { - // Expose JSDOM Element constructor - // @ts-ignore - global.Element = (new JSDOM()).window.Element; - // 'Implement' innerText in JSDOM: https://github.com/jsdom/jsdom/issues/1245 - // @ts-ignore - Object.defineProperty(global.Element.prototype, 'innerText', { - get() { - return this.textContent.replace(/(\r\n|\n|\r|^\s+|\s+$)/gm, "");; - }, - }); + // Expose JSDOM Element constructor + // @ts-ignore + global.Element = new JSDOM().window.Element; + // 'Implement' innerText in JSDOM: https://github.com/jsdom/jsdom/issues/1245 + // @ts-ignore + Object.defineProperty(global.Element.prototype, "innerText", { + get() { + return this.textContent.replace(/(\r\n|\n|\r|^\s+|\s+$)/gm, ""); + }, + }); }); test("create scraper with default engines", async () => { - const scraper = new NeoScraper(); + const scraper = new NeoScraper(); }); test("all default scrapers have unique names", () => { - const scraper = new NeoScraper(); - let alreadySeen: string[] = []; + const scraper = new NeoScraper(); + let alreadySeen: string[] = []; - for (const engine of scraper.engines) { - if (alreadySeen.indexOf(engine.name) != -1) { - fail(engine.name + " is not a unique name!"); - } else { - alreadySeen.push(engine.name); - } + for (const engine of scraper.engines) { + if (alreadySeen.indexOf(engine.name) != -1) { + fail(engine.name + " is not a unique name!"); + } else { + alreadySeen.push(engine.name); } + } }); // Danbooru // All links are SFW unless stated otherwise test("get danbooru.donmai.us post 3555872", async () => { - var pageUrl = "https://danbooru.donmai.us/posts/3555872"; - const res = await scrapeUrl(pageUrl); - expect(res.posts.length).toBe(1); - expect(res.posts[0].pageUrl).toBe(pageUrl); - expect(res.posts[0].imageUrl).toBe("https://danbooru.donmai.us/data/__flint_girls_und_panzer_drawn_by_omachi_slabco__90332299216db81b8793084e4cb7c15d.jpg"); + var pageUrl = "https://danbooru.donmai.us/posts/3555872"; + const res = await scrapeUrl(pageUrl); + expect(res.posts.length).toBe(1); + expect(res.posts[0].pageUrl).toBe(pageUrl); + expect(res.posts[0].contentUrl).toBe( + "https://danbooru.donmai.us/data/__flint_girls_und_panzer_drawn_by_omachi_slabco__90332299216db81b8793084e4cb7c15d.jpg" + ); }); // Gelbooru test("get safebooru.org post 2871532", async () => { - var pageUrl = "https://safebooru.org/index.php?page=post&s=view&id=2871532"; - const res = await scrapeUrl(pageUrl); - expect(res.posts.length).toBe(1); - expect(res.posts[0].pageUrl).toBe(pageUrl); - expect(res.posts[0].imageUrl).toBe("https://safebooru.org//images/2757/5f19c7744db918c3d4c4c6035bc65dfa9e18866f.jpg"); + var pageUrl = "https://safebooru.org/index.php?page=post&s=view&id=2871532"; + const res = await scrapeUrl(pageUrl); + expect(res.posts.length).toBe(1); + expect(res.posts[0].pageUrl).toBe(pageUrl); + expect(res.posts[0].contentUrl).toBe( + "https://safebooru.org//images/2757/5f19c7744db918c3d4c4c6035bc65dfa9e18866f.jpg" + ); }); test("get gelbooru.com post 4925055", async () => { - var pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=4925055&tags=kimetsu_no_yaiba"; - const res = await scrapeUrl(pageUrl); - expect(res.posts.length).toBe(1); - expect(res.posts[0].pageUrl).toBe(pageUrl); - expect(res.posts[0].imageUrl).toBe("https://img2.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg"); + var pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=4925055&tags=kimetsu_no_yaiba"; + const res = await scrapeUrl(pageUrl); + expect(res.posts.length).toBe(1); + expect(res.posts[0].pageUrl).toBe(pageUrl); + expect(res.posts[0].contentUrl).toBe("https://img2.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg"); }); test("get rule34.xxx post 2740994", async () => { - var pageUrl = "https://rule34.xxx/index.php?page=post&s=view&id=2740994"; - const res = await scrapeUrl(pageUrl); - expect(res.posts.length).toBe(1); - expect(res.posts[0].pageUrl).toBe(pageUrl); - expect(res.posts[0].imageUrl).toContain(".rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg"); + var pageUrl = "https://rule34.xxx/index.php?page=post&s=view&id=2740994"; + const res = await scrapeUrl(pageUrl); + expect(res.posts.length).toBe(1); + expect(res.posts[0].pageUrl).toBe(pageUrl); + expect(res.posts[0].contentUrl).toContain(".rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg"); }); // Moebooru test("get yande.re post 438510", async () => { - var pageUrl = "https://yande.re/post/show/438510"; - const res = await scrapeUrl(pageUrl); - expect(res.posts.length).toBe(1); - expect(res.posts[0].pageUrl).toBe(pageUrl); - expect(res.posts[0].imageUrl).toBe("https://files.yande.re/image/49b79bc372716a579d8525dcaccb56e2/yande.re%20438510%20landscape%20link%20nintendo%20princess_zelda%20sword%20the_legend_of_zelda%20the_legend_of_zelda%3A_breath_of_the_wild.jpg"); + var pageUrl = "https://yande.re/post/show/438510"; + const res = await scrapeUrl(pageUrl); + expect(res.posts.length).toBe(1); + expect(res.posts[0].pageUrl).toBe(pageUrl); + expect(res.posts[0].contentUrl).toBe( + "https://files.yande.re/image/49b79bc372716a579d8525dcaccb56e2/yande.re%20438510%20landscape%20link%20nintendo%20princess_zelda%20sword%20the_legend_of_zelda%20the_legend_of_zelda%3A_breath_of_the_wild.jpg" + ); }); test("get konachan.com post 289968", async () => { - var pageUrl = "https://konachan.com/post/show/289968/ahri_-league_of_legends-akali-animal_ears-blonde_h"; - const res = await scrapeUrl(pageUrl); - expect(res.posts.length).toBe(1); - expect(res.posts[0].pageUrl).toBe(pageUrl); - expect(res.posts[0].imageUrl).toBe("https://konachan.com/image/e40b38fb704434e4da3eae41e26bca39/Konachan.com%20-%20289968%20akali%20blue_eyes%20breasts%20choker%20cleavage%20evelynn%20foxgirl%20gloves%20group%20hat%20kai%27sa%20long_hair%20microphone%20ponytail%20saphirya%20short_hair%20signed%20watermark.jpg"); + var pageUrl = "https://konachan.com/post/show/289968/ahri_-league_of_legends-akali-animal_ears-blonde_h"; + const res = await scrapeUrl(pageUrl); + expect(res.posts.length).toBe(1); + expect(res.posts[0].pageUrl).toBe(pageUrl); + expect(res.posts[0].contentUrl).toBe( + "https://konachan.com/image/e40b38fb704434e4da3eae41e26bca39/Konachan.com%20-%20289968%20akali%20blue_eyes%20breasts%20choker%20cleavage%20evelynn%20foxgirl%20gloves%20group%20hat%20kai%27sa%20long_hair%20microphone%20ponytail%20saphirya%20short_hair%20signed%20watermark.jpg" + ); }); // Reddit test("get old.reddit.com post", async () => { - var pageUrl = "https://old.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/"; - const res = await scrapeUrl(pageUrl); - expect(res.posts.length).toBe(1); - expect(res.posts[0].pageUrl).toBe(pageUrl); - expect(res.posts[0].imageUrl).toBe("https://i.redd.it/ln8ahz17kpa31.jpg"); + var pageUrl = "https://old.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/"; + const res = await scrapeUrl(pageUrl); + expect(res.posts.length).toBe(1); + expect(res.posts[0].pageUrl).toBe(pageUrl); + expect(res.posts[0].contentUrl).toBe("https://i.redd.it/ln8ahz17kpa31.jpg"); }); test("get new.reddit.com post", async () => { - var pageUrl = "https://new.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/"; - const res = await scrapeUrl(pageUrl); - expect(res.posts.length).toBe(1); - expect(res.posts[0].pageUrl).toBe(pageUrl); - expect(res.posts[0].imageUrl).toBe("https://i.redd.it/ln8ahz17kpa31.jpg"); + var pageUrl = "https://new.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/"; + const res = await scrapeUrl(pageUrl); + expect(res.posts.length).toBe(1); + expect(res.posts[0].pageUrl).toBe(pageUrl); + expect(res.posts[0].contentUrl).toBe("https://i.redd.it/ln8ahz17kpa31.jpg"); }); // Shimmie2 test("get rule34.paheal.net post 2505982", async () => { - var pageUrl = "https://rule34.paheal.net/post/view/2505982"; // Somewhat sketchy - const res = await scrapeUrl(pageUrl); - expect(res.posts.length).toBe(1); - expect(res.posts[0].pageUrl).toBe(pageUrl); - expect(res.posts[0].imageUrl).toContain(".paheal.net/_images/15427ab5ed2cddbeef947e30f5c9cab6/2505982%20-%20Kai%27Sa%20League_of_Legends%20lawzilla.png"); + var pageUrl = "https://rule34.paheal.net/post/view/2505982"; // Somewhat sketchy + const res = await scrapeUrl(pageUrl); + expect(res.posts.length).toBe(1); + expect(res.posts[0].pageUrl).toBe(pageUrl); + expect(res.posts[0].contentUrl).toContain( + ".paheal.net/_images/15427ab5ed2cddbeef947e30f5c9cab6/2505982%20-%20Kai%27Sa%20League_of_Legends%20lawzilla.png" + ); }); // Twitter test("get twitter.com post", async () => { - pending("JSDOM can't parse this page"); - // TODO: JSDOM sucks - // var pageUrl = "https://twitter.com/Dishwasher1910/status/1171162651461746688/photo/1"; - // const res = await scrapeUrl(pageUrl); - // expect(res.posts.length).toBe(1); - // expect(res.posts[0].pageUrl).toBe(pageUrl); - // expect(res.posts[0].imageUrl).toBe("https://pbs.twimg.com/media/EEDOOCdU0AAHX00?format=jpg&name=orig"); + pending("JSDOM can't parse this page"); + // TODO: JSDOM sucks + // var pageUrl = "https://twitter.com/Dishwasher1910/status/1171162651461746688/photo/1"; + // const res = await scrapeUrl(pageUrl); + // expect(res.posts.length).toBe(1); + // expect(res.posts[0].pageUrl).toBe(pageUrl); + // expect(res.posts[0].contentUrl).toBe("https://pbs.twimg.com/media/EEDOOCdU0AAHX00?format=jpg&name=orig"); }); diff --git a/src/NeoScraper.ts b/src/NeoScraper.ts index 5aa396a..ebc5f8b 100644 --- a/src/NeoScraper.ts +++ b/src/NeoScraper.ts @@ -1,51 +1,56 @@ -import { ScrapeEngine, ScrapeResults, } from './ScrapeEngine'; +import { ScrapeEngine, ScrapeResults } from "./ScrapeEngine"; import * as engines from "./engines"; export default class NeoScraper { - engines: ScrapeEngine[] = []; - fallbackEngine: ScrapeEngine | null = null; + engines: ScrapeEngine[] = []; + fallbackEngine: ScrapeEngine | null = null; - constructor(useDefaultEngines: boolean = true) { - if (useDefaultEngines) { - this.engines.push( - new engines.AnimePictures(), - new engines.Danbooru(), - new engines.e621(), - new engines.Gelbooru(), - new engines.Moebooru(), - new engines.Pixiv(), - new engines.Reddit(), - new engines.SankakuComplex(), - new engines.Shimmie2(), - new engines.Shuushuu(), - new engines.Twitter(), - new engines.Zerochan(), - ); - this.fallbackEngine = new engines.Fallback(); - } + constructor(useDefaultEngines: boolean = true) { + if (useDefaultEngines) { + this.engines.push( + new engines.AnimePictures(), + new engines.Danbooru(), + new engines.e621(), + new engines.Gelbooru(), + new engines.Moebooru(), + new engines.Pixiv(), + new engines.Reddit(), + new engines.SankakuComplex(), + new engines.Shimmie2(), + new engines.Shuushuu(), + new engines.Twitter(), + new engines.Zerochan() + ); + this.fallbackEngine = new engines.Fallback(); + } + } + + scrapeDocument( + document: Document, + engines: string[] | null = null, + allowFallbackEngine: boolean = false, + ignoreCanImport: boolean = false + ): ScrapeResults { + const res = new ScrapeResults(); + + for (const engine of this.engines) { + if (ignoreCanImport || engine.canImport(document.location)) { + console.log("Using engine: " + engine.name); + res.results.push(engine.scrapeDocument(document)); + } } - scrapeDocument(document: Document, engines: string[] | null = null, allowFallbackEngine: boolean = false, ignoreCanImport: boolean = false): ScrapeResults { - const res = new ScrapeResults(); - - for (const engine of this.engines) { - if (ignoreCanImport || engine.canImport(document.location)) { - console.log("Using engine: " + engine.name); - res.results.push(engine.scrapeDocument(document)); - } - } - - if (allowFallbackEngine) { - if (!this.fallbackEngine) { - console.error("NeoScraper.fallbackEngine is unset!"); - } else { - console.log("Using fallback engine"); - res.results.push(this.fallbackEngine.scrapeDocument(document)); - } - } - - console.dir(res); - - return res; + if (allowFallbackEngine) { + if (!this.fallbackEngine) { + console.error("NeoScraper.fallbackEngine is unset!"); + } else { + console.log("Using fallback engine"); + res.results.push(this.fallbackEngine.scrapeDocument(document)); + } } + + console.dir(res); + + return res; + } } diff --git a/src/ScrapeEngine.ts b/src/ScrapeEngine.ts index 96b26c2..c6ed1d1 100644 --- a/src/ScrapeEngine.ts +++ b/src/ScrapeEngine.ts @@ -1,11 +1,15 @@ import { TagCategory, SafetyRating } from "./BooruTypes"; +export type ContentType = "image" | "video"; + export class ScrapedPost { - imageUrl: string = ""; + contentUrl: string = ""; pageUrl: string = ""; + contentType: ContentType = "image"; rating: SafetyRating = "safe"; tags: ScrapedTag[] = []; source: string | undefined = undefined; + referrer: string | undefined = undefined; } export class ScrapedTag { @@ -28,10 +32,8 @@ export class ScrapeResult { } tryAddPost(post: ScrapedPost) { - if (!post.imageUrl) { - console.log( - `[${this.engine}] Not adding post because imageUrl is unset!` - ); + if (!post.contentUrl) { + console.log(`[${this.engine}] Not adding post because imageUrl is unset!`); } else { this.posts.push(post); } diff --git a/src/Utility.test.ts b/src/Utility.test.ts new file mode 100644 index 0000000..56b0465 --- /dev/null +++ b/src/Utility.test.ts @@ -0,0 +1,21 @@ +import { guessContentType } from "./Utility"; + +test("guessContentType", () => { + expect(guessContentType("video.mp4")).toBe("video"); + expect(guessContentType("https://example.com/video.mp4")).toBe("video"); + expect(guessContentType("https://example.com/video.mp4?q=r")).toBe("video"); + expect(guessContentType("https://example.com/video.mp4?q=r&rrr=fss")).toBe("video"); + expect(guessContentType("https://example.com/video.mp4?q=r&rrr=fss,ef./s.e")).toBe("video"); + expect(guessContentType("file://example.com/video.mp4?q=r&rrr=fss,ef./s.e")).toBe("video"); + expect(guessContentType("example.com/video.mp4?q=r&rrr=fss,ef./s.e")).toBe("video"); + expect(guessContentType(".com/video.mp4?q=r&rrr=fss,ef./s.e")).toBe("video"); + expect(guessContentType("c:\\my\\video.mp4?q=r&rrr=fss,ef./s.e")).toBe("video"); + expect(guessContentType("")).toBe("image"); + expect(guessContentType("08435n gf g =0324")).toBe("image"); + expect(guessContentType("https://example.com/.mp4/image.png")).toBe("image"); + expect(guessContentType("https://example.com/file.mp4.png")).toBe("image"); + expect(guessContentType("https://example.com/.mp4.png")).toBe("image"); + expect(guessContentType(undefined as any)).toBe("image"); + expect(guessContentType(null as any)).toBe("image"); + expect(() => guessContentType(984 as any)).toThrow(); +}); diff --git a/src/Utility.ts b/src/Utility.ts new file mode 100644 index 0000000..cf7595d --- /dev/null +++ b/src/Utility.ts @@ -0,0 +1,50 @@ +import { ContentType } from "./ScrapeEngine"; + +// https://github.com/sindresorhus/video-extensions/blob/main/video-extensions.json +export const videoExtensions = [ + "3g2", + "3gp", + "aaf", + "asf", + "avchd", + "avi", + "drc", + "flv", + "m2v", + "m4p", + "m4v", + "mkv", + "mng", + "mov", + "mp2", + "mp4", + "mpe", + "mpeg", + "mpg", + "mpv", + "mxf", + "nsv", + "ogg", + "ogv", + "qt", + "rm", + "rmvb", + "roq", + "svi", + "vob", + "webm", + "wmv", + "yuv", +]; + +export function getUrlExtension(url: string) { + return url.split(/[#?]/)[0].split(".").pop()!.trim(); +} + +export function guessContentType(url: string): ContentType { + if (url != undefined) { + const ext = getUrlExtension(url); + if (videoExtensions.indexOf(ext) != -1) return "video"; + } + return "image"; +} diff --git a/src/engines/AnimePictures.ts b/src/engines/AnimePictures.ts index 76a6305..9274e08 100644 --- a/src/engines/AnimePictures.ts +++ b/src/engines/AnimePictures.ts @@ -1,56 +1,63 @@ import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; import { TagCategory } from "../BooruTypes"; +import { guessContentType } from "../Utility"; export default class AnimePictures implements ScrapeEngine { - name = "animepictures"; + name = "animepictures"; - canImport(url: Location): boolean { - return url.host == "anime-pictures.net"; + canImport(url: Location): boolean { + return url.host == "anime-pictures.net"; + } + + scrapeDocument(document: Document): ScrapeResult { + let result = new ScrapeResult(this.name); + let post = new ScrapedPost(); + post.source = document.location.href; + + // Set image url + const originalImageElements = Array.from(document.getElementsByClassName("download_icon")); + + if (originalImageElements.length > 0) { + post.contentUrl = (originalImageElements[0] as HTMLAnchorElement).href; + } else { + // No point in continuing when we don't have an image. + return result; } - scrapeDocument(document: Document): ScrapeResult { - let result = new ScrapeResult(this.name); - let post = new ScrapedPost(); - post.source = document.location.href; + // Set content type + post.contentType = guessContentType(post.contentUrl); - // Set image url - const originalImageElements = Array.from(document.getElementsByClassName("download_icon")); + // Set rating + post.rating = "safe"; // Probably - if (originalImageElements.length > 0) { - post.imageUrl = (originalImageElements[0] as HTMLAnchorElement).href; + // Set tags + const tagElements = Array.from(document.querySelectorAll("ul.tags > li > a")).map((x) => x as HTMLAnchorElement); + for (const tagElement of tagElements) { + let tagName = tagElement.innerText; + + let category: TagCategory | undefined; + for (const className of tagElement.classList) { + switch (className) { + case "copyright": + category = "copyright"; + break; + case "character": + category = "character"; + break; + case "artist": + category = "artist"; + break; } + } - // Set rating - post.rating = "safe"; // Probably - - // Set tags - const tagElements = Array.from(document.querySelectorAll("ul.tags > li > a")).map(x => x as HTMLAnchorElement); - for (const tagElement of tagElements) { - let tagName = tagElement.innerText; - - let category: TagCategory | undefined; - for (const className of tagElement.classList) { - switch (className) { - case "copyright": - category = "copyright"; - break; - case "character": - category = "character"; - break; - case "artist": - category = "artist"; - break; - } - } - - if (tagName) { - let tag = new ScrapedTag(tagName, category); - post.tags.push(tag); - } - } - - result.tryAddPost(post); - - return result; + if (tagName) { + let tag = new ScrapedTag(tagName, category); + post.tags.push(tag); + } } + + result.tryAddPost(post); + + return result; + } } diff --git a/src/engines/Danbooru.ts b/src/engines/Danbooru.ts index df78faf..7b8d995 100644 --- a/src/engines/Danbooru.ts +++ b/src/engines/Danbooru.ts @@ -1,5 +1,6 @@ import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; import { TagCategory } from "../BooruTypes"; +import { guessContentType } from "../Utility"; export default class Danbooru implements ScrapeEngine { name = "danbooru"; @@ -19,7 +20,17 @@ export default class Danbooru implements ScrapeEngine { ); if (originalImageElements.length > 0) { - post.imageUrl = (originalImageElements[0] as HTMLAnchorElement).href; + post.contentUrl = (originalImageElements[0] as HTMLAnchorElement).href; + } else { + // No point in continuing when we don't have an image. + return result; + } + + // Set content type + if (document.querySelector(".image-container > video") != null) { + post.contentType = "video"; + } else { + post.contentType = guessContentType(post.contentUrl); } // Set rating diff --git a/src/engines/Fallback.ts b/src/engines/Fallback.ts index fad8964..7edb5ea 100644 --- a/src/engines/Fallback.ts +++ b/src/engines/Fallback.ts @@ -1,4 +1,12 @@ -import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine"; +import { ScrapeEngine, ScrapeResult, ScrapedPost, ContentType } from "../ScrapeEngine"; + +class MediaElement { + constructor( + public readonly contentType: ContentType, + public readonly contentUrl: string, + public readonly size: number + ) {} +} export default class Fallback implements ScrapeEngine { name = "fallback"; @@ -8,21 +16,30 @@ export default class Fallback implements ScrapeEngine { } scrapeDocument(document: Document): ScrapeResult { - function getSize(el: HTMLImageElement) { - return el.width * el.height; - } + // Get all img elements. + const largestImgElement = Array.from(document.getElementsByTagName("img")).map( + (x) => new MediaElement("image", x.src, x.width * x.height) + ); - // Get largest img element on this page - const largestImgElement = Array.from(document.getElementsByTagName("img")).reduce((a, b) => - getSize(a) > getSize(b) ? a : b + // Get all video elements that have at least one source (child node). + const largestVideoElement = Array.from(document.getElementsByTagName("video")).flatMap((x) => { + const source = x.querySelector("source")?.src; + if (source != undefined) return new MediaElement("video", source, x.videoWidth * x.videoHeight); + return []; // Skip this item because it does not have a source. + }); + + // Get the largest media element. + const largestMediaElement = [...largestImgElement, ...largestVideoElement].reduce((a, b) => + a?.size > b?.size ? a : b ); let result = new ScrapeResult(this.name); - if (largestImgElement) { + if (largestMediaElement) { let post = new ScrapedPost(); post.pageUrl = document.location.href; - post.imageUrl = largestImgElement.src; + post.contentUrl = largestMediaElement.contentUrl; + post.contentType = largestMediaElement.contentType; result.posts.push(post); } diff --git a/src/engines/Gelbooru.ts b/src/engines/Gelbooru.ts index 00dd734..0b6d0d8 100644 --- a/src/engines/Gelbooru.ts +++ b/src/engines/Gelbooru.ts @@ -1,5 +1,6 @@ import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; import { TagCategory } from "../BooruTypes"; +import { guessContentType } from "../Utility"; // Small hack to avoid screwing with semver enum Version { @@ -44,7 +45,18 @@ export default class Gelbooru implements ScrapeEngine { Array.from(document.querySelectorAll("li > a")) .map((x) => x as HTMLAnchorElement) .filter((x) => x && x.innerText == "Original image") - .map((x) => (post.imageUrl = x.href)); + .map((x) => (post.contentUrl = x.href)); + + if (post.contentUrl == undefined) { + return result; + } + + // Set content type + if (document.getElementById("gelcomVideoPlayer") != null) { + post.contentType = "video"; + } else { + post.contentType = guessContentType(post.contentUrl); + } // Set rating and source const regex = new RegExp("(.*): (.*)"); diff --git a/src/engines/Moebooru.ts b/src/engines/Moebooru.ts index e4d7aeb..ce166aa 100644 --- a/src/engines/Moebooru.ts +++ b/src/engines/Moebooru.ts @@ -1,5 +1,6 @@ import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; import { TagCategory } from "../BooruTypes"; +import { guessContentType } from "../Utility"; export default class Moebooru implements ScrapeEngine { name = "moebooru"; @@ -16,9 +17,15 @@ export default class Moebooru implements ScrapeEngine { // Set image url const originalImageElement = document.querySelector("#highres") as HTMLAnchorElement; if (originalImageElement) { - post.imageUrl = originalImageElement.href; + post.contentUrl = originalImageElement.href; + } else { + // No point in continuing when we don't have an image. + return result; } + // Set content type + post.contentType = guessContentType(post.contentUrl); + // Set rating and source // Same method as Gelbooru v025. const regex = new RegExp("(.*): (.*)"); diff --git a/src/engines/Pixiv.ts b/src/engines/Pixiv.ts index 5c42a30..12f4d42 100644 --- a/src/engines/Pixiv.ts +++ b/src/engines/Pixiv.ts @@ -1,28 +1,30 @@ import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; import { TagCategory } from "../BooruTypes"; +import { guessContentType } from "../Utility"; export default class Pixiv implements ScrapeEngine { - name = "pixiv"; + name = "pixiv"; - canImport(url: Location): boolean { - return ( - url.host == "pixiv.net" || - url.host == "www.pixiv.net" - ); + canImport(url: Location): boolean { + return url.host == "pixiv.net" || url.host == "www.pixiv.net"; + } + + scrapeDocument(document: Document): ScrapeResult { + let result = new ScrapeResult(this.name); + + const imgLinks = Array.from(document.querySelectorAll("div[role='presentation'] > a")).map( + (x) => x as HTMLAnchorElement + ); + + for (const imgLink of imgLinks) { + let post = new ScrapedPost(); + post.pageUrl = document.location.href; + post.contentUrl = imgLink.href; + post.contentType = guessContentType(post.contentUrl); + post.referrer = document.location.origin; + result.tryAddPost(post); } - scrapeDocument(document: Document): ScrapeResult { - let result = new ScrapeResult(this.name); - - const imgLinks = Array.from(document.querySelectorAll("div[role='presentation'] > a")).map(x => x as HTMLAnchorElement); - - for (const imgLink of imgLinks) { - let post = new ScrapedPost(); - post.pageUrl = document.location.href; - post.imageUrl = imgLink.href; - result.tryAddPost(post); - } - - return result; - } + return result; + } } diff --git a/src/engines/Reddit.ts b/src/engines/Reddit.ts index 96c22ac..eef2a6e 100644 --- a/src/engines/Reddit.ts +++ b/src/engines/Reddit.ts @@ -1,32 +1,37 @@ import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine"; +import { guessContentType } from "../Utility"; export default class Reddit implements ScrapeEngine { - name = "reddit"; + name = "reddit"; - canImport(url: Location): boolean { - return url.host.endsWith("reddit.com"); + canImport(url: Location): boolean { + return url.host.endsWith("reddit.com"); + } + + scrapeDocument(document: Document): ScrapeResult { + let result = new ScrapeResult(this.name); + let post = new ScrapedPost(); + post.pageUrl = document.location.href; + + // Set image url + // For old.reddit.com + const oldLinkElements = document.querySelectorAll("a.title"); + if (oldLinkElements.length > 0) { + post.contentUrl = (oldLinkElements[0] as HTMLAnchorElement).href; } - scrapeDocument(document: Document): ScrapeResult { - let result = new ScrapeResult(this.name); - let post = new ScrapedPost(); - post.pageUrl = document.location.href; - - // Set image url - // For old.reddit.com - const oldLinkElements = document.querySelectorAll("a.title"); - if (oldLinkElements.length > 0) { - post.imageUrl = (oldLinkElements[0] as HTMLAnchorElement).href; - } - - // For new reddit - const newLinkElements = document.querySelectorAll("div[data-test-id='post-content'] > div > a"); - if (newLinkElements.length > 0) { - post.imageUrl = (newLinkElements[0] as HTMLAnchorElement).href; - } - - result.tryAddPost(post); - - return result; + // For new reddit + const newLinkElements = document.querySelectorAll("div[data-test-id='post-content'] > div > a"); + if (newLinkElements.length > 0) { + post.contentUrl = (newLinkElements[0] as HTMLAnchorElement).href; } + + if (post.contentUrl != undefined) { + post.contentType = guessContentType(post.contentUrl); + } + + result.tryAddPost(post); + + return result; + } } diff --git a/src/engines/SankakuComplex.ts b/src/engines/SankakuComplex.ts index 2d04eaf..9405b11 100644 --- a/src/engines/SankakuComplex.ts +++ b/src/engines/SankakuComplex.ts @@ -1,5 +1,6 @@ import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; import { TagCategory } from "../BooruTypes"; +import { guessContentType } from "../Utility"; export default class SankakuComplex implements ScrapeEngine { name = "sankakucomplex"; @@ -17,9 +18,15 @@ export default class SankakuComplex implements ScrapeEngine { const originalImageElement = document.getElementById("highres") as HTMLAnchorElement; if (originalImageElement) { - post.imageUrl = originalImageElement.href; + post.contentUrl = originalImageElement.href; + } else { + // No point in continuing when we don't have an image. + return result; } + // Set content type + post.contentType = guessContentType(post.contentUrl); + // Set rating const safetyExp = new RegExp("Rating: (.*)"); const safetyElements = Array.from(document.querySelectorAll("#stats > ul > li")) diff --git a/src/engines/Shimmie2.ts b/src/engines/Shimmie2.ts index 7343dad..8699472 100644 --- a/src/engines/Shimmie2.ts +++ b/src/engines/Shimmie2.ts @@ -1,4 +1,5 @@ import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; +import { guessContentType } from "../Utility"; export default class Shimmie2 implements ScrapeEngine { name = "shimmie2"; @@ -18,14 +19,21 @@ export default class Shimmie2 implements ScrapeEngine { .filter((x) => x.innerText == "Image Only" || x.innerText == "File Only"); if (originalImageElements.length > 0) { - post.imageUrl = originalImageElements[0].href; + post.contentUrl = originalImageElements[0].href; } else { const dowloadBtnEl = document.querySelector("a[download]") as HTMLAnchorElement; if (dowloadBtnEl) { - post.imageUrl = dowloadBtnEl.href; + post.contentUrl = dowloadBtnEl.href; } } + if (post.contentUrl == undefined) { + return result; + } + + // Set content type + post.contentType = guessContentType(post.contentUrl); + // Set safety post.rating = "unsafe"; // Usually it is unsafe diff --git a/src/engines/Shuushuu.ts b/src/engines/Shuushuu.ts index 6b81728..988c48d 100644 --- a/src/engines/Shuushuu.ts +++ b/src/engines/Shuushuu.ts @@ -1,65 +1,74 @@ import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; import { TagCategory } from "../BooruTypes"; +import { guessContentType } from "../Utility"; export default class Shuushuu implements ScrapeEngine { - name = "shuushuu"; + name = "shuushuu"; - canImport(url: Location): boolean { - return url.host == "e-shuushuu.net"; + canImport(url: Location): boolean { + return url.host == "e-shuushuu.net"; + } + + scrapeDocument(document: Document): ScrapeResult { + let result = new ScrapeResult(this.name); + let post = new ScrapedPost(); + post.pageUrl = document.location.href; + + // Set image url + const imgElement = document.querySelector(".thumb_image") as HTMLAnchorElement; + if (imgElement) { + post.contentUrl = imgElement.href; + } else { + // No point in continuing when we don't have an image. + return result; } - scrapeDocument(document: Document): ScrapeResult { - let result = new ScrapeResult(this.name); - let post = new ScrapedPost(); - post.pageUrl = document.location.href; + // Set content type + post.contentType = guessContentType(post.contentUrl); - // Set image url - const imgElement = document.querySelector(".thumb_image") as HTMLAnchorElement; - if (imgElement) { - post.imageUrl = imgElement.href; + // Set rating + post.rating = "safe"; + + // Set tags + const exp = new RegExp(/(.*)_(\w+)/); + const tagElements = document.getElementsByClassName("quicktag"); + + for (const tagElement of tagElements) { + // id = quicktag1_1009628 + // match[0] = quicktag1_1009628 (aka input str) + // match[1] = quicktag1 + // match[2] = 1009628 + const match = exp.exec(tagElement.id); + if (!match || match.length != 3) { + continue; + } + + let tagCategory: TagCategory | undefined; + switch (match[1]) { + // Source, aka copyright + case "quicktag2": + tagCategory = "copyright"; + break; + // Artist + case "quicktag3": + tagCategory = "artist"; + break; + // Characters + case "quicktag4": + tagCategory = "character"; + break; + } + + for (const el of tagElement.children) { + const tagName = (el?.firstElementChild as HTMLAnchorElement)?.innerText; + if (tagName) { + post.tags.push(new ScrapedTag(tagName, tagCategory)); } - - // Set rating - post.rating = "safe"; - - // Set tags - const exp = new RegExp(/(.*)_(\w+)/); - const tagElements = document.getElementsByClassName("quicktag"); - - for (const tagElement of tagElements) { - // id = quicktag1_1009628 - // match[0] = quicktag1_1009628 (aka input str) - // match[1] = quicktag1 - // match[2] = 1009628 - const match = exp.exec(tagElement.id); - if (!match || match.length != 3) { continue; } - - let tagCategory: TagCategory | undefined; - switch (match[1]) { - // Source, aka copyright - case "quicktag2": - tagCategory = "copyright"; - break; - // Artist - case "quicktag3": - tagCategory = "artist"; - break; - // Characters - case "quicktag4": - tagCategory = "character"; - break; - } - - for (const el of tagElement.children) { - const tagName = (el?.firstElementChild as HTMLAnchorElement)?.innerText; - if (tagName) { - post.tags.push(new ScrapedTag(tagName, tagCategory)); - } - } - } - - result.tryAddPost(post); - - return result; + } } + + result.tryAddPost(post); + + return result; + } } diff --git a/src/engines/Twitter.ts b/src/engines/Twitter.ts index 7b46cca..a1c6c81 100644 --- a/src/engines/Twitter.ts +++ b/src/engines/Twitter.ts @@ -1,40 +1,48 @@ import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine"; +import { guessContentType } from "../Utility"; export function getOriginalImageUrl(imageUrl: string) { - const x = imageUrl.replace(/:large|:med|:small|:orig$/, ""); // Remove `:large`, `:small` etc from url - return x + ":orig"; // Add :orig to url (to get original image) + const x = imageUrl.replace(/:large|:med|:small|:orig$/, ""); // Remove `:large`, `:small` etc from url + return x + ":orig"; // Add :orig to url (to get original image) } export default class Twitter implements ScrapeEngine { - name = "twitter"; + name = "twitter"; - canImport(url: Location): boolean { - return url.host == "twitter.com"; + canImport(url: Location): boolean { + return url.host == "twitter.com"; + } + + scrapeDocument(document: Document): ScrapeResult { + let result = new ScrapeResult(this.name); + let post = new ScrapedPost(); + post.pageUrl = document.location.href; + + // Set image url + // When clicked on post (not fullscreen, with post text etc) + const overlayElement = document.querySelector("#permalink-overlay-dialog"); + if (overlayElement) { + const cardImageElement = overlayElement.querySelector("div[data-element-context='platform_photo_card'] > img"); + if (cardImageElement) { + post.contentUrl = getOriginalImageUrl((cardImageElement as HTMLImageElement).src); + } } - scrapeDocument(document: Document): ScrapeResult { - let result = new ScrapeResult(this.name); - let post = new ScrapedPost(); - post.pageUrl = document.location.href; - - // Set image url - // When clicked on post (not fullscreen, with post text etc) - const overlayElement = document.querySelector("#permalink-overlay-dialog"); - if (overlayElement) { - const cardImageElement = overlayElement.querySelector("div[data-element-context='platform_photo_card'] > img"); - if (cardImageElement) { - post.imageUrl = getOriginalImageUrl((cardImageElement as HTMLImageElement).src); - } - } - - // When clicked on image (fullscreen) - const mediaImageElements = document.querySelectorAll("div.Gallery-media > img.media-image"); - if (mediaImageElements.length > 0) { - post.imageUrl = getOriginalImageUrl((mediaImageElements[0] as HTMLImageElement).src); - } - - result.tryAddPost(post); - - return result; + // When clicked on image (fullscreen) + const mediaImageElements = document.querySelectorAll("div.Gallery-media > img.media-image"); + if (mediaImageElements.length > 0) { + post.contentUrl = getOriginalImageUrl((mediaImageElements[0] as HTMLImageElement).src); } + + if (post.contentUrl == undefined) { + return result; + } + + // Set content type + post.contentType = guessContentType(post.contentUrl); + + result.tryAddPost(post); + + return result; + } } diff --git a/src/engines/Zerochan.ts b/src/engines/Zerochan.ts index ffb1fac..c73b174 100644 --- a/src/engines/Zerochan.ts +++ b/src/engines/Zerochan.ts @@ -1,5 +1,6 @@ import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; import { TagCategory } from "../BooruTypes"; +import { guessContentType } from "../Utility"; export default class Zerochan implements ScrapeEngine { name = "zerochan"; @@ -17,15 +18,23 @@ export default class Zerochan implements ScrapeEngine { const previewElements = document.getElementsByClassName("preview"); if (previewElements.length > 0) { // The preview element only exists when there is a larger size available. Preview's img element contains a downscaled version. - post.imageUrl = (previewElements[0] as HTMLAnchorElement).href; + post.contentUrl = (previewElements[0] as HTMLAnchorElement).href; } else { // When there is no larger version available just use the displayed version. const imgElement = document.querySelector("img[title='No larger size available']") as HTMLImageElement; if (imgElement) { - post.imageUrl = imgElement.src; + post.contentUrl = imgElement.src; } } + if (post.contentUrl == undefined) { + // No point in continuing when we don't have an image. + return result; + } + + // Set content type + post.contentType = guessContentType(post.contentUrl); + // Set rating post.rating = "safe"; diff --git a/src/engines/e621.ts b/src/engines/e621.ts index 1d9c60c..6a901b5 100644 --- a/src/engines/e621.ts +++ b/src/engines/e621.ts @@ -16,7 +16,16 @@ export default class e621 implements ScrapeEngine { // Set image url const downloadEl = document.querySelector("#image-download-link > a") as HTMLAnchorElement; - post.imageUrl = downloadEl?.href; + post.contentUrl = downloadEl?.href; + + if (post.contentUrl == undefined) { + return result; + } + + // Set content type + if (document.querySelector("video#image") != null) { + post.contentType = "video"; + } // Set rating const ratingEl = document.getElementById("post-rating-text") as HTMLSpanElement; diff --git a/tsconfig.json b/tsconfig.json index 99b1cfc..5c94b47 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { "lib": [ - "es6", + "es2019", "dom", "dom.iterable" ], @@ -19,4 +19,4 @@ "include": [ "src/index.ts" ] -} +} \ No newline at end of file