diff --git a/src/Utility.ts b/src/Utility.ts index feac920..cf568fb 100644 --- a/src/Utility.ts +++ b/src/Utility.ts @@ -179,3 +179,18 @@ export function createNotesFromMoebooruBoxes(document: Document, boxSize: [numbe } return notes; } + +export function parseResolutionString(str: string): [number, number] | undefined { + // Example string: 1600x2200 + const res = str + .split("x") // Split on 'x' character + .map((v, _) => parseInt(v)) // Parse ints + .filter(Number); // Filter NaN. Also removes the number 0, but that's fine because the resolution can't be 0. + + if (res.length == 2) { + // Kinda hacky, but this is currently the correct way to convert an array to a tuple. + return [res[0], res[1]]; + } else { + console.log(`[parseResolutionString] Couldn't parse resolution from '${str}'.`); + } +} diff --git a/src/engines/AnimePictures.test.ts b/src/engines/AnimePictures.test.ts index e842ee2..44923d3 100644 --- a/src/engines/AnimePictures.test.ts +++ b/src/engines/AnimePictures.test.ts @@ -4,6 +4,14 @@ test("get anime-pictures.net post 729833", async () => { const pageUrl = "https://anime-pictures.net/pictures/view_post/729833?lang=en"; const res = await scrapeUrl(pageUrl); expect(res.posts.length).toBe(1); + expect(res.posts[0]).toHaveTag("arknights", "copyright"); + expect(res.posts[0]).toHaveTag("ch'en_(arknights)", "character"); + expect(res.posts[0]).toHaveTag("ttk_(kirinottk)", "artist"); + expect(res.posts[0]).toHaveTag("single"); + expect(res.posts[0]).toHaveTag("holding"); + expect(res.posts[0]).toHaveTag("gloves"); + expect(res.posts[0]).toHaveTag("jacket"); + expect(res.posts[0]).toHaveResolution([4000, 6865]); expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].contentUrl).toBe( "https://anime-pictures.net/pictures/download_image/729833-4000x6865-arknights-ch%26%2339%3Ben+%28arknights%29-ttk+%28kirinottk%29-single-long+hair-tall+image.jpeg" diff --git a/src/engines/AnimePictures.ts b/src/engines/AnimePictures.ts index 5955136..60a0ddd 100644 --- a/src/engines/AnimePictures.ts +++ b/src/engines/AnimePictures.ts @@ -1,6 +1,6 @@ import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; import { TagCategory } from "../BooruTypes"; -import { guessContentType } from "../Utility"; +import { guessContentType, parseResolutionString } from "../Utility"; export default class AnimePictures implements ScrapeEngine { name = "animepictures"; @@ -20,9 +20,6 @@ export default class AnimePictures implements ScrapeEngine { if (originalImageElements.length > 0) { post.contentUrl = (originalImageElements[0] as HTMLAnchorElement).href; - } else { - // No point in continuing when we don't have an image. - return result; } // Set content type @@ -31,8 +28,17 @@ export default class AnimePictures implements ScrapeEngine { // Set rating post.rating = "safe"; // Probably + // Set resolution + const postContentLinkEls = Array.from( + document.querySelectorAll("#content > div > .post_content > a") + ); + const resEl = postContentLinkEls.find((x) => x.href.indexOf("res_x") != -1); + if (resEl) { + post.resolution = parseResolutionString(resEl.innerText); + } + // Set tags - const tagElements = Array.from(document.querySelectorAll("ul.tags > li > a")).map((x) => x as HTMLAnchorElement); + const tagElements = Array.from(document.querySelectorAll("ul.tags > li > a")); for (const tagElement of tagElements) { let tagName = tagElement.innerText; diff --git a/src/engines/Danbooru.test.ts b/src/engines/Danbooru.test.ts index 4b269c7..e559116 100644 --- a/src/engines/Danbooru.test.ts +++ b/src/engines/Danbooru.test.ts @@ -4,11 +4,15 @@ test("get danbooru.donmai.us post 3555872", async () => { const pageUrl = "https://danbooru.donmai.us/posts/3555872"; const res = await scrapeUrl(pageUrl); expect(res.posts.length).toBe(1); + expect(res.posts[0]).toHaveTag("omachi_(slabco)", "artist"); + expect(res.posts[0]).toHaveTag("dr_pepper", "copyright"); + expect(res.posts[0]).toHaveTag("flint_(girls_und_panzer)", "character"); expect(res.posts[0]).toHaveTag("bangs"); expect(res.posts[0]).toHaveTag("shirt"); expect(res.posts[0]).toHaveTag("skirt"); expect(res.posts[0]).toHaveTag("t-shirt"); expect(res.posts[0]).toHaveTag("long_hair"); + expect(res.posts[0]).toHaveResolution([1142, 1600]); expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].contentUrl).toBe( "https://cdn.donmai.us/original/90/33/__flint_girls_und_panzer_and_1_more_drawn_by_omachi_slabco__90332299216db81b8793084e4cb7c15d.jpg" @@ -19,6 +23,9 @@ test("get danbooru.donmai.us post 4924490", async () => { const pageUrl = "https://danbooru.donmai.us/posts/4924490"; const res = await scrapeUrl(pageUrl); expect(res.posts.length).toBe(1); + expect(res.posts[0]).toHaveTag("yuureidoushi_(yuurei6214)", "artist"); + expect(res.posts[0]).toHaveTag("comic"); + expect(res.posts[0]).toHaveResolution([1000, 2062]); expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].contentUrl).toBe( "https://cdn.donmai.us/original/c5/fb/__yuuki_and_tamaki_princess_connect_drawn_by_yuureidoushi_yuurei6214__c5fbc62758d0f1d8c07aac524282434e.png" diff --git a/src/engines/Gelbooru.test.ts b/src/engines/Gelbooru.test.ts index 8b5b315..dfbfa4e 100644 --- a/src/engines/Gelbooru.test.ts +++ b/src/engines/Gelbooru.test.ts @@ -4,6 +4,14 @@ test("get safebooru.org post 2871532", async () => { const pageUrl = "https://safebooru.org/index.php?page=post&s=view&id=2871532"; const res = await scrapeUrl(pageUrl); expect(res.posts.length).toBe(1); + expect(res.posts[0]).toHaveTag("omachi_(slabco)", "artist"); + expect(res.posts[0]).toHaveTag("flint_(girls_und_panzer)", "character"); + expect(res.posts[0]).toHaveTag("bangs"); + expect(res.posts[0]).toHaveTag("shirt"); + expect(res.posts[0]).toHaveTag("skirt"); + expect(res.posts[0]).toHaveTag("t-shirt"); + expect(res.posts[0]).toHaveTag("long_hair"); + expect(res.posts[0]).toHaveResolution([1142, 1600]); expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].contentUrl).toBe( "https://safebooru.org//images/2757/5f19c7744db918c3d4c4c6035bc65dfa9e18866f.jpg" @@ -14,6 +22,9 @@ test("get gelbooru.com post 4925055", async () => { const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=4925055&tags=kimetsu_no_yaiba"; const res = await scrapeUrl(pageUrl); expect(res.posts.length).toBe(1); + expect(res.posts[0]).toHaveTag("maosen", "artist"); + expect(res.posts[0]).toHaveTag("butterfly"); + expect(res.posts[0]).toHaveResolution([720, 1103]); expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg"); }); @@ -33,6 +44,7 @@ test("get gelbooru.com post 6225495 with notes", async () => { expect(res.posts[0]).toHaveNote("I'm fine, more or less."); expect(res.posts[0]).toHaveNote("But no need to worry!"); expect(res.posts[0]).toHaveNote("You're always the best just by being yourself-!!!"); + expect(res.posts[0]).toHaveResolution([1600, 2200]); expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/af/1d/af1dd661fab6714dffc29cb38b6b2269.png"); }); @@ -40,6 +52,12 @@ test("get gelbooru.com post 6225495 with notes", async () => { test("get rule34.xxx post 2740994", async () => { const pageUrl = "https://rule34.xxx/index.php?page=post&s=view&id=2740994"; const res = await scrapeUrl(pageUrl); + expect(res.posts[0]).toHaveTag("fire_emblem", "copyright"); + expect(res.posts[0]).toHaveTag("nintendo", "copyright"); + expect(res.posts[0]).toHaveTag("lucina", "character"); + expect(res.posts[0]).toHaveTag("blue_hair"); + expect(res.posts[0]).toHaveTag("solo"); + expect(res.posts[0]).toHaveResolution([600, 800]); expect(res.posts.length).toBe(1); expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].contentUrl).toContain(".rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg"); diff --git a/src/engines/Gelbooru.ts b/src/engines/Gelbooru.ts index 08ed49b..0018556 100644 --- a/src/engines/Gelbooru.ts +++ b/src/engines/Gelbooru.ts @@ -1,7 +1,7 @@ import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; import { TagCategory } from "../BooruTypes"; import { CategoryMap } from "./Common"; -import { createNoteFromDanbooruArticle, guessContentType } from "../Utility"; +import { createNoteFromDanbooruArticle, guessContentType, parseResolutionString } from "../Utility"; // Small hack to avoid screwing with semver enum Version { @@ -103,18 +103,7 @@ export default class Gelbooru extends ScrapeEngineBase { } // If size element else if (matches[1] == "Size") { - // Example string: 1600x2200 - const res = matches[2] - .split("x") // Split on 'x' character - .map((v, _) => parseInt(v)) // Parse ints - .filter(Number); // Filter NaN. Also removes the number 0, but that's fine because the resolution can't be 0. - - if (res.length == 2) { - // Kinda hacky, but this is currently the correct way to convert an array to a tuple. - post.resolution = [res[0], res[1]]; - } else { - this.log("Couldn't parse post resolution."); - } + post.resolution = parseResolutionString(matches[2]); } } } diff --git a/src/engines/Moebooru.test.ts b/src/engines/Moebooru.test.ts index 97ef63a..d2f7b2c 100644 --- a/src/engines/Moebooru.test.ts +++ b/src/engines/Moebooru.test.ts @@ -4,6 +4,10 @@ test("get yande.re post 438510", async () => { const pageUrl = "https://yande.re/post/show/438510"; const res = await scrapeUrl(pageUrl); expect(res.posts.length).toBe(1); + expect(res.posts[0]).toHaveTag("the_legend_of_zelda", "copyright"); + expect(res.posts[0]).toHaveTag("link", "character"); + expect(res.posts[0]).toHaveTag("landscape"); + expect(res.posts[0]).toHaveResolution([3000, 1688]); expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].contentUrl).toBe( "https://files.yande.re/image/49b79bc372716a579d8525dcaccb56e2/yande.re%20438510%20landscape%20link%20nintendo%20princess_zelda%20sword%20the_legend_of_zelda%20the_legend_of_zelda%3A_breath_of_the_wild.jpg" @@ -16,6 +20,7 @@ test("get yande.re post 684478 with notes", async () => { expect(res.posts.length).toBe(1); expect(res.posts[0]).toHaveTag("gym_uniform"); expect(res.posts[0]).toHaveNote("post #671879"); + expect(res.posts[0]).toHaveResolution([2480, 3508]); expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].contentUrl).toBe( "https://files.yande.re/image/8bdf7c54b0a459acee080f0aeb2a259c/yande.re%20684478%20gym_uniform%20pal_lett.jpg" @@ -26,6 +31,8 @@ test("get konachan.com post 289968", async () => { const pageUrl = "https://konachan.com/post/show/289968/ahri_-league_of_legends-akali-animal_ears-blonde_h"; const res = await scrapeUrl(pageUrl); expect(res.posts.length).toBe(1); + expect(res.posts[0]).toHaveTag("league_of_legends", "copyright"); + expect(res.posts[0]).toHaveResolution([1200, 783]); expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].contentUrl).toBe( "https://konachan.com/image/e40b38fb704434e4da3eae41e26bca39/Konachan.com%20-%20289968%20akali%20blue_eyes%20breasts%20choker%20cleavage%20evelynn%20foxgirl%20gloves%20group%20hat%20kai%27sa%20long_hair%20microphone%20ponytail%20saphirya%20short_hair%20signed%20watermark.jpg" @@ -41,6 +48,7 @@ test("get konachan.com post 327454 with notes", async () => { expect(res.posts[0]).toHaveTag("kexue"); expect(res.posts[0]).toHaveTag("water"); expect(res.posts[0]).toHaveNote("Under construction. \nBe careful of your safety"); + expect(res.posts[0]).toHaveResolution([5629, 2505]); expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].contentUrl).toBe( "https://konachan.com/image/94a13c8a1fcb9c0f6c9a6c6c5c316259/Konachan.com%20-%20327454%20apron%20brown_hair%20building%20car%20gloves%20kexue%20long_hair%20original%20rain%20see_through%20short_hair%20twintails%20uniform%20water.jpg" @@ -58,6 +66,7 @@ test("get konachan.com post 285782 with notes", async () => { expect(res.posts[0]).toHaveNote("I'M AWAKE!!"); expect(res.posts[0]).toHaveNote("Poi!"); expect(res.posts[0]).toHaveNote("Haaah!"); + expect(res.posts[0]).toHaveResolution([1000, 830]); expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].contentUrl).toBe( "https://konachan.com/image/24f5974ad8ed725292ccb3f682181bc1/Konachan.com%20-%20285782%20anthropomorphism%20bed%20black_eyes%20blonde_hair%20cropped%20fang%20kantai_collection%20long_hair%20momoniku%20pajamas%20white%20yuudachi_%28kancolle%29.jpg" diff --git a/src/engines/Moebooru.ts b/src/engines/Moebooru.ts index 73586e6..910f2b9 100644 --- a/src/engines/Moebooru.ts +++ b/src/engines/Moebooru.ts @@ -1,6 +1,6 @@ import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapedNote } from "../ScrapeEngine"; import { TagCategory } from "../BooruTypes"; -import { createNotesFromMoebooruBoxes, guessContentType } from "../Utility"; +import { createNotesFromMoebooruBoxes, guessContentType, parseResolutionString } from "../Utility"; export default class Moebooru extends ScrapeEngineBase { name = "moebooru"; @@ -54,18 +54,7 @@ export default class Moebooru extends ScrapeEngineBase { } // If size element else if (matches[1] == "Size") { - // Example string: 1600x2200 - const res = matches[2] - .split("x") // Split on 'x' character - .map((v, _) => parseInt(v)) // Parse ints - .filter(Number); // Filter NaN. Also removes the number 0, but that's fine because the resolution can't be 0. - - if (res.length == 2) { - // Kinda hacky, but this is currently the correct way to convert an array to a tuple. - post.resolution = [res[0], res[1]]; - } else { - this.log("Couldn't parse post resolution."); - } + post.resolution = parseResolutionString(matches[2]); } } } diff --git a/src/test/jest.d.ts b/src/test/jest.d.ts index fcb9e8a..c8a44a3 100644 --- a/src/test/jest.d.ts +++ b/src/test/jest.d.ts @@ -1,10 +1,13 @@ import "jest-extended"; +import { TagCategory } from "../BooruTypes"; declare global { namespace jest { interface Matchers { - toHaveTag(tagName: string): CustomMatcherResult; + // The `= undefined` part is incorrect according to VSCode, but it is actually required. + toHaveTag(tagName: string, tagCategory: TagCategory | undefined = undefined): CustomMatcherResult; toHaveNote(noteText: string): CustomMatcherResult; + toHaveResolution(resolution: [number, number]): CustomMatcherResult; } } } diff --git a/src/test/jest.setup.ts b/src/test/jest.setup.ts index d9c9c16..07e1924 100644 --- a/src/test/jest.setup.ts +++ b/src/test/jest.setup.ts @@ -1,4 +1,5 @@ import { JSDOM } from "jsdom"; +import { TagCategory } from "../BooruTypes"; import { ScrapedPost } from "../ScrapeEngine"; // Expose JSDOM Element constructor @@ -14,22 +15,42 @@ Object.defineProperty(global.Element.prototype, "innerText", { // Add our own functions expect.extend({ - toHaveTag(received: any, tagName: string): jest.CustomMatcherResult { + toHaveTag( + received: any, + tagName: string, + tagCategory: TagCategory | undefined = undefined + ): jest.CustomMatcherResult { let pass: boolean = false; - const message = () => (pass ? "" : `post does not have the '${tagName}' tag.`); + const message = () => (pass ? "" : `Post does not have the '${tagName}' tag, or the category is incorrect.`); if (received instanceof ScrapedPost) { - pass = received.tags.find((x) => x.name == tagName) != undefined; + const found = received.tags.find((x) => x.name == tagName); + if (found) { + if (tagCategory) { + pass = found.category == tagCategory; + } else { + pass = true; + } + } } return { pass, message }; }, toHaveNote(received: any, noteText: string): jest.CustomMatcherResult { let pass: boolean = false; - const message = () => (pass ? "" : `post does not have a note with '${noteText}'.`); + const message = () => (pass ? "" : `Post does not have a note with '${noteText}'.`); if (received instanceof ScrapedPost) { pass = received.notes.find((x) => x.text == noteText) != undefined; } return { pass, message }; }, + toHaveResolution(received: any, resolution: [number, number]): jest.CustomMatcherResult { + let pass: boolean = false; + const message = () => (pass ? "" : `Expected: ${resolution}\nReceived: ${received?.resolution}`); + + if (received instanceof ScrapedPost && received.resolution?.length == 2) { + pass = received.resolution[0] == resolution[0] && received.resolution[1] == resolution[1]; + } + return { pass, message }; + }, });