diff --git a/src/Utility.ts b/src/Utility.ts index cf568fb..c33062b 100644 --- a/src/Utility.ts +++ b/src/Utility.ts @@ -180,10 +180,13 @@ export function createNotesFromMoebooruBoxes(document: Document, boxSize: [numbe return notes; } -export function parseResolutionString(str: string): [number, number] | undefined { +export function parseResolutionString(str: string | undefined | null): [number, number] | undefined { + if (!str) return undefined; + // Example string: 1600x2200 const res = str .split("x") // Split on 'x' character + .map((v, _) => v.replace(/\D/g, "")) // Remove all non-digits .map((v, _) => parseInt(v)) // Parse ints .filter(Number); // Filter NaN. Also removes the number 0, but that's fine because the resolution can't be 0. diff --git a/src/engines/Gelbooru.test.ts b/src/engines/Gelbooru.test.ts index dfbfa4e..b48cbb0 100644 --- a/src/engines/Gelbooru.test.ts +++ b/src/engines/Gelbooru.test.ts @@ -62,3 +62,17 @@ test("get rule34.xxx post 2740994", async () => { expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].contentUrl).toContain(".rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg"); }); + +test("get xbooru.com post 834083", async () => { + // Don't worry this isn't a cursed image. + const pageUrl = "https://xbooru.com/index.php?page=post&s=view&id=834083"; + const res = await scrapeUrl(pageUrl); + expect(res.posts[0]).toHaveTag("friendship_is_magic", "copyright"); + expect(res.posts[0]).toHaveTag("nurse_redheart", "character"); + expect(res.posts[0]).toHaveTag("nurse"); + expect(res.posts[0]).toHaveTag("standing"); + expect(res.posts[0]).toHaveResolution([1024, 1878]); + expect(res.posts.length).toBe(1); + expect(res.posts[0].pageUrl).toBe(pageUrl); + expect(res.posts[0].contentUrl).toBe("https://img.xbooru.com//images/584/040dc1625c094a7d5d7b0918aa0d8324.png"); +}); diff --git a/src/engines/Gelbooru.ts b/src/engines/Gelbooru.ts index 0018556..dcd6a9e 100644 --- a/src/engines/Gelbooru.ts +++ b/src/engines/Gelbooru.ts @@ -21,7 +21,11 @@ export default class Gelbooru extends ScrapeEngineBase { canImport(url: Location): boolean { return ( - url.host == "safebooru.org" || url.host == "gelbooru.com" || url.host == "rule34.xxx" || url.host == "tbib.org" + url.host == "safebooru.org" || + url.host == "gelbooru.com" || + url.host == "rule34.xxx" || + url.host == "tbib.org" || + url.host == "xbooru.com" ); } @@ -32,6 +36,7 @@ export default class Gelbooru extends ScrapeEngineBase { case "safebooru.org": case "rule34.xxx": case "tbib.org": + case "xbooru.com": version = Version.v020; break; case "gelbooru.com": diff --git a/src/engines/Rule34us.test.ts b/src/engines/Rule34us.test.ts new file mode 100644 index 0000000..3876de2 --- /dev/null +++ b/src/engines/Rule34us.test.ts @@ -0,0 +1,18 @@ +import { scrapeUrl } from "../test/engine"; + +test("get rule34.us post 4646579", async () => { + // Kinda sketchy. + const pageUrl = "https://rule34.us/index.php?r=posts/view&id=4646579"; + const res = await scrapeUrl(pageUrl); + expect(res.posts.length).toBe(1); + expect(res.posts[0]).toHaveTag("sciamano240", "artist"); + expect(res.posts[0]).toHaveTag("mario", "character"); + expect(res.posts[0]).toHaveTag("nintendo", "copyright"); + expect(res.posts[0]).toHaveTag("crown"); + expect(res.posts[0]).toHaveTag("solo"); + expect(res.posts[0]).not.toHaveTag("original"); + expect(res.posts[0]).not.toHaveTag("edit"); + expect(res.posts[0]).toHaveResolution([2100, 3000]); + expect(res.posts[0].pageUrl).toBe(pageUrl); + expect(res.posts[0].contentUrl).toBe("https://img2.rule34.us/images/91/16/91160933dc1b3c737b3ef4a25b12680d.png"); +}); diff --git a/src/engines/Rule34us.ts b/src/engines/Rule34us.ts index 18f4118..5f50059 100644 --- a/src/engines/Rule34us.ts +++ b/src/engines/Rule34us.ts @@ -4,14 +4,14 @@ import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; import { TagCategory } from "../BooruTypes"; import { CategoryMap } from "./Common"; -import { guessContentType } from "../Utility"; +import { guessContentType, parseResolutionString } from "../Utility"; export default class Gelbooru implements ScrapeEngine { name = "rule34us"; private readonly classNameToCategoryMap: CategoryMap = { "character-tag": "character", - "artist-tag": "character", + "artist-tag": "artist", "copyright-tag": "copyright", "metadata-tag": "meta", }; @@ -40,6 +40,13 @@ export default class Gelbooru implements ScrapeEngine { // Set content type post.contentType = guessContentType(post.contentUrl); + // Set resolution + const statEls = Array.from(document.querySelectorAll(".container > ul > div > li")); + const sizeEl = statEls.find((x) => x.innerText.indexOf("Size:") != -1); + if (sizeEl) { + post.resolution = parseResolutionString(sizeEl.innerText); + } + // Patch contentUrl if contentType == "video" // This is because the "Original" button does link to "302 Found" for videos. Idk why. // Shit's fucked, there are multiple workarounds for that and this is one of them. diff --git a/src/engines/SankakuComplex.test.ts b/src/engines/SankakuComplex.test.ts index c2101d0..589b7f7 100644 --- a/src/engines/SankakuComplex.test.ts +++ b/src/engines/SankakuComplex.test.ts @@ -14,8 +14,8 @@ test("get chan.sankakucomplex.com post 19614481 with notes", async () => { expect(res.posts[0]).toHaveNote("What are you looking at..."); expect(res.posts[0]).toHaveNote("What...?"); expect(res.posts[0].pageUrl).toBe(pageUrl); - expect(res.posts[0].contentUrl).toBe( - "https://s.sankakucomplex.com/data/50/7e/507e986a62cc5b866d92ad491dcfb1a4.jpg?e=1637625160&m=e0n_DCg-3fnmrhGzpI0Ftg" + expect(res.posts[0].contentUrl).toContain( + "https://s.sankakucomplex.com/data/50/7e/507e986a62cc5b866d92ad491dcfb1a4.jpg" ); }); @@ -31,7 +31,7 @@ test("get chan.sankakucomplex.com post 29354061 with notes", async () => { expect(res.posts[0]).toHaveTag("note_translated"); expect(res.posts[0]).toHaveNote("No. \nI already told you..."); expect(res.posts[0].pageUrl).toBe(pageUrl); - expect(res.posts[0].contentUrl).toBe( - "https://s.sankakucomplex.com/data/c8/37/c8376f02608548365d394a658e85cb3f.jpg?e=1637625020&m=A9xq-zht_FoVfoCiSRA-GA" + expect(res.posts[0].contentUrl).toContain( + "https://s.sankakucomplex.com/data/c8/37/c8376f02608548365d394a658e85cb3f.jpg" ); }); diff --git a/src/engines/Shuushuu.test.ts b/src/engines/Shuushuu.test.ts index 05c8a26..f4da729 100644 --- a/src/engines/Shuushuu.test.ts +++ b/src/engines/Shuushuu.test.ts @@ -4,6 +4,12 @@ test("get e-shuushuu.net post 1063586", async () => { const pageUrl = "https://e-shuushuu.net/image/1063586/"; const res = await scrapeUrl(pageUrl); expect(res.posts.length).toBe(1); + expect(res.posts[0]).toHaveTag("monster"); + expect(res.posts[0]).toHaveTag("wings"); + expect(res.posts[0]).toHaveTag("celebi", "character"); + expect(res.posts[0]).toHaveTag("shiina_mirin", "artist"); + expect(res.posts[0]).toHaveTag("pokémon", "copyright"); + expect(res.posts[0]).toHaveResolution([1447, 1023]); expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].contentUrl).toBe("https://e-shuushuu.net/images/2021-08-28-1063586.jpeg"); }); diff --git a/src/engines/Shuushuu.ts b/src/engines/Shuushuu.ts index 988c48d..a2302af 100644 --- a/src/engines/Shuushuu.ts +++ b/src/engines/Shuushuu.ts @@ -1,6 +1,6 @@ import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; import { TagCategory } from "../BooruTypes"; -import { guessContentType } from "../Utility"; +import { guessContentType, parseResolutionString } from "../Utility"; export default class Shuushuu implements ScrapeEngine { name = "shuushuu"; @@ -26,6 +26,19 @@ export default class Shuushuu implements ScrapeEngine { // Set content type post.contentType = guessContentType(post.contentUrl); + // Set resolution + const metaEls = Array.from(document.querySelector(".meta > dl")?.children ?? []); + const dimsDtEl = metaEls.find((x) => (x).innerText == "Dimensions:"); + if (dimsDtEl) { + const dimsDtIdx = metaEls.indexOf(dimsDtEl); + let str = (metaEls[dimsDtIdx + 1])?.innerText; + if (str) { + // When str is "1447x1023 (1.480 MPixel)" + str = str.split("(")[0]; // Remove the "(1.480 MPixel)" part + post.resolution = parseResolutionString(str); + } + } + // Set rating post.rating = "safe"; diff --git a/src/engines/Zerochan.test.ts b/src/engines/Zerochan.test.ts index 2115c21..cfb8269 100644 --- a/src/engines/Zerochan.test.ts +++ b/src/engines/Zerochan.test.ts @@ -4,8 +4,11 @@ test("get zerochan.net post 2934998", async () => { const pageUrl = "https://www.zerochan.net/2934998"; const res = await scrapeUrl(pageUrl); expect(res.posts.length).toBe(1); + expect(res.posts[0]).toHaveTag("nintendo", "copyright"); + expect(res.posts[0]).toHaveTag("akihorisu", "artist"); + expect(res.posts[0]).toHaveTag("shirona_(pokémon)", "character"); + expect(res.posts[0]).toHaveResolution([1186, 1009]); expect(res.posts[0].pageUrl).toBe(pageUrl); - expect(res.posts[0].tags.find((x) => x.name == "nintendo")).not.toBeUndefined(); expect(res.posts[0].contentUrl).toBe("https://static.zerochan.net/Shirona.%28Pok%C3%A9mon%29.full.2934998.png"); }); @@ -13,7 +16,8 @@ test("get zerochan.net gif post 3511809", async () => { const pageUrl = "https://www.zerochan.net/3511809"; const res = await scrapeUrl(pageUrl); expect(res.posts.length).toBe(1); - expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0]).toHaveTag("rwby"); + expect(res.posts[0]).toHaveResolution([500, 150]); + expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].contentUrl).toBe("https://static.zerochan.net/Ruby.Rose.full.3511809.gif"); }); diff --git a/src/engines/Zerochan.ts b/src/engines/Zerochan.ts index 411fbb0..e05a489 100644 --- a/src/engines/Zerochan.ts +++ b/src/engines/Zerochan.ts @@ -1,6 +1,6 @@ import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; import { TagCategory } from "../BooruTypes"; -import { guessContentType } from "../Utility"; +import { guessContentType, parseResolutionString } from "../Utility"; export default class Zerochan implements ScrapeEngine { name = "zerochan"; @@ -30,6 +30,12 @@ export default class Zerochan implements ScrapeEngine { // Set content type post.contentType = guessContentType(post.contentUrl); + // Set resolution + const resSizeEl = document.querySelector("#large > p > br")?.parentNode as HTMLParagraphElement; + if (resSizeEl) { + post.resolution = parseResolutionString(resSizeEl.childNodes[0].textContent); + } + // Set rating post.rating = "safe"; @@ -50,9 +56,11 @@ export default class Zerochan implements ScrapeEngine { switch (tagElement.className) { case "game": case "series": + case "studio": category = "copyright"; break; case "character": + case "character group": category = "character"; break; case "mangaka": diff --git a/src/test/jest.setup.ts b/src/test/jest.setup.ts index 07e1924..32c2220 100644 --- a/src/test/jest.setup.ts +++ b/src/test/jest.setup.ts @@ -3,10 +3,9 @@ import { TagCategory } from "../BooruTypes"; import { ScrapedPost } from "../ScrapeEngine"; // Expose JSDOM Element constructor -// @ts-ignore global.Element = new JSDOM().window.Element; // 'Implement' innerText in JSDOM: https://github.com/jsdom/jsdom/issues/1245 -// @ts-ignore +// This implementation is not 100% correct, but usually good enough so that the tests pass. Object.defineProperty(global.Element.prototype, "innerText", { get() { return this.textContent.replace(/(\r\n|\n|\r|^\s+|\s+$)/gm, "");