diff --git a/package.json b/package.json index 501d8ba..98e96c6 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "types": "dist/index.d.js", "scripts": { "test": "jest", + "test:ci": "jest --colors --silent", "build": "tsc", "watch": "tsc -w", "prepare": "npm run build" @@ -38,7 +39,12 @@ "testRegex": "^.+\\.test\\.ts$", "setupFilesAfterEnv": [ "./src/test/jest.setup.ts" - ] + ], + "globals": { + "ts-jest": { + "tsconfig": "tsconfig.test.json" + } + } }, "dependencies": { "turndown": "^7.1.1" diff --git a/src/ScrapeEngine.ts b/src/ScrapeEngine.ts index f641086..e532e96 100644 --- a/src/ScrapeEngine.ts +++ b/src/ScrapeEngine.ts @@ -25,7 +25,7 @@ export class ScrapedTag { } export class ScrapedNote { - constructor(public text: string, public polygons: number[][]) {} + constructor(public text: string, public polygon: number[][]) {} } export class ScrapeResult { diff --git a/src/Utility.ts b/src/Utility.ts index ec32d32..feac920 100644 --- a/src/Utility.ts +++ b/src/Utility.ts @@ -54,6 +54,41 @@ export function htmlNoteToMarkdownNote(text: string) { return new TurndownService().turndown(text); } +export function validateNote(note: ScrapedNote) { + if (note.text.length == 0) { + console.log("[validateNote] No text."); + return false; + } + + for (const poly of note.polygon) { + if (poly.some((x) => x > 1 || x < 0)) { + console.log("[validateNote] Polygon out of range."); + return false; + } + } + + return true; +} + +export function xywhToNormalizedPolygon(x: number, y: number, w: number, h: number, resolution: [number, number]) { + // Normalize our numbers from 0 to 1 + x = x / resolution[0]; + y = y / resolution[1]; + w = w / resolution[0]; + h = h / resolution[1]; + + return [ + // Top left + [x, y], + // Top right + [x + w, y], + // Bottom right + [x + w, y + h], + // Bottom left + [x, y + h], + ]; +} + export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement): ScrapedNote | undefined { function logFail(str: string) { console.log("[createNoteFromDanbooruArticle] Can't create note. " + str); @@ -72,17 +107,6 @@ export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement return num; } - function validateNote(note: ScrapedNote) { - for (const poly of note.polygons) { - if (poly.some((x) => x > 1 || x < 0)) { - logFail("Polygon out of range."); - return false; - } - } - - return true; - } - if (!post.resolution) { logFail("Post resolution is undefined."); return undefined; @@ -110,27 +134,48 @@ export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement return undefined; } - // Normalize our numbers from 0 to 1 - x = x / post.resolution[0]; - y = y / post.resolution[1]; - w = w / post.resolution[0]; - h = h / post.resolution[1]; + const polygon = xywhToNormalizedPolygon(x, y, w, h, post.resolution); + const note = new ScrapedNote(text, polygon); - const polygons = [ - // Top left - [x, y], - // Top right - [x + w, y], - // Bottom right - [x + w, y + h], - // Bottom left - [x, y + h], - ]; - - const note = new ScrapedNote(text, polygons); if (validateNote(note)) { return note; } else { return undefined; } } + +/** + * Creates notes by reading ".note-box" and ".note-body" elements. + * @param document + * @param boxSize + * @returns + */ +export function createNotesFromMoebooruBoxes(document: Document, boxSize: [number, number]) { + let notes: ScrapedNote[] = []; + const noteBoxes = Array.from(document.querySelectorAll(".note-box")).map((x) => x as HTMLLIElement); + const noteBodies = Array.from(document.querySelectorAll(".note-body")).map((x) => x as HTMLLIElement); + + if (noteBoxes.length != noteBodies.length) { + console.log("[createNotesFromMoebooruBoxes] noteBoxes.length != noteBodies.length"); + } else { + for (let i = 0; i < noteBoxes.length; i++) { + const noteBox = noteBoxes[i]; + const noteBody = noteBodies[i]; + + const text = htmlNoteToMarkdownNote(noteBody.innerHTML); + + const w = parseInt(noteBox.style.width); + const h = parseInt(noteBox.style.height); + const y = parseInt(noteBox.style.top); + const x = parseInt(noteBox.style.left); + + const polygon = xywhToNormalizedPolygon(x, y, w, h, boxSize); + const note = new ScrapedNote(text, polygon); + + if (validateNote(note)) { + notes.push(note); + } + } + } + return notes; +} diff --git a/src/engines/Danbooru.test.ts b/src/engines/Danbooru.test.ts index 3f1b2bc..4b269c7 100644 --- a/src/engines/Danbooru.test.ts +++ b/src/engines/Danbooru.test.ts @@ -4,6 +4,11 @@ test("get danbooru.donmai.us post 3555872", async () => { const pageUrl = "https://danbooru.donmai.us/posts/3555872"; const res = await scrapeUrl(pageUrl); expect(res.posts.length).toBe(1); + expect(res.posts[0]).toHaveTag("bangs"); + expect(res.posts[0]).toHaveTag("shirt"); + expect(res.posts[0]).toHaveTag("skirt"); + expect(res.posts[0]).toHaveTag("t-shirt"); + expect(res.posts[0]).toHaveTag("long_hair"); expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].contentUrl).toBe( "https://cdn.donmai.us/original/90/33/__flint_girls_und_panzer_and_1_more_drawn_by_omachi_slabco__90332299216db81b8793084e4cb7c15d.jpg" @@ -19,5 +24,5 @@ test("get danbooru.donmai.us post 4924490", async () => { "https://cdn.donmai.us/original/c5/fb/__yuuki_and_tamaki_princess_connect_drawn_by_yuureidoushi_yuurei6214__c5fbc62758d0f1d8c07aac524282434e.png" ); expect(res.posts[0].notes).toHaveLength(17); - expect(res.posts[0].notes[0].text).toBe("Nya~nya ♪"); + expect(res.posts[0]).toHaveNote("Nya~nya ♪"); }); diff --git a/src/engines/Gelbooru.test.ts b/src/engines/Gelbooru.test.ts index b9cdedf..8b5b315 100644 --- a/src/engines/Gelbooru.test.ts +++ b/src/engines/Gelbooru.test.ts @@ -18,6 +18,25 @@ test("get gelbooru.com post 4925055", async () => { expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg"); }); +test("get gelbooru.com post 6225495 with notes", async () => { + const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=6225495"; + const res = await scrapeUrl(pageUrl); + expect(res.posts.length).toBe(1); + expect(res.posts[0]).toHaveTag("bangs"); + expect(res.posts[0]).toHaveTag("hair_up"); + expect(res.posts[0]).toHaveTag("hair_up"); + expect(res.posts[0]).toHaveTag("teeth"); + expect(res.posts[0]).toHaveTag("highres"); + expect(res.posts[0]).toHaveTag("nintendo"); + expect(res.posts[0]).toHaveTag("link"); + expect(res.posts[0]).toHaveTag("ttanuu."); + expect(res.posts[0]).toHaveNote("I'm fine, more or less."); + expect(res.posts[0]).toHaveNote("But no need to worry!"); + expect(res.posts[0]).toHaveNote("You're always the best just by being yourself-!!!"); + expect(res.posts[0].pageUrl).toBe(pageUrl); + expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/af/1d/af1dd661fab6714dffc29cb38b6b2269.png"); +}); + test("get rule34.xxx post 2740994", async () => { const pageUrl = "https://rule34.xxx/index.php?page=post&s=view&id=2740994"; const res = await scrapeUrl(pageUrl); diff --git a/src/engines/Inkbunny.test.ts b/src/engines/Inkbunny.test.ts new file mode 100644 index 0000000..529a1c1 --- /dev/null +++ b/src/engines/Inkbunny.test.ts @@ -0,0 +1,12 @@ +import { scrapeUrl } from "../test/engine"; + +test("get inkbunny.net post 2430530", async () => { + const pageUrl = "https://inkbunny.net/s/2430530"; + const res = await scrapeUrl(pageUrl); + expect(res.posts.length).toBe(1); + expect(res.posts[0]).toHaveTag("comic"); + expect(res.posts[0]).toHaveTag("flower"); + expect(res.posts[0]).toHaveTag("medusa"); + expect(res.posts[0].pageUrl).toBe(pageUrl); + expect(res.posts[0].contentUrl).toBe("https://nl.ib.metapix.net/files/full/3562/3562592_vavacung_medusa.png"); +}); diff --git a/src/engines/Moebooru.test.ts b/src/engines/Moebooru.test.ts index f2d8fce..97ef63a 100644 --- a/src/engines/Moebooru.test.ts +++ b/src/engines/Moebooru.test.ts @@ -1,4 +1,4 @@ -import { scrapeUrl } from "../test/engine"; +import { scrapeUrl, scrapeEvalUrl } from "../test/engine"; test("get yande.re post 438510", async () => { const pageUrl = "https://yande.re/post/show/438510"; @@ -10,6 +10,18 @@ test("get yande.re post 438510", async () => { ); }); +test("get yande.re post 684478 with notes", async () => { + const pageUrl = "https://yande.re/post/show/684478"; + const res = await scrapeEvalUrl(pageUrl, "#image"); + expect(res.posts.length).toBe(1); + expect(res.posts[0]).toHaveTag("gym_uniform"); + expect(res.posts[0]).toHaveNote("post #671879"); + expect(res.posts[0].pageUrl).toBe(pageUrl); + expect(res.posts[0].contentUrl).toBe( + "https://files.yande.re/image/8bdf7c54b0a459acee080f0aeb2a259c/yande.re%20684478%20gym_uniform%20pal_lett.jpg" + ); +}); + test("get konachan.com post 289968", async () => { const pageUrl = "https://konachan.com/post/show/289968/ahri_-league_of_legends-akali-animal_ears-blonde_h"; const res = await scrapeUrl(pageUrl); @@ -19,3 +31,35 @@ test("get konachan.com post 289968", async () => { "https://konachan.com/image/e40b38fb704434e4da3eae41e26bca39/Konachan.com%20-%20289968%20akali%20blue_eyes%20breasts%20choker%20cleavage%20evelynn%20foxgirl%20gloves%20group%20hat%20kai%27sa%20long_hair%20microphone%20ponytail%20saphirya%20short_hair%20signed%20watermark.jpg" ); }); + +test("get konachan.com post 327454 with notes", async () => { + // We need to use scrapeEvalUrl because otherwise it doesn't work correctly. + const pageUrl = "https://konachan.com/post/show/327454/apron-brown_hair-building-car-gloves-kexue-long_ha"; + const res = await scrapeEvalUrl(pageUrl, "#image"); + expect(res.posts.length).toBe(1); + expect(res.posts[0]).toHaveTag("original"); + expect(res.posts[0]).toHaveTag("kexue"); + expect(res.posts[0]).toHaveTag("water"); + expect(res.posts[0]).toHaveNote("Under construction. \nBe careful of your safety"); + expect(res.posts[0].pageUrl).toBe(pageUrl); + expect(res.posts[0].contentUrl).toBe( + "https://konachan.com/image/94a13c8a1fcb9c0f6c9a6c6c5c316259/Konachan.com%20-%20327454%20apron%20brown_hair%20building%20car%20gloves%20kexue%20long_hair%20original%20rain%20see_through%20short_hair%20twintails%20uniform%20water.jpg" + ); +}); + +test("get konachan.com post 285782 with notes", async () => { + const pageUrl = "https://konachan.com/post/show/285782/anthropomorphism-bed-black_eyes-blonde_hair-croppe"; + const res = await scrapeUrl(pageUrl); + expect(res.posts.length).toBe(1); + expect(res.posts[0]).toHaveTag("kantai_collection"); + expect(res.posts[0]).toHaveTag("momoniku"); + expect(res.posts[0]).toHaveTag("fang"); + expect(res.posts[0]).toHaveTag("cropped"); + expect(res.posts[0]).toHaveNote("I'M AWAKE!!"); + expect(res.posts[0]).toHaveNote("Poi!"); + expect(res.posts[0]).toHaveNote("Haaah!"); + expect(res.posts[0].pageUrl).toBe(pageUrl); + expect(res.posts[0].contentUrl).toBe( + "https://konachan.com/image/24f5974ad8ed725292ccb3f682181bc1/Konachan.com%20-%20285782%20anthropomorphism%20bed%20black_eyes%20blonde_hair%20cropped%20fang%20kantai_collection%20long_hair%20momoniku%20pajamas%20white%20yuudachi_%28kancolle%29.jpg" + ); +}); diff --git a/src/engines/Moebooru.ts b/src/engines/Moebooru.ts index ce166aa..73586e6 100644 --- a/src/engines/Moebooru.ts +++ b/src/engines/Moebooru.ts @@ -1,8 +1,8 @@ -import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; +import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapedNote } from "../ScrapeEngine"; import { TagCategory } from "../BooruTypes"; -import { guessContentType } from "../Utility"; +import { createNotesFromMoebooruBoxes, guessContentType } from "../Utility"; -export default class Moebooru implements ScrapeEngine { +export default class Moebooru extends ScrapeEngineBase { name = "moebooru"; canImport(url: Location): boolean { @@ -52,6 +52,21 @@ export default class Moebooru implements ScrapeEngine { break; } } + // If size element + else if (matches[1] == "Size") { + // Example string: 1600x2200 + const res = matches[2] + .split("x") // Split on 'x' character + .map((v, _) => parseInt(v)) // Parse ints + .filter(Number); // Filter NaN. Also removes the number 0, but that's fine because the resolution can't be 0. + + if (res.length == 2) { + // Kinda hacky, but this is currently the correct way to convert an array to a tuple. + post.resolution = [res[0], res[1]]; + } else { + this.log("Couldn't parse post resolution."); + } + } } } @@ -96,6 +111,15 @@ export default class Moebooru implements ScrapeEngine { post.tags.push(tag); } + // Set notes + const noteBoxSizeEl = document.getElementById("image") as HTMLImageElement; + if (!noteBoxSizeEl) { + this.log("noteBoxSizeEl is undefined."); + } else { + const boxSize: [number, number] = [noteBoxSizeEl.width, noteBoxSizeEl.height]; + post.notes = createNotesFromMoebooruBoxes(document, boxSize); + } + result.tryAddPost(post); return result; diff --git a/src/engines/SankakuComplex.test.ts b/src/engines/SankakuComplex.test.ts new file mode 100644 index 0000000..c2101d0 --- /dev/null +++ b/src/engines/SankakuComplex.test.ts @@ -0,0 +1,37 @@ +import { scrapeUrl } from "../test/engine"; + +test("get chan.sankakucomplex.com post 19614481 with notes", async () => { + const pageUrl = "https://chan.sankakucomplex.com/post/show/19614481"; + const res = await scrapeUrl(pageUrl); + expect(res.posts.length).toBe(1); + expect(res.posts[0]).toHaveTag("bangs"); + expect(res.posts[0]).toHaveTag("bare_shoulders"); + expect(res.posts[0]).toHaveTag("solo"); + expect(res.posts[0]).toHaveTag("translated"); + expect(res.posts[0]).toHaveTag("text"); + expect(res.posts[0]).toHaveTag("idolmaster"); + expect(res.posts[0]).toHaveTag("mayuzumi_fuyuko"); + expect(res.posts[0]).toHaveNote("What are you looking at..."); + expect(res.posts[0]).toHaveNote("What...?"); + expect(res.posts[0].pageUrl).toBe(pageUrl); + expect(res.posts[0].contentUrl).toBe( + "https://s.sankakucomplex.com/data/50/7e/507e986a62cc5b866d92ad491dcfb1a4.jpg?e=1637625160&m=e0n_DCg-3fnmrhGzpI0Ftg" + ); +}); + +test("get chan.sankakucomplex.com post 29354061 with notes", async () => { + // Kinda sketchy + const pageUrl = "https://chan.sankakucomplex.com/post/show/29354061"; + const res = await scrapeUrl(pageUrl); + expect(res.posts.length).toBe(1); + expect(res.posts[0]).toHaveTag("bangs"); + expect(res.posts[0]).toHaveTag("braid"); + expect(res.posts[0]).toHaveTag("sankuro_(agoitei)"); + expect(res.posts[0]).toHaveTag("translated"); + expect(res.posts[0]).toHaveTag("note_translated"); + expect(res.posts[0]).toHaveNote("No. \nI already told you..."); + expect(res.posts[0].pageUrl).toBe(pageUrl); + expect(res.posts[0].contentUrl).toBe( + "https://s.sankakucomplex.com/data/c8/37/c8376f02608548365d394a658e85cb3f.jpg?e=1637625020&m=A9xq-zht_FoVfoCiSRA-GA" + ); +}); diff --git a/src/engines/SankakuComplex.ts b/src/engines/SankakuComplex.ts index 9405b11..6cf3874 100644 --- a/src/engines/SankakuComplex.ts +++ b/src/engines/SankakuComplex.ts @@ -1,8 +1,8 @@ -import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; +import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapedNote } from "../ScrapeEngine"; import { TagCategory } from "../BooruTypes"; -import { guessContentType } from "../Utility"; +import { createNotesFromMoebooruBoxes, guessContentType } from "../Utility"; -export default class SankakuComplex implements ScrapeEngine { +export default class SankakuComplex extends ScrapeEngineBase { name = "sankakucomplex"; canImport(url: Location): boolean { @@ -81,6 +81,15 @@ export default class SankakuComplex implements ScrapeEngine { } } + // Set notes + const postContentEl = document.querySelector("#image-link > img") as HTMLImageElement; + if (!postContentEl) { + this.log("postContentEl is undefined."); + } else { + const boxSize: [number, number] = [postContentEl.width, postContentEl.height]; + post.notes = createNotesFromMoebooruBoxes(document, boxSize); + } + result.tryAddPost(post); return result; diff --git a/src/test/jest.d.ts b/src/test/jest.d.ts new file mode 100644 index 0000000..fcb9e8a --- /dev/null +++ b/src/test/jest.d.ts @@ -0,0 +1,10 @@ +import "jest-extended"; + +declare global { + namespace jest { + interface Matchers { + toHaveTag(tagName: string): CustomMatcherResult; + toHaveNote(noteText: string): CustomMatcherResult; + } + } +} diff --git a/src/test/jest.setup.ts b/src/test/jest.setup.ts index 7f9a75a..d9c9c16 100644 --- a/src/test/jest.setup.ts +++ b/src/test/jest.setup.ts @@ -12,14 +12,6 @@ Object.defineProperty(global.Element.prototype, "innerText", { }, }); -declare global { - namespace jest { - interface Matchers { - toHaveTag(tagName: string): CustomMatcherResult; - } - } -} - // Add our own functions expect.extend({ toHaveTag(received: any, tagName: string): jest.CustomMatcherResult { @@ -31,4 +23,13 @@ expect.extend({ } return { pass, message }; }, + toHaveNote(received: any, noteText: string): jest.CustomMatcherResult { + let pass: boolean = false; + const message = () => (pass ? "" : `post does not have a note with '${noteText}'.`); + + if (received instanceof ScrapedPost) { + pass = received.notes.find((x) => x.text == noteText) != undefined; + } + return { pass, message }; + }, }); diff --git a/tsconfig.test.json b/tsconfig.test.json new file mode 100644 index 0000000..b7a8112 --- /dev/null +++ b/tsconfig.test.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "lib": [ + "es2019", + "dom", + "dom.iterable" + ], + "outDir": "./dist/", + "sourceMap": true, + "strict": true, + "module": "es2015", + "moduleResolution": "node", + "target": "es2015", + "skipLibCheck": true, + "esModuleInterop": true, + "removeComments": true, + "declaration": true + }, + "include": [ + "src/index.ts", + "src/test/jest.d.ts" + ] +}