Load notes on SankakuComplex + more tests

package.json

@@ -6,6 +6,7 @@
   "types": "dist/index.d.js",
   "scripts": {
     "test": "jest",
+    "test:ci": "jest --colors --silent",
     "build": "tsc",
     "watch": "tsc -w",
     "prepare": "npm run build"
@@ -38,7 +39,12 @@
     "testRegex": "^.+\\.test\\.ts$",
     "setupFilesAfterEnv": [
       "./src/test/jest.setup.ts"
-    ]
+    ],
+    "globals": {
+      "ts-jest": {
+        "tsconfig": "tsconfig.test.json"
+      }
+    }
   },
   "dependencies": {
     "turndown": "^7.1.1"

src/ScrapeEngine.ts

@@ -25,7 +25,7 @@ export class ScrapedTag {
 }
 
 export class ScrapedNote {
-  constructor(public text: string, public polygons: number[][]) {}
+  constructor(public text: string, public polygon: number[][]) {}
 }
 
 export class ScrapeResult {

src/Utility.ts

@@ -54,6 +54,41 @@ export function htmlNoteToMarkdownNote(text: string) {
   return new TurndownService().turndown(text);
 }
 
+export function validateNote(note: ScrapedNote) {
+  if (note.text.length == 0) {
+    console.log("[validateNote] No text.");
+    return false;
+  }
+
+  for (const poly of note.polygon) {
+    if (poly.some((x) => x > 1 || x < 0)) {
+      console.log("[validateNote] Polygon out of range.");
+      return false;
+    }
+  }
+
+  return true;
+}
+
+export function xywhToNormalizedPolygon(x: number, y: number, w: number, h: number, resolution: [number, number]) {
+  // Normalize our numbers from 0 to 1
+  x = x / resolution[0];
+  y = y / resolution[1];
+  w = w / resolution[0];
+  h = h / resolution[1];
+
+  return [
+    // Top left
+    [x, y],
+    // Top right
+    [x + w, y],
+    // Bottom right
+    [x + w, y + h],
+    // Bottom left
+    [x, y + h],
+  ];
+}
+
 export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement): ScrapedNote | undefined {
   function logFail(str: string) {
     console.log("[createNoteFromDanbooruArticle] Can't create note. " + str);
@@ -72,17 +107,6 @@ export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement
     return num;
   }
 
-  function validateNote(note: ScrapedNote) {
-    for (const poly of note.polygons) {
-      if (poly.some((x) => x > 1 || x < 0)) {
-        logFail("Polygon out of range.");
-        return false;
-      }
-    }
-
-    return true;
-  }
-
   if (!post.resolution) {
     logFail("Post resolution is undefined.");
     return undefined;
@@ -110,27 +134,48 @@ export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement
     return undefined;
   }
 
-  // Normalize our numbers from 0 to 1
+  const polygon = xywhToNormalizedPolygon(x, y, w, h, post.resolution);
-  x = x / post.resolution[0];
+  const note = new ScrapedNote(text, polygon);
-  y = y / post.resolution[1];
-  w = w / post.resolution[0];
-  h = h / post.resolution[1];
 
-  const polygons = [
-    // Top left
-    [x, y],
-    // Top right
-    [x + w, y],
-    // Bottom right
-    [x + w, y + h],
-    // Bottom left
-    [x, y + h],
-  ];
-
-  const note = new ScrapedNote(text, polygons);
   if (validateNote(note)) {
     return note;
   } else {
     return undefined;
   }
 }
+
+/**
+ * Creates notes by reading ".note-box" and ".note-body" elements.
+ * @param document
+ * @param boxSize
+ * @returns
+ */
+export function createNotesFromMoebooruBoxes(document: Document, boxSize: [number, number]) {
+  let notes: ScrapedNote[] = [];
+  const noteBoxes = Array.from(document.querySelectorAll(".note-box")).map((x) => x as HTMLLIElement);
+  const noteBodies = Array.from(document.querySelectorAll(".note-body")).map((x) => x as HTMLLIElement);
+
+  if (noteBoxes.length != noteBodies.length) {
+    console.log("[createNotesFromMoebooruBoxes] noteBoxes.length != noteBodies.length");
+  } else {
+    for (let i = 0; i < noteBoxes.length; i++) {
+      const noteBox = noteBoxes[i];
+      const noteBody = noteBodies[i];
+
+      const text = htmlNoteToMarkdownNote(noteBody.innerHTML);
+
+      const w = parseInt(noteBox.style.width);
+      const h = parseInt(noteBox.style.height);
+      const y = parseInt(noteBox.style.top);
+      const x = parseInt(noteBox.style.left);
+
+      const polygon = xywhToNormalizedPolygon(x, y, w, h, boxSize);
+      const note = new ScrapedNote(text, polygon);
+
+      if (validateNote(note)) {
+        notes.push(note);
+      }
+    }
+  }
+  return notes;
+}

src/engines/Danbooru.test.ts

@@ -4,6 +4,11 @@ test("get danbooru.donmai.us post 3555872", async () => {
   const pageUrl = "https://danbooru.donmai.us/posts/3555872";
   const res = await scrapeUrl(pageUrl);
   expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("bangs");
+  expect(res.posts[0]).toHaveTag("shirt");
+  expect(res.posts[0]).toHaveTag("skirt");
+  expect(res.posts[0]).toHaveTag("t-shirt");
+  expect(res.posts[0]).toHaveTag("long_hair");
   expect(res.posts[0].pageUrl).toBe(pageUrl);
   expect(res.posts[0].contentUrl).toBe(
     "https://cdn.donmai.us/original/90/33/__flint_girls_und_panzer_and_1_more_drawn_by_omachi_slabco__90332299216db81b8793084e4cb7c15d.jpg"
@@ -19,5 +24,5 @@ test("get danbooru.donmai.us post 4924490", async () => {
     "https://cdn.donmai.us/original/c5/fb/__yuuki_and_tamaki_princess_connect_drawn_by_yuureidoushi_yuurei6214__c5fbc62758d0f1d8c07aac524282434e.png"
   );
   expect(res.posts[0].notes).toHaveLength(17);
-  expect(res.posts[0].notes[0].text).toBe("Nya~nya ♪");
+  expect(res.posts[0]).toHaveNote("Nya~nya ♪");
 });

src/engines/Gelbooru.test.ts

@@ -18,6 +18,25 @@ test("get gelbooru.com post 4925055", async () => {
   expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
 });
 
+test("get gelbooru.com post 6225495 with notes", async () => {
+  const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=6225495";
+  const res = await scrapeUrl(pageUrl);
+  expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("bangs");
+  expect(res.posts[0]).toHaveTag("hair_up");
+  expect(res.posts[0]).toHaveTag("hair_up");
+  expect(res.posts[0]).toHaveTag("teeth");
+  expect(res.posts[0]).toHaveTag("highres");
+  expect(res.posts[0]).toHaveTag("nintendo");
+  expect(res.posts[0]).toHaveTag("link");
+  expect(res.posts[0]).toHaveTag("ttanuu.");
+  expect(res.posts[0]).toHaveNote("I'm fine, more or less.");
+  expect(res.posts[0]).toHaveNote("But no need to worry!");
+  expect(res.posts[0]).toHaveNote("You're always the best just by being yourself-!!!");
+  expect(res.posts[0].pageUrl).toBe(pageUrl);
+  expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/af/1d/af1dd661fab6714dffc29cb38b6b2269.png");
+});
+
 test("get rule34.xxx post 2740994", async () => {
   const pageUrl = "https://rule34.xxx/index.php?page=post&s=view&id=2740994";
   const res = await scrapeUrl(pageUrl);

src/engines/Inkbunny.test.ts

@@ -0,0 +1,12 @@
+import { scrapeUrl } from "../test/engine";
+
+test("get inkbunny.net post 2430530", async () => {
+  const pageUrl = "https://inkbunny.net/s/2430530";
+  const res = await scrapeUrl(pageUrl);
+  expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("comic");
+  expect(res.posts[0]).toHaveTag("flower");
+  expect(res.posts[0]).toHaveTag("medusa");
+  expect(res.posts[0].pageUrl).toBe(pageUrl);
+  expect(res.posts[0].contentUrl).toBe("https://nl.ib.metapix.net/files/full/3562/3562592_vavacung_medusa.png");
+});

src/engines/Moebooru.test.ts

@@ -1,4 +1,4 @@
-import { scrapeUrl } from "../test/engine";
+import { scrapeUrl, scrapeEvalUrl } from "../test/engine";
 
 test("get yande.re post 438510", async () => {
   const pageUrl = "https://yande.re/post/show/438510";
@@ -10,6 +10,18 @@ test("get yande.re post 438510", async () => {
   );
 });
 
+test("get yande.re post 684478 with notes", async () => {
+  const pageUrl = "https://yande.re/post/show/684478";
+  const res = await scrapeEvalUrl(pageUrl, "#image");
+  expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("gym_uniform");
+  expect(res.posts[0]).toHaveNote("post #671879");
+  expect(res.posts[0].pageUrl).toBe(pageUrl);
+  expect(res.posts[0].contentUrl).toBe(
+    "https://files.yande.re/image/8bdf7c54b0a459acee080f0aeb2a259c/yande.re%20684478%20gym_uniform%20pal_lett.jpg"
+  );
+});
+
 test("get konachan.com post 289968", async () => {
   const pageUrl = "https://konachan.com/post/show/289968/ahri_-league_of_legends-akali-animal_ears-blonde_h";
   const res = await scrapeUrl(pageUrl);
@@ -19,3 +31,35 @@ test("get konachan.com post 289968", async () => {
     "https://konachan.com/image/e40b38fb704434e4da3eae41e26bca39/Konachan.com%20-%20289968%20akali%20blue_eyes%20breasts%20choker%20cleavage%20evelynn%20foxgirl%20gloves%20group%20hat%20kai%27sa%20long_hair%20microphone%20ponytail%20saphirya%20short_hair%20signed%20watermark.jpg"
   );
 });
+
+test("get konachan.com post 327454 with notes", async () => {
+  // We need to use scrapeEvalUrl because otherwise it doesn't work correctly.
+  const pageUrl = "https://konachan.com/post/show/327454/apron-brown_hair-building-car-gloves-kexue-long_ha";
+  const res = await scrapeEvalUrl(pageUrl, "#image");
+  expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("original");
+  expect(res.posts[0]).toHaveTag("kexue");
+  expect(res.posts[0]).toHaveTag("water");
+  expect(res.posts[0]).toHaveNote("Under construction.  \nBe careful of your safety");
+  expect(res.posts[0].pageUrl).toBe(pageUrl);
+  expect(res.posts[0].contentUrl).toBe(
+    "https://konachan.com/image/94a13c8a1fcb9c0f6c9a6c6c5c316259/Konachan.com%20-%20327454%20apron%20brown_hair%20building%20car%20gloves%20kexue%20long_hair%20original%20rain%20see_through%20short_hair%20twintails%20uniform%20water.jpg"
+  );
+});
+
+test("get konachan.com post 285782 with notes", async () => {
+  const pageUrl = "https://konachan.com/post/show/285782/anthropomorphism-bed-black_eyes-blonde_hair-croppe";
+  const res = await scrapeUrl(pageUrl);
+  expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("kantai_collection");
+  expect(res.posts[0]).toHaveTag("momoniku");
+  expect(res.posts[0]).toHaveTag("fang");
+  expect(res.posts[0]).toHaveTag("cropped");
+  expect(res.posts[0]).toHaveNote("I'M AWAKE!!");
+  expect(res.posts[0]).toHaveNote("Poi!");
+  expect(res.posts[0]).toHaveNote("Haaah!");
+  expect(res.posts[0].pageUrl).toBe(pageUrl);
+  expect(res.posts[0].contentUrl).toBe(
+    "https://konachan.com/image/24f5974ad8ed725292ccb3f682181bc1/Konachan.com%20-%20285782%20anthropomorphism%20bed%20black_eyes%20blonde_hair%20cropped%20fang%20kantai_collection%20long_hair%20momoniku%20pajamas%20white%20yuudachi_%28kancolle%29.jpg"
+  );
+});

src/engines/Moebooru.ts

@@ -1,8 +1,8 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapedNote } from "../ScrapeEngine";
 import { TagCategory } from "../BooruTypes";
-import { guessContentType } from "../Utility";
+import { createNotesFromMoebooruBoxes, guessContentType } from "../Utility";
 
-export default class Moebooru implements ScrapeEngine {
+export default class Moebooru extends ScrapeEngineBase {
   name = "moebooru";
 
   canImport(url: Location): boolean {
@@ -52,6 +52,21 @@ export default class Moebooru implements ScrapeEngine {
               break;
           }
         }
+        // If size element
+        else if (matches[1] == "Size") {
+          // Example string: 1600x2200
+          const res = matches[2]
+            .split("x") // Split on 'x' character
+            .map((v, _) => parseInt(v)) // Parse ints
+            .filter(Number); // Filter NaN. Also removes the number 0, but that's fine because the resolution can't be 0.
+
+          if (res.length == 2) {
+            // Kinda hacky, but this is currently the correct way to convert an array to a tuple.
+            post.resolution = [res[0], res[1]];
+          } else {
+            this.log("Couldn't parse post resolution.");
+          }
+        }
       }
     }
 
@@ -96,6 +111,15 @@ export default class Moebooru implements ScrapeEngine {
       post.tags.push(tag);
     }
 
+    // Set notes
+    const noteBoxSizeEl = document.getElementById("image") as HTMLImageElement;
+    if (!noteBoxSizeEl) {
+      this.log("noteBoxSizeEl is undefined.");
+    } else {
+      const boxSize: [number, number] = [noteBoxSizeEl.width, noteBoxSizeEl.height];
+      post.notes = createNotesFromMoebooruBoxes(document, boxSize);
+    }
+
     result.tryAddPost(post);
 
     return result;

src/engines/SankakuComplex.test.ts

@@ -0,0 +1,37 @@
+import { scrapeUrl } from "../test/engine";
+
+test("get chan.sankakucomplex.com post 19614481 with notes", async () => {
+  const pageUrl = "https://chan.sankakucomplex.com/post/show/19614481";
+  const res = await scrapeUrl(pageUrl);
+  expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("bangs");
+  expect(res.posts[0]).toHaveTag("bare_shoulders");
+  expect(res.posts[0]).toHaveTag("solo");
+  expect(res.posts[0]).toHaveTag("translated");
+  expect(res.posts[0]).toHaveTag("text");
+  expect(res.posts[0]).toHaveTag("idolmaster");
+  expect(res.posts[0]).toHaveTag("mayuzumi_fuyuko");
+  expect(res.posts[0]).toHaveNote("What are you looking at...");
+  expect(res.posts[0]).toHaveNote("What...?");
+  expect(res.posts[0].pageUrl).toBe(pageUrl);
+  expect(res.posts[0].contentUrl).toBe(
+    "https://s.sankakucomplex.com/data/50/7e/507e986a62cc5b866d92ad491dcfb1a4.jpg?e=1637625160&m=e0n_DCg-3fnmrhGzpI0Ftg"
+  );
+});
+
+test("get chan.sankakucomplex.com post 29354061 with notes", async () => {
+  // Kinda sketchy
+  const pageUrl = "https://chan.sankakucomplex.com/post/show/29354061";
+  const res = await scrapeUrl(pageUrl);
+  expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("bangs");
+  expect(res.posts[0]).toHaveTag("braid");
+  expect(res.posts[0]).toHaveTag("sankuro_(agoitei)");
+  expect(res.posts[0]).toHaveTag("translated");
+  expect(res.posts[0]).toHaveTag("note_translated");
+  expect(res.posts[0]).toHaveNote("No.  \nI already told you...");
+  expect(res.posts[0].pageUrl).toBe(pageUrl);
+  expect(res.posts[0].contentUrl).toBe(
+    "https://s.sankakucomplex.com/data/c8/37/c8376f02608548365d394a658e85cb3f.jpg?e=1637625020&m=A9xq-zht_FoVfoCiSRA-GA"
+  );
+});

src/engines/SankakuComplex.ts

@@ -1,8 +1,8 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
+import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapedNote } from "../ScrapeEngine";
 import { TagCategory } from "../BooruTypes";
-import { guessContentType } from "../Utility";
+import { createNotesFromMoebooruBoxes, guessContentType } from "../Utility";
 
-export default class SankakuComplex implements ScrapeEngine {
+export default class SankakuComplex extends ScrapeEngineBase {
   name = "sankakucomplex";
 
   canImport(url: Location): boolean {
@@ -81,6 +81,15 @@ export default class SankakuComplex implements ScrapeEngine {
       }
     }
 
+    // Set notes
+    const postContentEl = document.querySelector("#image-link > img") as HTMLImageElement;
+    if (!postContentEl) {
+      this.log("postContentEl is undefined.");
+    } else {
+      const boxSize: [number, number] = [postContentEl.width, postContentEl.height];
+      post.notes = createNotesFromMoebooruBoxes(document, boxSize);
+    }
+
     result.tryAddPost(post);
 
     return result;

src/test/jest.d.ts

@@ -0,0 +1,10 @@
+import "jest-extended";
+
+declare global {
+  namespace jest {
+    interface Matchers<R> {
+      toHaveTag(tagName: string): CustomMatcherResult;
+      toHaveNote(noteText: string): CustomMatcherResult;
+    }
+  }
+}

src/test/jest.setup.ts

@@ -12,14 +12,6 @@ Object.defineProperty(global.Element.prototype, "innerText", {
   },
 });
 
-declare global {
-  namespace jest {
-    interface Matchers<R> {
-      toHaveTag(tagName: string): CustomMatcherResult;
-    }
-  }
-}
-
 // Add our own functions
 expect.extend({
   toHaveTag(received: any, tagName: string): jest.CustomMatcherResult {
@@ -31,4 +23,13 @@ expect.extend({
     }
     return { pass, message };
   },
+  toHaveNote(received: any, noteText: string): jest.CustomMatcherResult {
+    let pass: boolean = false;
+    const message = () => (pass ? "" : `post does not have a note with '${noteText}'.`);
+
+    if (received instanceof ScrapedPost) {
+      pass = received.notes.find((x) => x.text == noteText) != undefined;
+    }
+    return { pass, message };
+  },
 });

tsconfig.test.json

@@ -0,0 +1,23 @@
+{
+  "compilerOptions": {
+    "lib": [
+      "es2019",
+      "dom",
+      "dom.iterable"
+    ],
+    "outDir": "./dist/",
+    "sourceMap": true,
+    "strict": true,
+    "module": "es2015",
+    "moduleResolution": "node",
+    "target": "es2015",
+    "skipLibCheck": true,
+    "esModuleInterop": true,
+    "removeComments": true,
+    "declaration": true
+  },
+  "include": [
+    "src/index.ts",
+    "src/test/jest.d.ts"
+  ]
+}