AnimePictures parse resolution + more tests

src/Utility.ts

@@ -179,3 +179,18 @@ export function createNotesFromMoebooruBoxes(document: Document, boxSize: [numbe
   }
   return notes;
 }
+
+export function parseResolutionString(str: string): [number, number] | undefined {
+  // Example string: 1600x2200
+  const res = str
+    .split("x") // Split on 'x' character
+    .map((v, _) => parseInt(v)) // Parse ints
+    .filter(Number); // Filter NaN. Also removes the number 0, but that's fine because the resolution can't be 0.
+
+  if (res.length == 2) {
+    // Kinda hacky, but this is currently the correct way to convert an array to a tuple.
+    return [res[0], res[1]];
+  } else {
+    console.log(`[parseResolutionString] Couldn't parse resolution from '${str}'.`);
+  }
+}

src/engines/AnimePictures.test.ts

@@ -4,6 +4,14 @@ test("get anime-pictures.net post 729833", async () => {
   const pageUrl = "https://anime-pictures.net/pictures/view_post/729833?lang=en";
   const res = await scrapeUrl(pageUrl);
   expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("arknights", "copyright");
+  expect(res.posts[0]).toHaveTag("ch'en_(arknights)", "character");
+  expect(res.posts[0]).toHaveTag("ttk_(kirinottk)", "artist");
+  expect(res.posts[0]).toHaveTag("single");
+  expect(res.posts[0]).toHaveTag("holding");
+  expect(res.posts[0]).toHaveTag("gloves");
+  expect(res.posts[0]).toHaveTag("jacket");
+  expect(res.posts[0]).toHaveResolution([4000, 6865]);
   expect(res.posts[0].pageUrl).toBe(pageUrl);
   expect(res.posts[0].contentUrl).toBe(
     "https://anime-pictures.net/pictures/download_image/729833-4000x6865-arknights-ch%26%2339%3Ben+%28arknights%29-ttk+%28kirinottk%29-single-long+hair-tall+image.jpeg"

src/engines/AnimePictures.ts

@@ -1,6 +1,6 @@
 import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
 import { TagCategory } from "../BooruTypes";
-import { guessContentType } from "../Utility";
+import { guessContentType, parseResolutionString } from "../Utility";
 
 export default class AnimePictures implements ScrapeEngine {
   name = "animepictures";
@@ -20,9 +20,6 @@ export default class AnimePictures implements ScrapeEngine {
 
     if (originalImageElements.length > 0) {
       post.contentUrl = (originalImageElements[0] as HTMLAnchorElement).href;
-    } else {
-      // No point in continuing when we don't have an image.
-      return result;
     }
 
     // Set content type
@@ -31,8 +28,17 @@ export default class AnimePictures implements ScrapeEngine {
     // Set rating
     post.rating = "safe"; // Probably
 
+    // Set resolution
+    const postContentLinkEls = Array.from(
+      document.querySelectorAll<HTMLAnchorElement>("#content > div > .post_content > a")
+    );
+    const resEl = postContentLinkEls.find((x) => x.href.indexOf("res_x") != -1);
+    if (resEl) {
+      post.resolution = parseResolutionString(resEl.innerText);
+    }
+
     // Set tags
-    const tagElements = Array.from(document.querySelectorAll("ul.tags > li > a")).map((x) => x as HTMLAnchorElement);
+    const tagElements = Array.from(document.querySelectorAll<HTMLAnchorElement>("ul.tags > li > a"));
     for (const tagElement of tagElements) {
       let tagName = tagElement.innerText;
 

src/engines/Danbooru.test.ts

@@ -4,11 +4,15 @@ test("get danbooru.donmai.us post 3555872", async () => {
   const pageUrl = "https://danbooru.donmai.us/posts/3555872";
   const res = await scrapeUrl(pageUrl);
   expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("omachi_(slabco)", "artist");
+  expect(res.posts[0]).toHaveTag("dr_pepper", "copyright");
+  expect(res.posts[0]).toHaveTag("flint_(girls_und_panzer)", "character");
   expect(res.posts[0]).toHaveTag("bangs");
   expect(res.posts[0]).toHaveTag("shirt");
   expect(res.posts[0]).toHaveTag("skirt");
   expect(res.posts[0]).toHaveTag("t-shirt");
   expect(res.posts[0]).toHaveTag("long_hair");
+  expect(res.posts[0]).toHaveResolution([1142, 1600]);
   expect(res.posts[0].pageUrl).toBe(pageUrl);
   expect(res.posts[0].contentUrl).toBe(
     "https://cdn.donmai.us/original/90/33/__flint_girls_und_panzer_and_1_more_drawn_by_omachi_slabco__90332299216db81b8793084e4cb7c15d.jpg"
@@ -19,6 +23,9 @@ test("get danbooru.donmai.us post 4924490", async () => {
   const pageUrl = "https://danbooru.donmai.us/posts/4924490";
   const res = await scrapeUrl(pageUrl);
   expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("yuureidoushi_(yuurei6214)", "artist");
+  expect(res.posts[0]).toHaveTag("comic");
+  expect(res.posts[0]).toHaveResolution([1000, 2062]);
   expect(res.posts[0].pageUrl).toBe(pageUrl);
   expect(res.posts[0].contentUrl).toBe(
     "https://cdn.donmai.us/original/c5/fb/__yuuki_and_tamaki_princess_connect_drawn_by_yuureidoushi_yuurei6214__c5fbc62758d0f1d8c07aac524282434e.png"

src/engines/Gelbooru.test.ts

@@ -4,6 +4,14 @@ test("get safebooru.org post 2871532", async () => {
   const pageUrl = "https://safebooru.org/index.php?page=post&s=view&id=2871532";
   const res = await scrapeUrl(pageUrl);
   expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("omachi_(slabco)", "artist");
+  expect(res.posts[0]).toHaveTag("flint_(girls_und_panzer)", "character");
+  expect(res.posts[0]).toHaveTag("bangs");
+  expect(res.posts[0]).toHaveTag("shirt");
+  expect(res.posts[0]).toHaveTag("skirt");
+  expect(res.posts[0]).toHaveTag("t-shirt");
+  expect(res.posts[0]).toHaveTag("long_hair");
+  expect(res.posts[0]).toHaveResolution([1142, 1600]);
   expect(res.posts[0].pageUrl).toBe(pageUrl);
   expect(res.posts[0].contentUrl).toBe(
     "https://safebooru.org//images/2757/5f19c7744db918c3d4c4c6035bc65dfa9e18866f.jpg"
@@ -14,6 +22,9 @@ test("get gelbooru.com post 4925055", async () => {
   const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=4925055&tags=kimetsu_no_yaiba";
   const res = await scrapeUrl(pageUrl);
   expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("maosen", "artist");
+  expect(res.posts[0]).toHaveTag("butterfly");
+  expect(res.posts[0]).toHaveResolution([720, 1103]);
   expect(res.posts[0].pageUrl).toBe(pageUrl);
   expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
 });
@@ -33,6 +44,7 @@ test("get gelbooru.com post 6225495 with notes", async () => {
   expect(res.posts[0]).toHaveNote("I'm fine, more or less.");
   expect(res.posts[0]).toHaveNote("But no need to worry!");
   expect(res.posts[0]).toHaveNote("You're always the best just by being yourself-!!!");
+  expect(res.posts[0]).toHaveResolution([1600, 2200]);
   expect(res.posts[0].pageUrl).toBe(pageUrl);
   expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/af/1d/af1dd661fab6714dffc29cb38b6b2269.png");
 });
@@ -40,6 +52,12 @@ test("get gelbooru.com post 6225495 with notes", async () => {
 test("get rule34.xxx post 2740994", async () => {
   const pageUrl = "https://rule34.xxx/index.php?page=post&s=view&id=2740994";
   const res = await scrapeUrl(pageUrl);
+  expect(res.posts[0]).toHaveTag("fire_emblem", "copyright");
+  expect(res.posts[0]).toHaveTag("nintendo", "copyright");
+  expect(res.posts[0]).toHaveTag("lucina", "character");
+  expect(res.posts[0]).toHaveTag("blue_hair");
+  expect(res.posts[0]).toHaveTag("solo");
+  expect(res.posts[0]).toHaveResolution([600, 800]);
   expect(res.posts.length).toBe(1);
   expect(res.posts[0].pageUrl).toBe(pageUrl);
   expect(res.posts[0].contentUrl).toContain(".rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg");

src/engines/Gelbooru.ts

@@ -1,7 +1,7 @@
 import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
 import { TagCategory } from "../BooruTypes";
 import { CategoryMap } from "./Common";
-import { createNoteFromDanbooruArticle, guessContentType } from "../Utility";
+import { createNoteFromDanbooruArticle, guessContentType, parseResolutionString } from "../Utility";
 
 // Small hack to avoid screwing with semver
 enum Version {
@@ -103,18 +103,7 @@ export default class Gelbooru extends ScrapeEngineBase {
           }
           // If size element
           else if (matches[1] == "Size") {
-            // Example string: 1600x2200
+            post.resolution = parseResolutionString(matches[2]);
-            const res = matches[2]
-              .split("x") // Split on 'x' character
-              .map((v, _) => parseInt(v)) // Parse ints
-              .filter(Number); // Filter NaN. Also removes the number 0, but that's fine because the resolution can't be 0.
-
-            if (res.length == 2) {
-              // Kinda hacky, but this is currently the correct way to convert an array to a tuple.
-              post.resolution = [res[0], res[1]];
-            } else {
-              this.log("Couldn't parse post resolution.");
-            }
           }
         }
       }

src/engines/Moebooru.test.ts

@@ -4,6 +4,10 @@ test("get yande.re post 438510", async () => {
   const pageUrl = "https://yande.re/post/show/438510";
   const res = await scrapeUrl(pageUrl);
   expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("the_legend_of_zelda", "copyright");
+  expect(res.posts[0]).toHaveTag("link", "character");
+  expect(res.posts[0]).toHaveTag("landscape");
+  expect(res.posts[0]).toHaveResolution([3000, 1688]);
   expect(res.posts[0].pageUrl).toBe(pageUrl);
   expect(res.posts[0].contentUrl).toBe(
     "https://files.yande.re/image/49b79bc372716a579d8525dcaccb56e2/yande.re%20438510%20landscape%20link%20nintendo%20princess_zelda%20sword%20the_legend_of_zelda%20the_legend_of_zelda%3A_breath_of_the_wild.jpg"
@@ -16,6 +20,7 @@ test("get yande.re post 684478 with notes", async () => {
   expect(res.posts.length).toBe(1);
   expect(res.posts[0]).toHaveTag("gym_uniform");
   expect(res.posts[0]).toHaveNote("post #671879");
+  expect(res.posts[0]).toHaveResolution([2480, 3508]);
   expect(res.posts[0].pageUrl).toBe(pageUrl);
   expect(res.posts[0].contentUrl).toBe(
     "https://files.yande.re/image/8bdf7c54b0a459acee080f0aeb2a259c/yande.re%20684478%20gym_uniform%20pal_lett.jpg"
@@ -26,6 +31,8 @@ test("get konachan.com post 289968", async () => {
   const pageUrl = "https://konachan.com/post/show/289968/ahri_-league_of_legends-akali-animal_ears-blonde_h";
   const res = await scrapeUrl(pageUrl);
   expect(res.posts.length).toBe(1);
+  expect(res.posts[0]).toHaveTag("league_of_legends", "copyright");
+  expect(res.posts[0]).toHaveResolution([1200, 783]);
   expect(res.posts[0].pageUrl).toBe(pageUrl);
   expect(res.posts[0].contentUrl).toBe(
     "https://konachan.com/image/e40b38fb704434e4da3eae41e26bca39/Konachan.com%20-%20289968%20akali%20blue_eyes%20breasts%20choker%20cleavage%20evelynn%20foxgirl%20gloves%20group%20hat%20kai%27sa%20long_hair%20microphone%20ponytail%20saphirya%20short_hair%20signed%20watermark.jpg"
@@ -41,6 +48,7 @@ test("get konachan.com post 327454 with notes", async () => {
   expect(res.posts[0]).toHaveTag("kexue");
   expect(res.posts[0]).toHaveTag("water");
   expect(res.posts[0]).toHaveNote("Under construction.  \nBe careful of your safety");
+  expect(res.posts[0]).toHaveResolution([5629, 2505]);
   expect(res.posts[0].pageUrl).toBe(pageUrl);
   expect(res.posts[0].contentUrl).toBe(
     "https://konachan.com/image/94a13c8a1fcb9c0f6c9a6c6c5c316259/Konachan.com%20-%20327454%20apron%20brown_hair%20building%20car%20gloves%20kexue%20long_hair%20original%20rain%20see_through%20short_hair%20twintails%20uniform%20water.jpg"
@@ -58,6 +66,7 @@ test("get konachan.com post 285782 with notes", async () => {
   expect(res.posts[0]).toHaveNote("I'M AWAKE!!");
   expect(res.posts[0]).toHaveNote("Poi!");
   expect(res.posts[0]).toHaveNote("Haaah!");
+  expect(res.posts[0]).toHaveResolution([1000, 830]);
   expect(res.posts[0].pageUrl).toBe(pageUrl);
   expect(res.posts[0].contentUrl).toBe(
     "https://konachan.com/image/24f5974ad8ed725292ccb3f682181bc1/Konachan.com%20-%20285782%20anthropomorphism%20bed%20black_eyes%20blonde_hair%20cropped%20fang%20kantai_collection%20long_hair%20momoniku%20pajamas%20white%20yuudachi_%28kancolle%29.jpg"

src/engines/Moebooru.ts

@@ -1,6 +1,6 @@
 import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapedNote } from "../ScrapeEngine";
 import { TagCategory } from "../BooruTypes";
-import { createNotesFromMoebooruBoxes, guessContentType } from "../Utility";
+import { createNotesFromMoebooruBoxes, guessContentType, parseResolutionString } from "../Utility";
 
 export default class Moebooru extends ScrapeEngineBase {
   name = "moebooru";
@@ -54,18 +54,7 @@ export default class Moebooru extends ScrapeEngineBase {
         }
         // If size element
         else if (matches[1] == "Size") {
-          // Example string: 1600x2200
+          post.resolution = parseResolutionString(matches[2]);
-          const res = matches[2]
-            .split("x") // Split on 'x' character
-            .map((v, _) => parseInt(v)) // Parse ints
-            .filter(Number); // Filter NaN. Also removes the number 0, but that's fine because the resolution can't be 0.
-
-          if (res.length == 2) {
-            // Kinda hacky, but this is currently the correct way to convert an array to a tuple.
-            post.resolution = [res[0], res[1]];
-          } else {
-            this.log("Couldn't parse post resolution.");
-          }
         }
       }
     }

src/test/jest.d.ts

@@ -1,10 +1,13 @@
 import "jest-extended";
+import { TagCategory } from "../BooruTypes";
 
 declare global {
   namespace jest {
     interface Matchers<R> {
-      toHaveTag(tagName: string): CustomMatcherResult;
+      // The `= undefined` part is incorrect according to VSCode, but it is actually required.
+      toHaveTag(tagName: string, tagCategory: TagCategory | undefined = undefined): CustomMatcherResult;
       toHaveNote(noteText: string): CustomMatcherResult;
+      toHaveResolution(resolution: [number, number]): CustomMatcherResult;
     }
   }
 }

src/test/jest.setup.ts

@@ -1,4 +1,5 @@
 import { JSDOM } from "jsdom";
+import { TagCategory } from "../BooruTypes";
 import { ScrapedPost } from "../ScrapeEngine";
 
 // Expose JSDOM Element constructor
@@ -14,22 +15,42 @@ Object.defineProperty(global.Element.prototype, "innerText", {
 
 // Add our own functions
 expect.extend({
-  toHaveTag(received: any, tagName: string): jest.CustomMatcherResult {
+  toHaveTag(
+    received: any,
+    tagName: string,
+    tagCategory: TagCategory | undefined = undefined
+  ): jest.CustomMatcherResult {
     let pass: boolean = false;
-    const message = () => (pass ? "" : `post does not have the '${tagName}' tag.`);
+    const message = () => (pass ? "" : `Post does not have the '${tagName}' tag, or the category is incorrect.`);
 
     if (received instanceof ScrapedPost) {
-      pass = received.tags.find((x) => x.name == tagName) != undefined;
+      const found = received.tags.find((x) => x.name == tagName);
+      if (found) {
+        if (tagCategory) {
+          pass = found.category == tagCategory;
+        } else {
+          pass = true;
+        }
+      }
     }
     return { pass, message };
   },
   toHaveNote(received: any, noteText: string): jest.CustomMatcherResult {
     let pass: boolean = false;
-    const message = () => (pass ? "" : `post does not have a note with '${noteText}'.`);
+    const message = () => (pass ? "" : `Post does not have a note with '${noteText}'.`);
 
     if (received instanceof ScrapedPost) {
       pass = received.notes.find((x) => x.text == noteText) != undefined;
     }
     return { pass, message };
   },
+  toHaveResolution(received: any, resolution: [number, number]): jest.CustomMatcherResult {
+    let pass: boolean = false;
+    const message = () => (pass ? "" : `Expected: ${resolution}\nReceived: ${received?.resolution}`);
+
+    if (received instanceof ScrapedPost && received.resolution?.length == 2) {
+      pass = received.resolution[0] == resolution[0] && received.resolution[1] == resolution[1];
+    }
+    return { pass, message };
+  },
 });