Add support for multiple source fields + add tests

src/ScrapeEngine.ts

@@ -10,7 +10,7 @@ export class ScrapedPost {
   rating: SafetyRating = "safe";
   tags: ScrapedTag[] = [];
   notes: ScrapedNote[] = [];
-  source: string | undefined;
+  sources: string[] = [];
   referrer: string | undefined;
 }
 

src/Utility.ts

@@ -120,6 +120,7 @@ export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement
     return undefined;
   }
 
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
   const data = (<any>el).dataset as DOMStringMap;
 
   if (!data) {
@@ -194,8 +195,8 @@ export function parseResolutionString(str: string | undefined | null): [number,
   // Example string: 1600x2200
   const res = str
     .split("x") // Split on 'x' character
-    .map((v, _) => v.replace(/\D/g, "")) // Remove all non-digits
-    .map((v, _) => parseInt(v)) // Parse ints
+    .map((v) => v.replace(/\D/g, "")) // Remove all non-digits
+    .map((v) => parseInt(v)) // Parse ints
     .filter(Number); // Filter NaN. Also removes the number 0, but that's fine because the resolution can't be 0.
 
   if (res.length == 2) {

src/engines/Danbooru.test.ts

@@ -13,6 +13,7 @@ test("get danbooru.donmai.us post 3555872", async () => {
   expect(res.posts[0]).toHaveTag("t-shirt");
   expect(res.posts[0]).toHaveTag("long_hair");
   expect(res.posts[0]).toHaveResolution([1142, 1600]);
+  expect(res.posts[0]).toHaveSource("https://twitter.com/omatiosake/status/1124306601584672768");
   expect(res.posts[0].pageUrl).toBe(pageUrl);
   expect(res.posts[0].contentUrl).toBe(
     "https://cdn.donmai.us/original/90/33/__flint_girls_und_panzer_and_1_more_drawn_by_omachi_slabco__90332299216db81b8793084e4cb7c15d.jpg"

src/engines/Danbooru.ts

@@ -101,7 +101,7 @@ export default class Danbooru extends ScrapeEngineBase {
     if (sourceEl && sourceEl.children.length > 0) {
       const sourceLink = sourceEl.children[0] as HTMLAnchorElement;
       if (sourceLink) {
-        post.source = sourceLink.href;
+        post.sources = [sourceLink.href];
       }
     }
 

src/engines/Gelbooru.test.ts

@@ -25,6 +25,7 @@ test("get gelbooru.com post 4925055", async () => {
   expect(res.posts[0]).toHaveTag("maosen", "artist");
   expect(res.posts[0]).toHaveTag("butterfly");
   expect(res.posts[0]).toHaveResolution([720, 1103]);
+  expect(res.posts[0]).toHaveSource("https://www.pixiv.net/artworks/76853958");
   expect(res.posts[0].pageUrl).toBe(pageUrl);
   expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
 });
@@ -45,6 +46,7 @@ test("get gelbooru.com post 6225495 with notes", async () => {
   expect(res.posts[0]).toHaveNote("But no need to worry!");
   expect(res.posts[0]).toHaveNote("You're always the best just by being yourself-!!!");
   expect(res.posts[0]).toHaveResolution([1600, 2200]);
+  expect(res.posts[0]).toHaveSource("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=88721816");
   expect(res.posts[0].pageUrl).toBe(pageUrl);
   expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/af/1d/af1dd661fab6714dffc29cb38b6b2269.png");
 });

src/engines/Gelbooru.ts

@@ -80,9 +80,9 @@ export default class Gelbooru extends ScrapeEngineBase {
           // If Source element
           if (matches[1] == "Source") {
             if (version == Version.v020) {
-              post.source = matches[2];
+              post.sources = [matches[2]];
             } else {
-              post.source = (el.children[0] as HTMLAnchorElement).href;
+              post.sources = [(el.children[0] as HTMLAnchorElement).href];
             }
           }
           // If Rating element

src/engines/Moebooru.test.ts

@@ -21,6 +21,7 @@ test("get yande.re post 684478 with notes", async () => {
   expect(res.posts[0]).toHaveTag("gym_uniform");
   expect(res.posts[0]).toHaveNote("post #671879");
   expect(res.posts[0]).toHaveResolution([2480, 3508]);
+  expect(res.posts[0]).toHaveSource("https://www.pixiv.net/artworks/84228323");
   expect(res.posts[0].pageUrl).toBe(pageUrl);
   expect(res.posts[0].contentUrl).toBe(
     "https://files.yande.re/image/8bdf7c54b0a459acee080f0aeb2a259c/yande.re%20684478%20gym_uniform%20pal_lett.jpg"
@@ -33,6 +34,7 @@ test("get konachan.com post 289968", async () => {
   expect(res.posts.length).toBe(1);
   expect(res.posts[0]).toHaveTag("league_of_legends", "copyright");
   expect(res.posts[0]).toHaveResolution([1200, 783]);
+  expect(res.posts[0]).toHaveSource("https://www.pixiv.net/member_illust.php?mode=medium&illust_id=72418059");
   expect(res.posts[0].pageUrl).toBe(pageUrl);
   expect(res.posts[0].contentUrl).toBe(
     "https://konachan.com/image/e40b38fb704434e4da3eae41e26bca39/Konachan.com%20-%20289968%20akali%20blue_eyes%20breasts%20choker%20cleavage%20evelynn%20foxgirl%20gloves%20group%20hat%20kai%27sa%20long_hair%20microphone%20ponytail%20saphirya%20short_hair%20signed%20watermark.jpg"
@@ -49,6 +51,7 @@ test("get konachan.com post 327454 with notes", async () => {
   expect(res.posts[0]).toHaveTag("water");
   expect(res.posts[0]).toHaveNote("Under construction.  \nBe careful of your safety");
   expect(res.posts[0]).toHaveResolution([5629, 2505]);
+  expect(res.posts[0]).toHaveSource("https://www.pixiv.net/en/artworks/86644963");
   expect(res.posts[0].pageUrl).toBe(pageUrl);
   expect(res.posts[0].contentUrl).toBe(
     "https://konachan.com/image/94a13c8a1fcb9c0f6c9a6c6c5c316259/Konachan.com%20-%20327454%20apron%20brown_hair%20building%20car%20gloves%20kexue%20long_hair%20original%20rain%20see_through%20short_hair%20twintails%20uniform%20water.jpg"
@@ -71,4 +74,5 @@ test("get konachan.com post 285782 with notes", async () => {
   expect(res.posts[0].contentUrl).toBe(
     "https://konachan.com/image/24f5974ad8ed725292ccb3f682181bc1/Konachan.com%20-%20285782%20anthropomorphism%20bed%20black_eyes%20blonde_hair%20cropped%20fang%20kantai_collection%20long_hair%20momoniku%20pajamas%20white%20yuudachi_%28kancolle%29.jpg"
   );
+  expect(res.posts[0]).toHaveSource("https://www.pixiv.net/member_illust.php?mode=medium&illust_id=75747354");
 });

src/engines/Moebooru.ts

@@ -35,7 +35,7 @@ export default class Moebooru extends ScrapeEngineBase {
       if (matches && matches.length == 3) {
         // If Source element
         if (matches[1] == "Source") {
-          post.source = (el.children[0] as HTMLAnchorElement).href;
+          post.sources = [(el.children[0] as HTMLAnchorElement).href];
         }
         // If Rating element
         else if (matches[1] == "Rating") {

src/engines/Shimmie2.test.ts

@@ -5,6 +5,9 @@ test("get rule34.paheal.net post 2505982", async () => {
   const res = await scrapeUrl(pageUrl);
   expect(res.posts.length).toBe(1);
   expect(res.posts[0].pageUrl).toBe(pageUrl);
+  expect(res.posts[0]).toHaveSource(
+    "http://law-zilla.tumblr.com/post/171249285285/finished-subdraw-37-kaisa-from-league-of-legends"
+  );
   expect(res.posts[0].contentUrl).toContain(
     ".paheal.net/_images/15427ab5ed2cddbeef947e30f5c9cab6/2505982%20-%20Kai%27Sa%20League_of_Legends%20lawzilla.png"
   );

src/engines/Shimmie2.ts

@@ -44,7 +44,7 @@ export default class Shimmie2 extends ScrapeEngineBase {
     // Set source
     const sourceEl = document.querySelector(".image_info > tbody > tr > td > div > a") as HTMLAnchorElement;
     if (sourceEl) {
-      post.source = sourceEl.href;
+      post.sources = [sourceEl.href];
     }
 
     result.tryAddPost(post);

src/engines/e621.test.ts

@@ -9,6 +9,7 @@ test("get e621.net post 380315", async () => {
   expect(res.posts[0]).toHaveTag("ambiguous_gender");
   expect(res.posts[0]).toHaveTag("low_res", "meta");
   expect(res.posts[0]).toHaveResolution([500, 500]);
+  expect(res.posts[0]).toHaveSource("https://inkinesss.deviantart.com/art/Nasus-388347996");
   expect(res.posts[0].pageUrl).toBe(pageUrl);
   expect(res.posts[0].contentUrl).toBe("https://static1.e621.net/data/92/e7/92e71bd73ab219d80516c4d9adad3db5.jpg");
 });

src/engines/e621.ts

@@ -82,10 +82,7 @@ export default class e621 extends ScrapeEngineBase {
 
     // Set source
     const sourceEls = Array.from(document.querySelectorAll(".source-link > a")).map((x) => x as HTMLAnchorElement);
-    // We currently only support one source.
-    if (sourceEls.length > 0) {
-      post.source = sourceEls[0].href;
-    }
+    post.sources = sourceEls.map((x) => x.href);
 
     result.tryAddPost(post);
 

src/test/jest.d.ts

@@ -5,9 +5,10 @@ declare global {
   namespace jest {
     interface Matchers<R> {
       // The `= undefined` part is incorrect according to VSCode, but it is actually required.
-      toHaveTag(tagName: string, tagCategory: TagCategory | undefined = undefined): R;
-      toHaveNote(noteText: string): R;
-      toHaveResolution(resolution: [number, number]): R;
+      toHaveTag(tagName: string, tagCategory: TagCategory | undefined = undefined): CustomMatcherResult;
+      toHaveNote(noteText: string): CustomMatcherResult;
+      toHaveResolution(resolution: [number, number]): CustomMatcherResult;
+      toHaveSource(source: string): CustomMatcherResult;
     }
   }
 }

src/test/jest.setup.ts

@@ -52,4 +52,13 @@ expect.extend({
     }
     return { pass, message };
   },
+  toHaveSource(received: any, source: string): jest.CustomMatcherResult {
+    let pass = false;
+    const message = () => (pass ? "" : `Post does not have source '${source}'.`);
+
+    if (received instanceof ScrapedPost) {
+      pass = received.sources.find((x) => x == source) != undefined;
+    }
+    return { pass, message };
+  },
 });