Update tests

.gitignore

@@ -1,3 +1,4 @@
 .vscode/
 node_modules/
 dist/
+coverage/

jest.preset.js

@@ -0,0 +1,4 @@
+const ts_preset = require("ts-jest/jest-preset");
+const puppeteer_preset = require("jest-puppeteer/jest-preset");
+
+module.exports = Object.assign(ts_preset, puppeteer_preset);

package.json

@@ -13,15 +13,21 @@
   "author": "neobooru",
   "license": "MIT",
   "devDependencies": {
-    "@types/jest": "^24.9.1",
+    "@types/expect-puppeteer": "^4.4.7",
+    "@types/jest": "^27.0.3",
+    "@types/jest-environment-puppeteer": "^4.4.1",
     "@types/jsdom": "^12.2.4",
+    "@types/puppeteer": "^5.4.4",
     "@types/turndown": "^5.0.1",
-    "jest": "^24.9.0",
+    "jest": "^27.3.1",
+    "jest-puppeteer": "^6.0.0",
     "jsdom": "^15.2.1",
-    "ts-jest": "^24.3.0",
+    "puppeteer": "^11.0.0",
-    "typescript": "^3.8.3"
+    "ts-jest": "^27.0.7",
+    "typescript": "^4.5.2"
   },
   "jest": {
+    "preset": "./jest.preset.js",
     "transform": {
       "^.+\\.tsx?$": "ts-jest"
     },

src/NeoScraper.test.ts

@@ -1,10 +1,27 @@
-import { JSDOM } from "jsdom";
+import { ConstructorOptions, JSDOM } from "jsdom";
 import NeoScraper from "./NeoScraper";
+import { ScrapeResults } from "./ScrapeEngine";
 
 async function scrapeUrl(url: string) {
-  const scraper = new NeoScraper();
   const dom = await JSDOM.fromURL(url);
-  return scraper.scrapeDocument(dom.window.document);
+  return scrapeDom(dom.window.document);
+}
+
+async function scrapeEvalUrl(url: string, waitForSelector: string) {
+  await page.goto(url);
+  await page.waitForSelector(waitForSelector, { visible: true });
+  const document = await page.content();
+  return scrapeHtml(url, document);
+}
+
+async function scrapeHtml(url: string, html: string) {
+  const dom = new JSDOM(html, <ConstructorOptions>{ url });
+  return scrapeDom(dom.window.document);
+}
+
+async function scrapeDom(document: Document) {
+  const scraper = new NeoScraper();
+  return scraper.scrapeDocument(document);
 }
 
 beforeAll(() => {
@@ -37,11 +54,24 @@ test("all default scrapers have unique names", () => {
   }
 });
 
-// Danbooru
 // All links are SFW unless stated otherwise
 
+// Anime pictures
+describe("anime-pictures.net", () => {
+  test("get anime-pictures.net post 729833", async () => {
+    const pageUrl = "https://anime-pictures.net/pictures/view_post/729833?lang=en";
+    const res = await scrapeUrl(pageUrl);
+    expect(res.posts.length).toBe(1);
+    expect(res.posts[0].pageUrl).toBe(pageUrl);
+    expect(res.posts[0].contentUrl).toBe(
+      "https://anime-pictures.net/pictures/download_image/729833-4000x6865-arknights-ch%26%2339%3Ben+%28arknights%29-ttk+%28kirinottk%29-single-long+hair-tall+image.jpeg"
+    );
+  });
+});
+
+describe("danbooru", () => {
   test("get danbooru.donmai.us post 3555872", async () => {
-  var pageUrl = "https://danbooru.donmai.us/posts/3555872";
+    const pageUrl = "https://danbooru.donmai.us/posts/3555872";
     const res = await scrapeUrl(pageUrl);
     expect(res.posts.length).toBe(1);
     expect(res.posts[0].pageUrl).toBe(pageUrl);
@@ -51,7 +81,7 @@ test("get danbooru.donmai.us post 3555872", async () => {
   });
 
   test("get danbooru.donmai.us post 4924490", async () => {
-  var pageUrl = "https://danbooru.donmai.us/posts/4924490";
+    const pageUrl = "https://danbooru.donmai.us/posts/4924490";
     const res = await scrapeUrl(pageUrl);
     expect(res.posts.length).toBe(1);
     expect(res.posts[0].pageUrl).toBe(pageUrl);
@@ -61,11 +91,11 @@ test("get danbooru.donmai.us post 4924490", async () => {
     expect(res.posts[0].notes).toHaveLength(17);
     expect(res.posts[0].notes[0].text).toBe("Nya~nya ♪");
   });
+});
 
-// Gelbooru
+describe("gelbooru", () => {
-
   test("get safebooru.org post 2871532", async () => {
-  var pageUrl = "https://safebooru.org/index.php?page=post&s=view&id=2871532";
+    const pageUrl = "https://safebooru.org/index.php?page=post&s=view&id=2871532";
     const res = await scrapeUrl(pageUrl);
     expect(res.posts.length).toBe(1);
     expect(res.posts[0].pageUrl).toBe(pageUrl);
@@ -75,7 +105,7 @@ test("get safebooru.org post 2871532", async () => {
   });
 
   test("get gelbooru.com post 4925055", async () => {
-  var pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=4925055&tags=kimetsu_no_yaiba";
+    const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=4925055&tags=kimetsu_no_yaiba";
     const res = await scrapeUrl(pageUrl);
     expect(res.posts.length).toBe(1);
     expect(res.posts[0].pageUrl).toBe(pageUrl);
@@ -83,17 +113,17 @@ test("get gelbooru.com post 4925055", async () => {
   });
 
   test("get rule34.xxx post 2740994", async () => {
-  var pageUrl = "https://rule34.xxx/index.php?page=post&s=view&id=2740994";
+    const pageUrl = "https://rule34.xxx/index.php?page=post&s=view&id=2740994";
     const res = await scrapeUrl(pageUrl);
     expect(res.posts.length).toBe(1);
     expect(res.posts[0].pageUrl).toBe(pageUrl);
     expect(res.posts[0].contentUrl).toContain(".rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg");
   });
+});
 
-// Moebooru
+describe("moebooru", () => {
-
   test("get yande.re post 438510", async () => {
-  var pageUrl = "https://yande.re/post/show/438510";
+    const pageUrl = "https://yande.re/post/show/438510";
     const res = await scrapeUrl(pageUrl);
     expect(res.posts.length).toBe(1);
     expect(res.posts[0].pageUrl).toBe(pageUrl);
@@ -103,7 +133,7 @@ test("get yande.re post 438510", async () => {
   });
 
   test("get konachan.com post 289968", async () => {
-  var pageUrl = "https://konachan.com/post/show/289968/ahri_-league_of_legends-akali-animal_ears-blonde_h";
+    const pageUrl = "https://konachan.com/post/show/289968/ahri_-league_of_legends-akali-animal_ears-blonde_h";
     const res = await scrapeUrl(pageUrl);
     expect(res.posts.length).toBe(1);
     expect(res.posts[0].pageUrl).toBe(pageUrl);
@@ -111,11 +141,29 @@ test("get konachan.com post 289968", async () => {
       "https://konachan.com/image/e40b38fb704434e4da3eae41e26bca39/Konachan.com%20-%20289968%20akali%20blue_eyes%20breasts%20choker%20cleavage%20evelynn%20foxgirl%20gloves%20group%20hat%20kai%27sa%20long_hair%20microphone%20ponytail%20saphirya%20short_hair%20signed%20watermark.jpg"
     );
   });
+});
 
-// Reddit
+// Commented out because it doesn't work.
+// describe("pixiv", () => {
+//   describe("get pixiv post 93964654", () => {
+//     const pageUrl = "https://www.pixiv.net/en/artworks/93964654";
+//     let res: ScrapeResults;
 
+//     beforeAll(async () => {
+//       res = await scrapeEvalUrl(pageUrl, ".gtm-expand-full-size-illust");
+//     });
+
+//     it("should grab original image url", async () => {
+//       expect(res.posts.length).toBe(1);
+//       expect(res.posts[0].pageUrl).toBe(pageUrl);
+//       expect(res.posts[0].contentUrl).toBe("https://i.pximg.net/img-original/img/2021/11/07/00/01/02/93964654_p0.jpg");
+//     });
+//   });
+// });
+
+describe("reddit", () => {
   test("get old.reddit.com post", async () => {
-  var pageUrl = "https://old.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/";
+    const pageUrl = "https://old.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/";
     const res = await scrapeUrl(pageUrl);
     expect(res.posts.length).toBe(1);
     expect(res.posts[0].pageUrl).toBe(pageUrl);
@@ -123,17 +171,17 @@ test("get old.reddit.com post", async () => {
   });
 
   //   test("get new.reddit.com post", async () => {
-//   var pageUrl = "https://new.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/";
+  //     const pageUrl = "https://new.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/";
   //     const res = await scrapeUrl(pageUrl);
   //     expect(res.posts.length).toBe(1);
   //     expect(res.posts[0].pageUrl).toBe(pageUrl);
   //     expect(res.posts[0].contentUrl).toBe("https://i.redd.it/ln8ahz17kpa31.jpg");
   //   });
+});
 
-// Shimmie2
+describe("shimmie2", () => {
-
   test("get rule34.paheal.net post 2505982", async () => {
-  var pageUrl = "https://rule34.paheal.net/post/view/2505982"; // Somewhat sketchy
+    const pageUrl = "https://rule34.paheal.net/post/view/2505982"; // Somewhat sketchy
     const res = await scrapeUrl(pageUrl);
     expect(res.posts.length).toBe(1);
     expect(res.posts[0].pageUrl).toBe(pageUrl);
@@ -141,15 +189,21 @@ test("get rule34.paheal.net post 2505982", async () => {
       ".paheal.net/_images/15427ab5ed2cddbeef947e30f5c9cab6/2505982%20-%20Kai%27Sa%20League_of_Legends%20lawzilla.png"
     );
   });
-
+});
-// Twitter
+
-
+describe("twitter", () => {
-test("get twitter.com post", async () => {
+  describe("get twitter.com post", () => {
-  pending("JSDOM can't parse this page");
+    const pageUrl = "https://twitter.com/Dishwasher1910/status/1171162651461746688/photo/1";
-  // TODO: JSDOM sucks
+    let res: ScrapeResults;
-  // var pageUrl = "https://twitter.com/Dishwasher1910/status/1171162651461746688/photo/1";
+
-  // const res = await scrapeUrl(pageUrl);
+    beforeAll(async () => {
-  // expect(res.posts.length).toBe(1);
+      res = await scrapeEvalUrl(pageUrl, "img[alt='Image']");
-  // expect(res.posts[0].pageUrl).toBe(pageUrl);
+    });
-  // expect(res.posts[0].contentUrl).toBe("https://pbs.twimg.com/media/EEDOOCdU0AAHX00?format=jpg&name=orig");
+
+    it("should grab original image url", async () => {
+      expect(res.posts.length).toBe(1);
+      expect(res.posts[0].pageUrl).toBe(pageUrl);
+      expect(res.posts[0].contentUrl).toBe("https://pbs.twimg.com/media/EEDOOCdU0AAHX00?format=jpg&name=orig");
+    });
+  });
 });

src/engines/AnimePictures.ts

@@ -12,6 +12,7 @@ export default class AnimePictures implements ScrapeEngine {
   scrapeDocument(document: Document): ScrapeResult {
     let result = new ScrapeResult(this.name);
     let post = new ScrapedPost();
+    post.pageUrl = document.location.href;
     post.source = document.location.href;
 
     // Set image url