Fix broken engines and update tests where needed

src/engines/AnimePictures.test.ts

@@ -1,7 +1,7 @@
 import { scrapeUrl } from "../test/engine.js";
 
 test("get anime-pictures.net post 729833", async () => {
-  const pageUrl = "https://anime-pictures.net/pictures/view_post/729833?lang=en";
+  const pageUrl = "https://anime-pictures.net/posts/729833?lang=en";
   const res = await scrapeUrl(pageUrl);
   expect(res.posts.length).toBe(1);
   expect(res.posts[0]).toHaveTag("arknights", "copyright");
@@ -14,6 +14,6 @@ test("get anime-pictures.net post 729833", async () => {
   expect(res.posts[0]).toHaveResolution([4000, 6865]);
   expect(res.posts[0].pageUrl).toBe(pageUrl);
   expect(res.posts[0].contentUrl).toBe(
-    "https://anime-pictures.net/pictures/download_image/729833-4000x6865-arknights-ch%26%2339%3Ben+%28arknights%29-ttk+%28kirinottk%29-single-long+hair-tall+image.jpeg"
+    "https://anime-pictures.net/pictures/download_image/729833-4000x6865-arknights-ch'en+(arknights)-ttk+(kirinottk)-single-long+hair-tall+image.jpeg"
   );
 });

src/engines/AnimePictures.ts

@@ -13,7 +13,6 @@ export default class AnimePictures extends ScrapeEngineBase {
     const post = new ScrapedPost();
     post.pageUrl = document.location.href;
 
-
     // Set image url
     const originalImageElements = Array.from(document.getElementsByClassName("download_icon"));
 
@@ -28,9 +27,7 @@ export default class AnimePictures extends ScrapeEngineBase {
     post.rating = "safe"; // Probably
 
     // Set resolution
-    const postContentLinkEls = Array.from(
+    const postContentLinkEls = Array.from(document.querySelectorAll<HTMLAnchorElement>(".post_content > .body a"));
-      document.querySelectorAll<HTMLAnchorElement>("#content > div > .post_content > a")
-    );
     const resEl = postContentLinkEls.find((x) => x.href.indexOf("res_x") != -1);
     if (resEl) {
       post.resolution = parseResolutionString(resEl.innerText);

src/engines/Danbooru.test.ts

@@ -7,7 +7,6 @@ test("get danbooru.donmai.us post 3555872", async () => {
   expect(res.posts[0]).toHaveTag("omachi_(slabco)", "artist");
   expect(res.posts[0]).toHaveTag("dr_pepper", "copyright");
   expect(res.posts[0]).toHaveTag("flint_(girls_und_panzer)", "character");
-  expect(res.posts[0]).toHaveTag("bangs");
   expect(res.posts[0]).toHaveTag("shirt");
   expect(res.posts[0]).toHaveTag("skirt");
   expect(res.posts[0]).toHaveTag("t-shirt");
@@ -44,5 +43,6 @@ test("get danbooru.donmai.us post 4530532", async () => {
   expect(res.posts[0]).toHaveResolution([1105, 1803]);
   expect(res.posts[0].pageUrl).toBe(pageUrl);
   expect(res.posts[0].contentUrl).toBe(
-    "https://cdn.donmai.us/original/fc/e3/__makima_power_and_reze_chainsaw_man_drawn_by_hews__fce35c7a8515a00c566b288a591fd5f9.png");
+    "https://cdn.donmai.us/original/fc/e3/__makima_power_and_reze_chainsaw_man_drawn_by_hews__fce35c7a8515a00c566b288a591fd5f9.png"
+  );
 });

src/engines/Gelbooru.test.ts

@@ -34,7 +34,6 @@ test("get gelbooru.com post 6225495 with notes", async () => {
   const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=6225495";
   const res = await scrapeUrl(pageUrl);
   expect(res.posts.length).toBe(1);
-  expect(res.posts[0]).toHaveTag("bangs");
   expect(res.posts[0]).toHaveTag("hair_up");
   expect(res.posts[0]).toHaveTag("hair_up");
   expect(res.posts[0]).toHaveTag("teeth");
@@ -56,7 +55,7 @@ test("get rule34.xxx post 2740994", async () => {
   const res = await scrapeUrl(pageUrl);
   expect(res.posts[0]).toHaveTag("fire_emblem", "copyright");
   expect(res.posts[0]).toHaveTag("nintendo", "copyright");
-  expect(res.posts[0]).toHaveTag("lucina", "character");
+  expect(res.posts[0]).toHaveTag("lucina_(fire_emblem)", "character");
   expect(res.posts[0]).toHaveTag("blue_hair");
   expect(res.posts[0]).toHaveTag("solo");
   expect(res.posts[0]).toHaveResolution([600, 800]);

src/engines/Pixiv.test.ts

@@ -73,7 +73,7 @@ describe("get pixiv post 96268450 (multipage, expanded)", () => {
     await jestPuppeteer.resetPage();
     await page.goto(pageUrl);
     await page.waitForSelector("div[role='presentation'] img", { visible: true });
-    await expect(page).toClick("button", { text: "See all" });
+    await expect(page).toClick("button", { text: "Show all" });
     await page.waitForSelector(".gtm-expand-full-size-illust", { visible: true });
     // await jestPuppeteer.debug();
     await page.waitForTimeout(2000);

src/engines/Pixiv.ts

@@ -31,19 +31,21 @@ export default class Pixiv extends ScrapeEngineBase {
     const result = new ScrapeResult(this.name);
     const imgLinks = [] as string[];
 
-    // Query for when the user is on the desktop site, is logged in, and has optionally clicked on "See all".
+    // Query for when the user is on the desktop site, is logged in, and has optionally clicked on "Show all".
-    // If the user has not clicked on "See all" then this will only grab the first post.
+    // If the user has not clicked on "Show all" then this will only grab the first post.
     imgLinks.push(
       ...Array.from(document.querySelectorAll<HTMLAnchorElement>("div[role='presentation'] > a")).map((x) => x.href)
     );
 
     if (imgLinks.length == 0) {
       // Query for when the user is on the desktop site and not logged in.
-      imgLinks.push(...Array.from(document.querySelectorAll<HTMLImageElement>("img[srcset]")).map((x) => x.src));
+      imgLinks.push(
+        ...Array.from(document.querySelectorAll<HTMLImageElement>("div[role='presentation'] img")).map((x) => x.src)
+      );
     }
 
     if (imgLinks.length == 0) {
-      // Query for when the user is on the mobile site, and has clicked on "See all".
+      // Query for when the user is on the mobile site, and has clicked on "Show all".
       // This only works on pages with multiple posts. The user can optionally be logged in.
       imgLinks.push(
         ...Array.from(document.querySelectorAll<HTMLImageElement>(".manga-page > a > img[data-big]")).map(

src/engines/SankakuComplex.test.ts

@@ -1,37 +1,30 @@
 import { scrapeUrl } from "../test/engine.js";
 
 test("get chan.sankakucomplex.com post 19614481 with notes", async () => {
-  const pageUrl = "https://chan.sankakucomplex.com/post/show/19614481";
+  const pageUrl = "https://chan.sankakucomplex.com/post/show/507e986a62cc5b866d92ad491dcfb1a4";
   const res = await scrapeUrl(pageUrl);
   expect(res.posts.length).toBe(1);
   expect(res.posts[0]).toHaveTag("bangs");
   expect(res.posts[0]).toHaveTag("bare_shoulders");
   expect(res.posts[0]).toHaveTag("solo");
-  expect(res.posts[0]).toHaveTag("translated");
   expect(res.posts[0]).toHaveTag("text");
   expect(res.posts[0]).toHaveTag("idolmaster");
   expect(res.posts[0]).toHaveTag("mayuzumi_fuyuko");
   expect(res.posts[0]).toHaveNote("What are you looking at...");
   expect(res.posts[0]).toHaveNote("What...?");
   expect(res.posts[0].pageUrl).toBe(pageUrl);
-  expect(res.posts[0].contentUrl).toContain(
+  expect(res.posts[0].contentUrl).toContain("sankakucomplex.com/data/50/7e/507e986a62cc5b866d92ad491dcfb1a4.jpg");
-    "https://s.sankakucomplex.com/data/50/7e/507e986a62cc5b866d92ad491dcfb1a4.jpg"
-  );
 });
 
 test("get chan.sankakucomplex.com post 29354061 with notes", async () => {
   // Kinda sketchy
-  const pageUrl = "https://chan.sankakucomplex.com/post/show/29354061";
+  const pageUrl = "https://chan.sankakucomplex.com/post/show/c8376f02608548365d394a658e85cb3f";
   const res = await scrapeUrl(pageUrl);
   expect(res.posts.length).toBe(1);
   expect(res.posts[0]).toHaveTag("bangs");
   expect(res.posts[0]).toHaveTag("braid");
   expect(res.posts[0]).toHaveTag("sankuro");
-  expect(res.posts[0]).toHaveTag("translated");
-  expect(res.posts[0]).toHaveTag("note_translated");
   expect(res.posts[0]).toHaveNote("No.  \nI already told you...");
   expect(res.posts[0].pageUrl).toBe(pageUrl);
-  expect(res.posts[0].contentUrl).toContain(
+  expect(res.posts[0].contentUrl).toContain("sankakucomplex.com/data/c8/37/c8376f02608548365d394a658e85cb3f.jpg");
-    "https://s.sankakucomplex.com/data/c8/37/c8376f02608548365d394a658e85cb3f.jpg"
-  );
 });

src/engines/Zerochan.ts

@@ -52,7 +52,8 @@ export default class Zerochan extends ScrapeEngineBase {
       }
 
       let category: TagCategory | undefined;
-      switch (tagElement.className) {
+      const className = tagElement.className.replace("emphasis", "").trim();
+      switch (className) {
         case "game":
         case "series":
         case "studio":

src/test/engine.ts

@@ -1,5 +1,4 @@
 import { JSDOM, ConstructorOptions } from "jsdom";
-import { Page } from "puppeteer";
 import NeoScraper from "../NeoScraper.js";
 
 export const MobileUserAgent = "Mozilla/5.0 (Android 12; Mobile; rv:68.0) Gecko/68.0 Firefox/103.0";
@@ -32,7 +31,7 @@ export async function scrapeEvalUrl(url: string, waitForSelector: string, userAg
   return scrapePage(url, page);
 }
 
-export async function scrapePage(url: string, page: Page) {
+export async function scrapePage(url: string, page: any) {
   const document = await page.content();
   return scrapeHtml(url, document);
 }