Update Pixiv engine and tests

docs/TESTING.md

@@ -0,0 +1,24 @@
+# Testing notes
+
+## Run all tests in testfiles matching some pattern 
+
+```sh
+yarn test pixiv
+```
+
+## Run one specific test ({{describe name}} {{it name}})
+
+```sh
+yarn test pixiv -t "get pixiv post 96268450 \(multipage, expanded\) should find exactly four posts"
+```
+
+For some reason you need to escape the parentheses.
+
+## Debug puppeteer tests
+
+```sh
+yarn test:debug # you can add more arguments if you want
+```
+
+This will disable headless mode.
+Use `await jestPuppeteer.debug();` in the test code to pause code execution.

package.json

@@ -1,18 +1,22 @@
 {
   "name": "neo-scraper",
-  "version": "0.5.3",
+  "version": "0.5.4",
   "description": "Scrape a booru-like post object from the DOM.",
   "main": "dist/index.js",
   "types": "dist/index.d.js",
   "scripts": {
     "test": "jest",
     "test:ci": "jest --colors --silent",
+    "test:debug": "cross-env HEADLESS=false jest --testTimeout=100000000",
     "build": "tsc",
     "watch": "tsc -w",
     "prepare": "npm run build"
   },
   "author": "neobooru",
   "license": "MIT",
+  "dependencies": {
+    "turndown": "^7.1.1"
+  },
   "devDependencies": {
     "@types/expect-puppeteer": "^4.4.7",
     "@types/jest": "^26.0.14",
@@ -36,6 +40,7 @@
       "ts",
       "js"
     ],
+    "testTimeout": 15000,
     "testRegex": "^.+\\.test\\.ts$",
     "setupFilesAfterEnv": [
       "./src/test/jest.setup.ts"
@@ -45,8 +50,5 @@
         "tsconfig": "tsconfig.test.json"
       }
     }
-  },
-  "dependencies": {
-    "turndown": "^7.1.1"
   }
 }

src/engines/Pixiv.test.ts

@@ -1,5 +1,18 @@
 import { ScrapeResults } from "../ScrapeEngine";
-import { scrapeEvalUrl } from "../test/engine";
+import { MobileUserAgent, scrapeEvalUrl, scrapePage } from "../test/engine";
+import { upgradeUrlToOriginalQuality } from "./Pixiv";
+
+test("upgradeUrlToOriginalQuality", async () => {
+  expect(
+    upgradeUrlToOriginalQuality("https://i.pximg.net/img-master/img/2022/02/15/06/50/28/96268450_p2_master1200.jpg")
+  ).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p2.jpg");
+
+  expect(
+    upgradeUrlToOriginalQuality(
+      "https://i.pximg.net/c/600x1200_90_webp/img-master/img/2022/02/15/06/50/28/96268450_p2_master1200.jpg"
+    )
+  ).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p2.jpg");
+});
 
 describe("get pixiv post 93964654", () => {
   const pageUrl = "https://www.pixiv.net/en/artworks/93964654";
@@ -9,10 +22,72 @@ describe("get pixiv post 93964654", () => {
     res = await scrapeEvalUrl(pageUrl, ".gtm-expand-full-size-illust");
   });
 
-  it("should grab original image url", async () => {
+  it("should find exactly one post", () => {
-    pending("Test does not work.");
+    expect(res.posts.length).toBe(1);
-    // expect(res.posts.length).toBe(1);
+  });
-    // expect(res.posts[0].pageUrl).toBe(pageUrl);
+
-    // expect(res.posts[0].contentUrl).toBe("https://i.pximg.net/img-original/img/2021/11/07/00/01/02/93964654_p0.jpg");
+  it("should grab original image url", () => {
+    expect(res.posts[0].contentUrl).toBe("https://i.pximg.net/img-original/img/2021/11/07/00/01/02/93964654_p0.jpg");
+  });
+});
+
+describe("get pixiv post 93964654 (mobile)", () => {
+  const pageUrl = "https://www.pixiv.net/en/artworks/93964654";
+  let res: ScrapeResults;
+
+  beforeAll(async () => {
+    res = await scrapeEvalUrl(pageUrl, ".work-thumb", MobileUserAgent);
+  });
+
+  it("should find exactly one post", () => {
+    expect(res.posts.length).toBe(1);
+  });
+
+  it("should grab original image url", () => {
+    expect(res.posts[0].contentUrl).toBe("https://i.pximg.net/img-original/img/2021/11/07/00/01/02/93964654_p0.jpg");
+  });
+});
+
+describe("get pixiv post 96268450 (multipage, not expanded)", () => {
+  const pageUrl = "https://www.pixiv.net/en/artworks/96268450";
+  let res: ScrapeResults;
+
+  beforeAll(async () => {
+    res = await scrapeEvalUrl(pageUrl, "div[role='presentation'] img");
+  });
+
+  it("should find exactly one post", () => {
+    expect(res.posts.length).toBe(1);
+  });
+
+  it("should grab original image url", () => {
+    expect(res.posts[0].contentUrl).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p0.jpg");
+  });
+});
+
+describe("get pixiv post 96268450 (multipage, expanded)", () => {
+  const pageUrl = "https://www.pixiv.net/en/artworks/96268450";
+  let res: ScrapeResults;
+
+  beforeAll(async () => {
+    await jestPuppeteer.resetPage();
+    await page.goto(pageUrl);
+    await page.waitForSelector("div[role='presentation'] img", { visible: true });
+    await expect(page).toClick("button", { text: "See all" });
+    await page.waitForSelector(".gtm-expand-full-size-illust", { visible: true });
+    // await jestPuppeteer.debug();
+    await page.waitForTimeout(2000);
+    res = await scrapePage(pageUrl, page);
+  });
+
+  it("should find exactly four posts", () => {
+    expect(res.posts.length).toBe(4);
+  });
+
+  it("should grab original image url", () => {
+    expect(res.posts[0].contentUrl).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p0.jpg");
+    expect(res.posts[1].contentUrl).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p1.jpg");
+    expect(res.posts[2].contentUrl).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p2.jpg");
+    expect(res.posts[3].contentUrl).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p3.jpg");
   });
 });

src/engines/Pixiv.ts

@@ -1,7 +1,26 @@
-import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
+import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine";
-import { TagCategory } from "../BooruTypes";
 import { guessContentType } from "../Utility";
 
+export function upgradeUrlToOriginalQuality(imgLink: string) {
+  if (imgLink.indexOf("img-master") != -1) {
+    // Example:
+    // Input:  https://i.pximg.net/img-master/img/2022/02/15/06/50/28/96268450_p2_master1200.jpg
+    // Input:  https://i.pximg.net/c/600x1200_90_webp/img-master/img/2022/02/15/06/50/28/96268450_p2_master1200.jpg
+    // Output: https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p2.jpg
+    //
+    // Warning:
+    // This will NOT work for a srcset like this:
+    // https://i.pximg.net/c/540x540_70/img-master/img/2022/02/15/06/50/28/96268450_p0_master1200.jpg 540w,https://i.pximg.net/img-master/img/2022/02/15/06/50/28/96268450_p0_master1200.jpg
+    //
+    // See regexr.com/6rcag for an example.
+
+    // TODO: Add a check to verify the newly constructed/guessed url?
+    return imgLink.replace(/(https?:\/\/i\.pximg\.net).*\/img-master\/(.*)_master\d+(.*)/, "$1/img-original/$2$3");
+  }
+
+  return imgLink;
+}
+
 export default class Pixiv implements ScrapeEngine {
   name = "pixiv";
 
@@ -12,14 +31,41 @@ export default class Pixiv implements ScrapeEngine {
   scrapeDocument(document: Document): ScrapeResult {
     let result = new ScrapeResult(this.name);
 
-    const imgLinks = Array.from(document.querySelectorAll("div[role='presentation'] > a")).map(
+    let imgLinks = [] as string[];
-      (x) => x as HTMLAnchorElement
+
+    // Query for when the user is on the desktop site, is logged in, and has optionally clicked on "See all".
+    // If the user has not clicked on "See all" then this will only grab the first post.
+    imgLinks.push(
+      ...Array.from(document.querySelectorAll<HTMLAnchorElement>("div[role='presentation'] > a")).map((x) => x.href)
     );
 
+    if (imgLinks.length == 0) {
+      // Query for when the user is on the desktop site and not logged in.
+      imgLinks.push(...Array.from(document.querySelectorAll<HTMLImageElement>("img[srcset]")).map((x) => x.src));
+    }
+
+    if (imgLinks.length == 0) {
+      // Query for when the user is on the mobile site, and has clicked on "See all".
+      // This only works on pages with multiple posts. The user can optionally be logged in.
+      imgLinks.push(
+        ...Array.from(document.querySelectorAll<HTMLImageElement>(".manga-page > a > img[data-big]")).map(
+          (x) => x.getAttribute("data-big")!
+        )
+      );
+    }
+
+    if (imgLinks.length == 0) {
+      // Query for when the user is on the mobile site, and has not clicked on "See all" -or- there is only one post.
+      const imgLink = document.querySelector<HTMLImageElement>(".work-main-image img")?.src;
+      if (imgLink) {
+        imgLinks.push(imgLink);
+      }
+    }
+
     for (const imgLink of imgLinks) {
       let post = new ScrapedPost();
       post.pageUrl = document.location.href;
-      post.contentUrl = imgLink.href;
+      post.contentUrl = upgradeUrlToOriginalQuality(imgLink);
       post.contentType = guessContentType(post.contentUrl);
       post.referrer = document.location.origin;
       result.tryAddPost(post);

src/engines/SankakuComplex.test.ts

@@ -26,7 +26,7 @@ test("get chan.sankakucomplex.com post 29354061 with notes", async () => {
   expect(res.posts.length).toBe(1);
   expect(res.posts[0]).toHaveTag("bangs");
   expect(res.posts[0]).toHaveTag("braid");
-  expect(res.posts[0]).toHaveTag("sankuro_(agoitei)");
+  expect(res.posts[0]).toHaveTag("sankuro");
   expect(res.posts[0]).toHaveTag("translated");
   expect(res.posts[0]).toHaveTag("note_translated");
   expect(res.posts[0]).toHaveNote("No.  \nI already told you...");

src/test/engine.ts

@@ -1,6 +1,9 @@
 import { JSDOM, ConstructorOptions } from "jsdom";
+import { Page } from "puppeteer";
 import NeoScraper from "../NeoScraper";
 
+export const MobileUserAgent = "Mozilla/5.0 (Android 12; Mobile; rv:68.0) Gecko/68.0 Firefox/103.0";
+
 /**
  * Go to an URL and parse it's raw HTML without executing any client-side code.
  * @param url
@@ -17,9 +20,19 @@ export async function scrapeUrl(url: string) {
  * @param waitForSelector
  * @returns
  */
-export async function scrapeEvalUrl(url: string, waitForSelector: string) {
+export async function scrapeEvalUrl(url: string, waitForSelector: string, userAgent: string | undefined = undefined) {
+  await jestPuppeteer.resetPage();
+
+  if (userAgent) {
+    page.setUserAgent(userAgent);
+  }
+
   await page.goto(url);
   await page.waitForSelector(waitForSelector, { visible: true });
+  return scrapePage(url, page);
+}
+
+export async function scrapePage(url: string, page: Page) {
   const document = await page.content();
   return scrapeHtml(url, document);
 }