diff --git a/docs/TESTING.md b/docs/TESTING.md new file mode 100644 index 0000000..a7e611e --- /dev/null +++ b/docs/TESTING.md @@ -0,0 +1,24 @@ +# Testing notes + +## Run all tests in testfiles matching some pattern + +```sh +yarn test pixiv +``` + +## Run one specific test ({{describe name}} {{it name}}) + +```sh +yarn test pixiv -t "get pixiv post 96268450 \(multipage, expanded\) should find exactly four posts" +``` + +For some reason you need to escape the parentheses. + +## Debug puppeteer tests + +```sh +yarn test:debug # you can add more arguments if you want +``` + +This will disable headless mode. +Use `await jestPuppeteer.debug();` in the test code to pause code execution. diff --git a/package.json b/package.json index 13255d4..5608c0e 100644 --- a/package.json +++ b/package.json @@ -1,18 +1,22 @@ { "name": "neo-scraper", - "version": "0.5.3", + "version": "0.5.4", "description": "Scrape a booru-like post object from the DOM.", "main": "dist/index.js", "types": "dist/index.d.js", "scripts": { "test": "jest", "test:ci": "jest --colors --silent", + "test:debug": "cross-env HEADLESS=false jest --testTimeout=100000000", "build": "tsc", "watch": "tsc -w", "prepare": "npm run build" }, "author": "neobooru", "license": "MIT", + "dependencies": { + "turndown": "^7.1.1" + }, "devDependencies": { "@types/expect-puppeteer": "^4.4.7", "@types/jest": "^26.0.14", @@ -36,6 +40,7 @@ "ts", "js" ], + "testTimeout": 15000, "testRegex": "^.+\\.test\\.ts$", "setupFilesAfterEnv": [ "./src/test/jest.setup.ts" @@ -45,8 +50,5 @@ "tsconfig": "tsconfig.test.json" } } - }, - "dependencies": { - "turndown": "^7.1.1" } } diff --git a/src/engines/Pixiv.test.ts b/src/engines/Pixiv.test.ts index 0ca31c2..1ea240d 100644 --- a/src/engines/Pixiv.test.ts +++ b/src/engines/Pixiv.test.ts @@ -1,5 +1,18 @@ import { ScrapeResults } from "../ScrapeEngine"; -import { scrapeEvalUrl } from "../test/engine"; +import { MobileUserAgent, scrapeEvalUrl, scrapePage } from "../test/engine"; +import { upgradeUrlToOriginalQuality } from "./Pixiv"; + +test("upgradeUrlToOriginalQuality", async () => { + expect( + upgradeUrlToOriginalQuality("https://i.pximg.net/img-master/img/2022/02/15/06/50/28/96268450_p2_master1200.jpg") + ).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p2.jpg"); + + expect( + upgradeUrlToOriginalQuality( + "https://i.pximg.net/c/600x1200_90_webp/img-master/img/2022/02/15/06/50/28/96268450_p2_master1200.jpg" + ) + ).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p2.jpg"); +}); describe("get pixiv post 93964654", () => { const pageUrl = "https://www.pixiv.net/en/artworks/93964654"; @@ -9,10 +22,72 @@ describe("get pixiv post 93964654", () => { res = await scrapeEvalUrl(pageUrl, ".gtm-expand-full-size-illust"); }); - it("should grab original image url", async () => { - pending("Test does not work."); - // expect(res.posts.length).toBe(1); - // expect(res.posts[0].pageUrl).toBe(pageUrl); - // expect(res.posts[0].contentUrl).toBe("https://i.pximg.net/img-original/img/2021/11/07/00/01/02/93964654_p0.jpg"); + it("should find exactly one post", () => { + expect(res.posts.length).toBe(1); + }); + + it("should grab original image url", () => { + expect(res.posts[0].contentUrl).toBe("https://i.pximg.net/img-original/img/2021/11/07/00/01/02/93964654_p0.jpg"); + }); +}); + +describe("get pixiv post 93964654 (mobile)", () => { + const pageUrl = "https://www.pixiv.net/en/artworks/93964654"; + let res: ScrapeResults; + + beforeAll(async () => { + res = await scrapeEvalUrl(pageUrl, ".work-thumb", MobileUserAgent); + }); + + it("should find exactly one post", () => { + expect(res.posts.length).toBe(1); + }); + + it("should grab original image url", () => { + expect(res.posts[0].contentUrl).toBe("https://i.pximg.net/img-original/img/2021/11/07/00/01/02/93964654_p0.jpg"); + }); +}); + +describe("get pixiv post 96268450 (multipage, not expanded)", () => { + const pageUrl = "https://www.pixiv.net/en/artworks/96268450"; + let res: ScrapeResults; + + beforeAll(async () => { + res = await scrapeEvalUrl(pageUrl, "div[role='presentation'] img"); + }); + + it("should find exactly one post", () => { + expect(res.posts.length).toBe(1); + }); + + it("should grab original image url", () => { + expect(res.posts[0].contentUrl).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p0.jpg"); + }); +}); + +describe("get pixiv post 96268450 (multipage, expanded)", () => { + const pageUrl = "https://www.pixiv.net/en/artworks/96268450"; + let res: ScrapeResults; + + beforeAll(async () => { + await jestPuppeteer.resetPage(); + await page.goto(pageUrl); + await page.waitForSelector("div[role='presentation'] img", { visible: true }); + await expect(page).toClick("button", { text: "See all" }); + await page.waitForSelector(".gtm-expand-full-size-illust", { visible: true }); + // await jestPuppeteer.debug(); + await page.waitForTimeout(2000); + res = await scrapePage(pageUrl, page); + }); + + it("should find exactly four posts", () => { + expect(res.posts.length).toBe(4); + }); + + it("should grab original image url", () => { + expect(res.posts[0].contentUrl).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p0.jpg"); + expect(res.posts[1].contentUrl).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p1.jpg"); + expect(res.posts[2].contentUrl).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p2.jpg"); + expect(res.posts[3].contentUrl).toBe("https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p3.jpg"); }); }); diff --git a/src/engines/Pixiv.ts b/src/engines/Pixiv.ts index 12f4d42..248906b 100644 --- a/src/engines/Pixiv.ts +++ b/src/engines/Pixiv.ts @@ -1,7 +1,26 @@ -import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; -import { TagCategory } from "../BooruTypes"; +import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine"; import { guessContentType } from "../Utility"; +export function upgradeUrlToOriginalQuality(imgLink: string) { + if (imgLink.indexOf("img-master") != -1) { + // Example: + // Input: https://i.pximg.net/img-master/img/2022/02/15/06/50/28/96268450_p2_master1200.jpg + // Input: https://i.pximg.net/c/600x1200_90_webp/img-master/img/2022/02/15/06/50/28/96268450_p2_master1200.jpg + // Output: https://i.pximg.net/img-original/img/2022/02/15/06/50/28/96268450_p2.jpg + // + // Warning: + // This will NOT work for a srcset like this: + // https://i.pximg.net/c/540x540_70/img-master/img/2022/02/15/06/50/28/96268450_p0_master1200.jpg 540w,https://i.pximg.net/img-master/img/2022/02/15/06/50/28/96268450_p0_master1200.jpg + // + // See regexr.com/6rcag for an example. + + // TODO: Add a check to verify the newly constructed/guessed url? + return imgLink.replace(/(https?:\/\/i\.pximg\.net).*\/img-master\/(.*)_master\d+(.*)/, "$1/img-original/$2$3"); + } + + return imgLink; +} + export default class Pixiv implements ScrapeEngine { name = "pixiv"; @@ -12,14 +31,41 @@ export default class Pixiv implements ScrapeEngine { scrapeDocument(document: Document): ScrapeResult { let result = new ScrapeResult(this.name); - const imgLinks = Array.from(document.querySelectorAll("div[role='presentation'] > a")).map( - (x) => x as HTMLAnchorElement + let imgLinks = [] as string[]; + + // Query for when the user is on the desktop site, is logged in, and has optionally clicked on "See all". + // If the user has not clicked on "See all" then this will only grab the first post. + imgLinks.push( + ...Array.from(document.querySelectorAll("div[role='presentation'] > a")).map((x) => x.href) ); + if (imgLinks.length == 0) { + // Query for when the user is on the desktop site and not logged in. + imgLinks.push(...Array.from(document.querySelectorAll("img[srcset]")).map((x) => x.src)); + } + + if (imgLinks.length == 0) { + // Query for when the user is on the mobile site, and has clicked on "See all". + // This only works on pages with multiple posts. The user can optionally be logged in. + imgLinks.push( + ...Array.from(document.querySelectorAll(".manga-page > a > img[data-big]")).map( + (x) => x.getAttribute("data-big")! + ) + ); + } + + if (imgLinks.length == 0) { + // Query for when the user is on the mobile site, and has not clicked on "See all" -or- there is only one post. + const imgLink = document.querySelector(".work-main-image img")?.src; + if (imgLink) { + imgLinks.push(imgLink); + } + } + for (const imgLink of imgLinks) { let post = new ScrapedPost(); post.pageUrl = document.location.href; - post.contentUrl = imgLink.href; + post.contentUrl = upgradeUrlToOriginalQuality(imgLink); post.contentType = guessContentType(post.contentUrl); post.referrer = document.location.origin; result.tryAddPost(post); diff --git a/src/engines/SankakuComplex.test.ts b/src/engines/SankakuComplex.test.ts index 589b7f7..39e5442 100644 --- a/src/engines/SankakuComplex.test.ts +++ b/src/engines/SankakuComplex.test.ts @@ -26,7 +26,7 @@ test("get chan.sankakucomplex.com post 29354061 with notes", async () => { expect(res.posts.length).toBe(1); expect(res.posts[0]).toHaveTag("bangs"); expect(res.posts[0]).toHaveTag("braid"); - expect(res.posts[0]).toHaveTag("sankuro_(agoitei)"); + expect(res.posts[0]).toHaveTag("sankuro"); expect(res.posts[0]).toHaveTag("translated"); expect(res.posts[0]).toHaveTag("note_translated"); expect(res.posts[0]).toHaveNote("No. \nI already told you..."); diff --git a/src/test/engine.ts b/src/test/engine.ts index 782fb08..5852795 100644 --- a/src/test/engine.ts +++ b/src/test/engine.ts @@ -1,6 +1,9 @@ import { JSDOM, ConstructorOptions } from "jsdom"; +import { Page } from "puppeteer"; import NeoScraper from "../NeoScraper"; +export const MobileUserAgent = "Mozilla/5.0 (Android 12; Mobile; rv:68.0) Gecko/68.0 Firefox/103.0"; + /** * Go to an URL and parse it's raw HTML without executing any client-side code. * @param url @@ -17,9 +20,19 @@ export async function scrapeUrl(url: string) { * @param waitForSelector * @returns */ -export async function scrapeEvalUrl(url: string, waitForSelector: string) { +export async function scrapeEvalUrl(url: string, waitForSelector: string, userAgent: string | undefined = undefined) { + await jestPuppeteer.resetPage(); + + if (userAgent) { + page.setUserAgent(userAgent); + } + await page.goto(url); await page.waitForSelector(waitForSelector, { visible: true }); + return scrapePage(url, page); +} + +export async function scrapePage(url: string, page: Page) { const document = await page.content(); return scrapeHtml(url, document); }