From b67d024274b9ab89af801841b05b33453b2417e2 Mon Sep 17 00:00:00 2001 From: neobooru <50623835+neobooru@users.noreply.github.com> Date: Mon, 19 Jun 2023 21:22:23 +0200 Subject: [PATCH] Fix broken engines and update tests where needed --- src/engines/AnimePictures.test.ts | 4 ++-- src/engines/AnimePictures.ts | 5 +---- src/engines/Danbooru.test.ts | 4 ++-- src/engines/Gelbooru.test.ts | 3 +-- src/engines/Pixiv.test.ts | 2 +- src/engines/Pixiv.ts | 10 ++++++---- src/engines/SankakuComplex.test.ts | 15 ++++----------- src/engines/Zerochan.ts | 3 ++- src/test/engine.ts | 3 +-- 9 files changed, 20 insertions(+), 29 deletions(-) diff --git a/src/engines/AnimePictures.test.ts b/src/engines/AnimePictures.test.ts index 1d23693..bbbfafe 100644 --- a/src/engines/AnimePictures.test.ts +++ b/src/engines/AnimePictures.test.ts @@ -1,7 +1,7 @@ import { scrapeUrl } from "../test/engine.js"; test("get anime-pictures.net post 729833", async () => { - const pageUrl = "https://anime-pictures.net/pictures/view_post/729833?lang=en"; + const pageUrl = "https://anime-pictures.net/posts/729833?lang=en"; const res = await scrapeUrl(pageUrl); expect(res.posts.length).toBe(1); expect(res.posts[0]).toHaveTag("arknights", "copyright"); @@ -14,6 +14,6 @@ test("get anime-pictures.net post 729833", async () => { expect(res.posts[0]).toHaveResolution([4000, 6865]); expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].contentUrl).toBe( - "https://anime-pictures.net/pictures/download_image/729833-4000x6865-arknights-ch%26%2339%3Ben+%28arknights%29-ttk+%28kirinottk%29-single-long+hair-tall+image.jpeg" + "https://anime-pictures.net/pictures/download_image/729833-4000x6865-arknights-ch'en+(arknights)-ttk+(kirinottk)-single-long+hair-tall+image.jpeg" ); }); diff --git a/src/engines/AnimePictures.ts b/src/engines/AnimePictures.ts index bd60d19..f746f33 100644 --- a/src/engines/AnimePictures.ts +++ b/src/engines/AnimePictures.ts @@ -12,7 +12,6 @@ export default class AnimePictures extends ScrapeEngineBase { const result = new ScrapeResult(this.name); const post = new ScrapedPost(); post.pageUrl = document.location.href; - // Set image url const originalImageElements = Array.from(document.getElementsByClassName("download_icon")); @@ -28,9 +27,7 @@ export default class AnimePictures extends ScrapeEngineBase { post.rating = "safe"; // Probably // Set resolution - const postContentLinkEls = Array.from( - document.querySelectorAll("#content > div > .post_content > a") - ); + const postContentLinkEls = Array.from(document.querySelectorAll(".post_content > .body a")); const resEl = postContentLinkEls.find((x) => x.href.indexOf("res_x") != -1); if (resEl) { post.resolution = parseResolutionString(resEl.innerText); diff --git a/src/engines/Danbooru.test.ts b/src/engines/Danbooru.test.ts index 4cb01dd..52d1529 100644 --- a/src/engines/Danbooru.test.ts +++ b/src/engines/Danbooru.test.ts @@ -7,7 +7,6 @@ test("get danbooru.donmai.us post 3555872", async () => { expect(res.posts[0]).toHaveTag("omachi_(slabco)", "artist"); expect(res.posts[0]).toHaveTag("dr_pepper", "copyright"); expect(res.posts[0]).toHaveTag("flint_(girls_und_panzer)", "character"); - expect(res.posts[0]).toHaveTag("bangs"); expect(res.posts[0]).toHaveTag("shirt"); expect(res.posts[0]).toHaveTag("skirt"); expect(res.posts[0]).toHaveTag("t-shirt"); @@ -44,5 +43,6 @@ test("get danbooru.donmai.us post 4530532", async () => { expect(res.posts[0]).toHaveResolution([1105, 1803]); expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].contentUrl).toBe( - "https://cdn.donmai.us/original/fc/e3/__makima_power_and_reze_chainsaw_man_drawn_by_hews__fce35c7a8515a00c566b288a591fd5f9.png"); + "https://cdn.donmai.us/original/fc/e3/__makima_power_and_reze_chainsaw_man_drawn_by_hews__fce35c7a8515a00c566b288a591fd5f9.png" + ); }); diff --git a/src/engines/Gelbooru.test.ts b/src/engines/Gelbooru.test.ts index b926f99..68c8c4e 100644 --- a/src/engines/Gelbooru.test.ts +++ b/src/engines/Gelbooru.test.ts @@ -34,7 +34,6 @@ test("get gelbooru.com post 6225495 with notes", async () => { const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=6225495"; const res = await scrapeUrl(pageUrl); expect(res.posts.length).toBe(1); - expect(res.posts[0]).toHaveTag("bangs"); expect(res.posts[0]).toHaveTag("hair_up"); expect(res.posts[0]).toHaveTag("hair_up"); expect(res.posts[0]).toHaveTag("teeth"); @@ -56,7 +55,7 @@ test("get rule34.xxx post 2740994", async () => { const res = await scrapeUrl(pageUrl); expect(res.posts[0]).toHaveTag("fire_emblem", "copyright"); expect(res.posts[0]).toHaveTag("nintendo", "copyright"); - expect(res.posts[0]).toHaveTag("lucina", "character"); + expect(res.posts[0]).toHaveTag("lucina_(fire_emblem)", "character"); expect(res.posts[0]).toHaveTag("blue_hair"); expect(res.posts[0]).toHaveTag("solo"); expect(res.posts[0]).toHaveResolution([600, 800]); diff --git a/src/engines/Pixiv.test.ts b/src/engines/Pixiv.test.ts index 19782cd..a8f195b 100644 --- a/src/engines/Pixiv.test.ts +++ b/src/engines/Pixiv.test.ts @@ -73,7 +73,7 @@ describe("get pixiv post 96268450 (multipage, expanded)", () => { await jestPuppeteer.resetPage(); await page.goto(pageUrl); await page.waitForSelector("div[role='presentation'] img", { visible: true }); - await expect(page).toClick("button", { text: "See all" }); + await expect(page).toClick("button", { text: "Show all" }); await page.waitForSelector(".gtm-expand-full-size-illust", { visible: true }); // await jestPuppeteer.debug(); await page.waitForTimeout(2000); diff --git a/src/engines/Pixiv.ts b/src/engines/Pixiv.ts index 2114158..ba1ab91 100644 --- a/src/engines/Pixiv.ts +++ b/src/engines/Pixiv.ts @@ -31,19 +31,21 @@ export default class Pixiv extends ScrapeEngineBase { const result = new ScrapeResult(this.name); const imgLinks = [] as string[]; - // Query for when the user is on the desktop site, is logged in, and has optionally clicked on "See all". - // If the user has not clicked on "See all" then this will only grab the first post. + // Query for when the user is on the desktop site, is logged in, and has optionally clicked on "Show all". + // If the user has not clicked on "Show all" then this will only grab the first post. imgLinks.push( ...Array.from(document.querySelectorAll("div[role='presentation'] > a")).map((x) => x.href) ); if (imgLinks.length == 0) { // Query for when the user is on the desktop site and not logged in. - imgLinks.push(...Array.from(document.querySelectorAll("img[srcset]")).map((x) => x.src)); + imgLinks.push( + ...Array.from(document.querySelectorAll("div[role='presentation'] img")).map((x) => x.src) + ); } if (imgLinks.length == 0) { - // Query for when the user is on the mobile site, and has clicked on "See all". + // Query for when the user is on the mobile site, and has clicked on "Show all". // This only works on pages with multiple posts. The user can optionally be logged in. imgLinks.push( ...Array.from(document.querySelectorAll(".manga-page > a > img[data-big]")).map( diff --git a/src/engines/SankakuComplex.test.ts b/src/engines/SankakuComplex.test.ts index c752d8d..99d1f30 100644 --- a/src/engines/SankakuComplex.test.ts +++ b/src/engines/SankakuComplex.test.ts @@ -1,37 +1,30 @@ import { scrapeUrl } from "../test/engine.js"; test("get chan.sankakucomplex.com post 19614481 with notes", async () => { - const pageUrl = "https://chan.sankakucomplex.com/post/show/19614481"; + const pageUrl = "https://chan.sankakucomplex.com/post/show/507e986a62cc5b866d92ad491dcfb1a4"; const res = await scrapeUrl(pageUrl); expect(res.posts.length).toBe(1); expect(res.posts[0]).toHaveTag("bangs"); expect(res.posts[0]).toHaveTag("bare_shoulders"); expect(res.posts[0]).toHaveTag("solo"); - expect(res.posts[0]).toHaveTag("translated"); expect(res.posts[0]).toHaveTag("text"); expect(res.posts[0]).toHaveTag("idolmaster"); expect(res.posts[0]).toHaveTag("mayuzumi_fuyuko"); expect(res.posts[0]).toHaveNote("What are you looking at..."); expect(res.posts[0]).toHaveNote("What...?"); expect(res.posts[0].pageUrl).toBe(pageUrl); - expect(res.posts[0].contentUrl).toContain( - "https://s.sankakucomplex.com/data/50/7e/507e986a62cc5b866d92ad491dcfb1a4.jpg" - ); + expect(res.posts[0].contentUrl).toContain("sankakucomplex.com/data/50/7e/507e986a62cc5b866d92ad491dcfb1a4.jpg"); }); test("get chan.sankakucomplex.com post 29354061 with notes", async () => { // Kinda sketchy - const pageUrl = "https://chan.sankakucomplex.com/post/show/29354061"; + const pageUrl = "https://chan.sankakucomplex.com/post/show/c8376f02608548365d394a658e85cb3f"; const res = await scrapeUrl(pageUrl); expect(res.posts.length).toBe(1); expect(res.posts[0]).toHaveTag("bangs"); expect(res.posts[0]).toHaveTag("braid"); expect(res.posts[0]).toHaveTag("sankuro"); - expect(res.posts[0]).toHaveTag("translated"); - expect(res.posts[0]).toHaveTag("note_translated"); expect(res.posts[0]).toHaveNote("No. \nI already told you..."); expect(res.posts[0].pageUrl).toBe(pageUrl); - expect(res.posts[0].contentUrl).toContain( - "https://s.sankakucomplex.com/data/c8/37/c8376f02608548365d394a658e85cb3f.jpg" - ); + expect(res.posts[0].contentUrl).toContain("sankakucomplex.com/data/c8/37/c8376f02608548365d394a658e85cb3f.jpg"); }); diff --git a/src/engines/Zerochan.ts b/src/engines/Zerochan.ts index c189306..3175b5c 100644 --- a/src/engines/Zerochan.ts +++ b/src/engines/Zerochan.ts @@ -52,7 +52,8 @@ export default class Zerochan extends ScrapeEngineBase { } let category: TagCategory | undefined; - switch (tagElement.className) { + const className = tagElement.className.replace("emphasis", "").trim(); + switch (className) { case "game": case "series": case "studio": diff --git a/src/test/engine.ts b/src/test/engine.ts index 6ee032d..d370c59 100644 --- a/src/test/engine.ts +++ b/src/test/engine.ts @@ -1,5 +1,4 @@ import { JSDOM, ConstructorOptions } from "jsdom"; -import { Page } from "puppeteer"; import NeoScraper from "../NeoScraper.js"; export const MobileUserAgent = "Mozilla/5.0 (Android 12; Mobile; rv:68.0) Gecko/68.0 Firefox/103.0"; @@ -32,7 +31,7 @@ export async function scrapeEvalUrl(url: string, waitForSelector: string, userAg return scrapePage(url, page); } -export async function scrapePage(url: string, page: Page) { +export async function scrapePage(url: string, page: any) { const document = await page.content(); return scrapeHtml(url, document); }