Nes
/
neo-scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix broken engines and update tests where needed

pull/16/head
neobooru 2023-06-19 21:22:23 +02:00
parent bf1e9d9e62
commit b67d024274
9 changed files with 20 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/engines/AnimePictures.test.ts
View File

@ -1,7 +1,7 @@
import { scrapeUrl } from "../test/engine.js";
test("get anime-pictures.net post 729833", async () => {
const pageUrl = "https://anime-pictures.net/pictures/view_post/729833?lang=en";
const pageUrl = "https://anime-pictures.net/posts/729833?lang=en";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("arknights", "copyright");
@ -14,6 +14,6 @@ test("get anime-pictures.net post 729833", async () => {
expect(res.posts[0]).toHaveResolution([4000, 6865]);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://anime-pictures.net/pictures/download_image/729833-4000x6865-arknights-ch%26%2339%3Ben+%28arknights%29-ttk+%28kirinottk%29-single-long+hair-tall+image.jpeg"
"https://anime-pictures.net/pictures/download_image/729833-4000x6865-arknights-ch'en+(arknights)-ttk+(kirinottk)-single-long+hair-tall+image.jpeg"
);
});

5
src/engines/AnimePictures.ts
View File

@ -13,7 +13,6 @@ export default class AnimePictures extends ScrapeEngineBase {
const post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url
const originalImageElements = Array.from(document.getElementsByClassName("download_icon"));
@ -28,9 +27,7 @@ export default class AnimePictures extends ScrapeEngineBase {
post.rating = "safe"; // Probably
// Set resolution
const postContentLinkEls = Array.from(
document.querySelectorAll<HTMLAnchorElement>("#content > div > .post_content > a")
);
const postContentLinkEls = Array.from(document.querySelectorAll<HTMLAnchorElement>(".post_content > .body a"));
const resEl = postContentLinkEls.find((x) => x.href.indexOf("res_x") != -1);
if (resEl) {
post.resolution = parseResolutionString(resEl.innerText);

4
src/engines/Danbooru.test.ts
View File

@ -7,7 +7,6 @@ test("get danbooru.donmai.us post 3555872", async () => {
expect(res.posts[0]).toHaveTag("omachi_(slabco)", "artist");
expect(res.posts[0]).toHaveTag("dr_pepper", "copyright");
expect(res.posts[0]).toHaveTag("flint_(girls_und_panzer)", "character");
expect(res.posts[0]).toHaveTag("bangs");
expect(res.posts[0]).toHaveTag("shirt");
expect(res.posts[0]).toHaveTag("skirt");
expect(res.posts[0]).toHaveTag("t-shirt");
@ -44,5 +43,6 @@ test("get danbooru.donmai.us post 4530532", async () => {
expect(res.posts[0]).toHaveResolution([1105, 1803]);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://cdn.donmai.us/original/fc/e3/__makima_power_and_reze_chainsaw_man_drawn_by_hews__fce35c7a8515a00c566b288a591fd5f9.png");
"https://cdn.donmai.us/original/fc/e3/__makima_power_and_reze_chainsaw_man_drawn_by_hews__fce35c7a8515a00c566b288a591fd5f9.png"
);
});

3
src/engines/Gelbooru.test.ts
View File

@ -34,7 +34,6 @@ test("get gelbooru.com post 6225495 with notes", async () => {
const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=6225495";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("bangs");
expect(res.posts[0]).toHaveTag("hair_up");
expect(res.posts[0]).toHaveTag("hair_up");
expect(res.posts[0]).toHaveTag("teeth");
@ -56,7 +55,7 @@ test("get rule34.xxx post 2740994", async () => {
const res = await scrapeUrl(pageUrl);
expect(res.posts[0]).toHaveTag("fire_emblem", "copyright");
expect(res.posts[0]).toHaveTag("nintendo", "copyright");
expect(res.posts[0]).toHaveTag("lucina", "character");
expect(res.posts[0]).toHaveTag("lucina_(fire_emblem)", "character");
expect(res.posts[0]).toHaveTag("blue_hair");
expect(res.posts[0]).toHaveTag("solo");
expect(res.posts[0]).toHaveResolution([600, 800]);

2
src/engines/Pixiv.test.ts
View File

@ -73,7 +73,7 @@ describe("get pixiv post 96268450 (multipage, expanded)", () => {
await jestPuppeteer.resetPage();
await page.goto(pageUrl);
await page.waitForSelector("div[role='presentation'] img", { visible: true });
await expect(page).toClick("button", { text: "See all" });
await expect(page).toClick("button", { text: "Show all" });
await page.waitForSelector(".gtm-expand-full-size-illust", { visible: true });
// await jestPuppeteer.debug();
await page.waitForTimeout(2000);

10
src/engines/Pixiv.ts
View File

@ -31,19 +31,21 @@ export default class Pixiv extends ScrapeEngineBase {
const result = new ScrapeResult(this.name);
const imgLinks = [] as string[];
// Query for when the user is on the desktop site, is logged in, and has optionally clicked on "See all".
// If the user has not clicked on "See all" then this will only grab the first post.
// Query for when the user is on the desktop site, is logged in, and has optionally clicked on "Show all".
// If the user has not clicked on "Show all" then this will only grab the first post.
imgLinks.push(
...Array.from(document.querySelectorAll<HTMLAnchorElement>("div[role='presentation'] > a")).map((x) => x.href)
);
if (imgLinks.length == 0) {
// Query for when the user is on the desktop site and not logged in.
imgLinks.push(...Array.from(document.querySelectorAll<HTMLImageElement>("img[srcset]")).map((x) => x.src));
imgLinks.push(
...Array.from(document.querySelectorAll<HTMLImageElement>("div[role='presentation'] img")).map((x) => x.src)
);
}
if (imgLinks.length == 0) {
// Query for when the user is on the mobile site, and has clicked on "See all".
// Query for when the user is on the mobile site, and has clicked on "Show all".
// This only works on pages with multiple posts. The user can optionally be logged in.
imgLinks.push(
...Array.from(document.querySelectorAll<HTMLImageElement>(".manga-page > a > img[data-big]")).map(

15
src/engines/SankakuComplex.test.ts
View File

@ -1,37 +1,30 @@
import { scrapeUrl } from "../test/engine.js";
test("get chan.sankakucomplex.com post 19614481 with notes", async () => {
const pageUrl = "https://chan.sankakucomplex.com/post/show/19614481";
const pageUrl = "https://chan.sankakucomplex.com/post/show/507e986a62cc5b866d92ad491dcfb1a4";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("bangs");
expect(res.posts[0]).toHaveTag("bare_shoulders");
expect(res.posts[0]).toHaveTag("solo");
expect(res.posts[0]).toHaveTag("translated");
expect(res.posts[0]).toHaveTag("text");
expect(res.posts[0]).toHaveTag("idolmaster");
expect(res.posts[0]).toHaveTag("mayuzumi_fuyuko");
expect(res.posts[0]).toHaveNote("What are you looking at...");
expect(res.posts[0]).toHaveNote("What...?");
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toContain(
"https://s.sankakucomplex.com/data/50/7e/507e986a62cc5b866d92ad491dcfb1a4.jpg"
);
expect(res.posts[0].contentUrl).toContain("sankakucomplex.com/data/50/7e/507e986a62cc5b866d92ad491dcfb1a4.jpg");
});
test("get chan.sankakucomplex.com post 29354061 with notes", async () => {
// Kinda sketchy
const pageUrl = "https://chan.sankakucomplex.com/post/show/29354061";
const pageUrl = "https://chan.sankakucomplex.com/post/show/c8376f02608548365d394a658e85cb3f";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("bangs");
expect(res.posts[0]).toHaveTag("braid");
expect(res.posts[0]).toHaveTag("sankuro");
expect(res.posts[0]).toHaveTag("translated");
expect(res.posts[0]).toHaveTag("note_translated");
expect(res.posts[0]).toHaveNote("No. \nI already told you...");
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toContain(
"https://s.sankakucomplex.com/data/c8/37/c8376f02608548365d394a658e85cb3f.jpg"
);
expect(res.posts[0].contentUrl).toContain("sankakucomplex.com/data/c8/37/c8376f02608548365d394a658e85cb3f.jpg");
});

3
src/engines/Zerochan.ts
View File

@ -52,7 +52,8 @@ export default class Zerochan extends ScrapeEngineBase {
}
let category: TagCategory | undefined;
switch (tagElement.className) {
const className = tagElement.className.replace("emphasis", "").trim();
switch (className) {
case "game":
case "series":
case "studio":

3
src/test/engine.ts
View File

@ -1,5 +1,4 @@
import { JSDOM, ConstructorOptions } from "jsdom";
import { Page } from "puppeteer";
import NeoScraper from "../NeoScraper.js";
export const MobileUserAgent = "Mozilla/5.0 (Android 12; Mobile; rv:68.0) Gecko/68.0 Firefox/103.0";
@ -32,7 +31,7 @@ export async function scrapeEvalUrl(url: string, waitForSelector: string, userAg
return scrapePage(url, page);
}
export async function scrapePage(url: string, page: Page) {
export async function scrapePage(url: string, page: any) {
const document = await page.content();
return scrapeHtml(url, document);
}