Nes
/
neo-scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update tests

pull/16/head
neobooru 2021-11-20 15:19:19 +01:00
parent b26f2862ef
commit 5c395475e4
7 changed files with 2430 additions and 2232 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,3 +1,4 @@
.vscode/ .vscode/
node_modules/ node_modules/
dist/ dist/
coverage/

4
jest.preset.js Normal file
View File

@ -0,0 +1,4 @@
const ts_preset = require("ts-jest/jest-preset");
const puppeteer_preset = require("jest-puppeteer/jest-preset");
module.exports = Object.assign(ts_preset, puppeteer_preset);

14
package.json
View File

@ -13,15 +13,21 @@
"author": "neobooru", "author": "neobooru",
"license": "MIT", "license": "MIT",
"devDependencies": { "devDependencies": {
"@types/jest": "^24.9.1", "@types/expect-puppeteer": "^4.4.7",
"@types/jest": "^27.0.3",
"@types/jest-environment-puppeteer": "^4.4.1",
"@types/jsdom": "^12.2.4", "@types/jsdom": "^12.2.4",
"@types/puppeteer": "^5.4.4",
"@types/turndown": "^5.0.1", "@types/turndown": "^5.0.1",
"jest": "^24.9.0", "jest": "^27.3.1",
"jest-puppeteer": "^6.0.0",
"jsdom": "^15.2.1", "jsdom": "^15.2.1",
"ts-jest": "^24.3.0", "puppeteer": "^11.0.0",
"typescript": "^3.8.3" "ts-jest": "^27.0.7",
"typescript": "^4.5.2"
}, },
"jest": { "jest": {
"preset": "./jest.preset.js",
"transform": { "transform": {
"^.+\\.tsx?$": "ts-jest" "^.+\\.tsx?$": "ts-jest"
}, },

254
src/NeoScraper.test.ts
View File

@ -1,10 +1,27 @@
import { JSDOM } from "jsdom"; import { ConstructorOptions, JSDOM } from "jsdom";
import NeoScraper from "./NeoScraper"; import NeoScraper from "./NeoScraper";
import { ScrapeResults } from "./ScrapeEngine";
async function scrapeUrl(url: string) { async function scrapeUrl(url: string) {
const scraper = new NeoScraper();
const dom = await JSDOM.fromURL(url); const dom = await JSDOM.fromURL(url);
return scraper.scrapeDocument(dom.window.document); return scrapeDom(dom.window.document);
}
async function scrapeEvalUrl(url: string, waitForSelector: string) {
await page.goto(url);
await page.waitForSelector(waitForSelector, { visible: true });
const document = await page.content();
return scrapeHtml(url, document);
}
async function scrapeHtml(url: string, html: string) {
const dom = new JSDOM(html, <ConstructorOptions>{ url });
return scrapeDom(dom.window.document);
}
async function scrapeDom(document: Document) {
const scraper = new NeoScraper();
return scraper.scrapeDocument(document);
} }
beforeAll(() => { beforeAll(() => {
@ -37,119 +54,156 @@ test("all default scrapers have unique names", () => {
} }
}); });
// Danbooru
// All links are SFW unless stated otherwise // All links are SFW unless stated otherwise
test("get danbooru.donmai.us post 3555872", async () => { // Anime pictures
var pageUrl = "https://danbooru.donmai.us/posts/3555872"; describe("anime-pictures.net", () => {
const res = await scrapeUrl(pageUrl); test("get anime-pictures.net post 729833", async () => {
expect(res.posts.length).toBe(1); const pageUrl = "https://anime-pictures.net/pictures/view_post/729833?lang=en";
expect(res.posts[0].pageUrl).toBe(pageUrl); const res = await scrapeUrl(pageUrl);
expect(res.posts[0].contentUrl).toBe( expect(res.posts.length).toBe(1);
"https://cdn.donmai.us/original/90/33/__flint_girls_und_panzer_and_1_more_drawn_by_omachi_slabco__90332299216db81b8793084e4cb7c15d.jpg" expect(res.posts[0].pageUrl).toBe(pageUrl);
); expect(res.posts[0].contentUrl).toBe(
"https://anime-pictures.net/pictures/download_image/729833-4000x6865-arknights-ch%26%2339%3Ben+%28arknights%29-ttk+%28kirinottk%29-single-long+hair-tall+image.jpeg"
);
});
}); });
test("get danbooru.donmai.us post 4924490", async () => { describe("danbooru", () => {
var pageUrl = "https://danbooru.donmai.us/posts/4924490"; test("get danbooru.donmai.us post 3555872", async () => {
const res = await scrapeUrl(pageUrl); const pageUrl = "https://danbooru.donmai.us/posts/3555872";
expect(res.posts.length).toBe(1); const res = await scrapeUrl(pageUrl);
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts.length).toBe(1);
expect(res.posts[0].contentUrl).toBe( expect(res.posts[0].pageUrl).toBe(pageUrl);
"https://cdn.donmai.us/original/c5/fb/__yuuki_and_tamaki_princess_connect_drawn_by_yuureidoushi_yuurei6214__c5fbc62758d0f1d8c07aac524282434e.png" expect(res.posts[0].contentUrl).toBe(
); "https://cdn.donmai.us/original/90/33/__flint_girls_und_panzer_and_1_more_drawn_by_omachi_slabco__90332299216db81b8793084e4cb7c15d.jpg"
expect(res.posts[0].notes).toHaveLength(17); );
expect(res.posts[0].notes[0].text).toBe("Nya~nya ♪"); });
test("get danbooru.donmai.us post 4924490", async () => {
const pageUrl = "https://danbooru.donmai.us/posts/4924490";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://cdn.donmai.us/original/c5/fb/__yuuki_and_tamaki_princess_connect_drawn_by_yuureidoushi_yuurei6214__c5fbc62758d0f1d8c07aac524282434e.png"
);
expect(res.posts[0].notes).toHaveLength(17);
expect(res.posts[0].notes[0].text).toBe("Nya~nya ♪");
});
}); });
// Gelbooru describe("gelbooru", () => {
test("get safebooru.org post 2871532", async () => {
const pageUrl = "https://safebooru.org/index.php?page=post&s=view&id=2871532";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://safebooru.org//images/2757/5f19c7744db918c3d4c4c6035bc65dfa9e18866f.jpg"
);
});
test("get safebooru.org post 2871532", async () => { test("get gelbooru.com post 4925055", async () => {
var pageUrl = "https://safebooru.org/index.php?page=post&s=view&id=2871532"; const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=4925055&tags=kimetsu_no_yaiba";
const res = await scrapeUrl(pageUrl); const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe( expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
"https://safebooru.org//images/2757/5f19c7744db918c3d4c4c6035bc65dfa9e18866f.jpg" });
);
test("get rule34.xxx post 2740994", async () => {
const pageUrl = "https://rule34.xxx/index.php?page=post&s=view&id=2740994";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toContain(".rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg");
});
}); });
test("get gelbooru.com post 4925055", async () => { describe("moebooru", () => {
var pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=4925055&tags=kimetsu_no_yaiba"; test("get yande.re post 438510", async () => {
const res = await scrapeUrl(pageUrl); const pageUrl = "https://yande.re/post/show/438510";
expect(res.posts.length).toBe(1); const res = await scrapeUrl(pageUrl);
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts.length).toBe(1);
expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg"); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://files.yande.re/image/49b79bc372716a579d8525dcaccb56e2/yande.re%20438510%20landscape%20link%20nintendo%20princess_zelda%20sword%20the_legend_of_zelda%20the_legend_of_zelda%3A_breath_of_the_wild.jpg"
);
});
test("get konachan.com post 289968", async () => {
const pageUrl = "https://konachan.com/post/show/289968/ahri_-league_of_legends-akali-animal_ears-blonde_h";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://konachan.com/image/e40b38fb704434e4da3eae41e26bca39/Konachan.com%20-%20289968%20akali%20blue_eyes%20breasts%20choker%20cleavage%20evelynn%20foxgirl%20gloves%20group%20hat%20kai%27sa%20long_hair%20microphone%20ponytail%20saphirya%20short_hair%20signed%20watermark.jpg"
);
});
}); });
test("get rule34.xxx post 2740994", async () => { // Commented out because it doesn't work.
var pageUrl = "https://rule34.xxx/index.php?page=post&s=view&id=2740994"; // describe("pixiv", () => {
const res = await scrapeUrl(pageUrl); // describe("get pixiv post 93964654", () => {
expect(res.posts.length).toBe(1); // const pageUrl = "https://www.pixiv.net/en/artworks/93964654";
expect(res.posts[0].pageUrl).toBe(pageUrl); // let res: ScrapeResults;
expect(res.posts[0].contentUrl).toContain(".rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg");
});
// Moebooru // beforeAll(async () => {
// res = await scrapeEvalUrl(pageUrl, ".gtm-expand-full-size-illust");
// });
test("get yande.re post 438510", async () => { // it("should grab original image url", async () => {
var pageUrl = "https://yande.re/post/show/438510"; // expect(res.posts.length).toBe(1);
const res = await scrapeUrl(pageUrl); // expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts.length).toBe(1); // expect(res.posts[0].contentUrl).toBe("https://i.pximg.net/img-original/img/2021/11/07/00/01/02/93964654_p0.jpg");
expect(res.posts[0].pageUrl).toBe(pageUrl); // });
expect(res.posts[0].contentUrl).toBe( // });
"https://files.yande.re/image/49b79bc372716a579d8525dcaccb56e2/yande.re%20438510%20landscape%20link%20nintendo%20princess_zelda%20sword%20the_legend_of_zelda%20the_legend_of_zelda%3A_breath_of_the_wild.jpg"
);
});
test("get konachan.com post 289968", async () => {
var pageUrl = "https://konachan.com/post/show/289968/ahri_-league_of_legends-akali-animal_ears-blonde_h";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://konachan.com/image/e40b38fb704434e4da3eae41e26bca39/Konachan.com%20-%20289968%20akali%20blue_eyes%20breasts%20choker%20cleavage%20evelynn%20foxgirl%20gloves%20group%20hat%20kai%27sa%20long_hair%20microphone%20ponytail%20saphirya%20short_hair%20signed%20watermark.jpg"
);
});
// Reddit
test("get old.reddit.com post", async () => {
var pageUrl = "https://old.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe("https://i.redd.it/ln8ahz17kpa31.jpg");
});
// test("get new.reddit.com post", async () => {
// var pageUrl = "https://new.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/";
// const res = await scrapeUrl(pageUrl);
// expect(res.posts.length).toBe(1);
// expect(res.posts[0].pageUrl).toBe(pageUrl);
// expect(res.posts[0].contentUrl).toBe("https://i.redd.it/ln8ahz17kpa31.jpg");
// }); // });
// Shimmie2 describe("reddit", () => {
test("get old.reddit.com post", async () => {
const pageUrl = "https://old.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe("https://i.redd.it/ln8ahz17kpa31.jpg");
});
test("get rule34.paheal.net post 2505982", async () => { // test("get new.reddit.com post", async () => {
var pageUrl = "https://rule34.paheal.net/post/view/2505982"; // Somewhat sketchy // const pageUrl = "https://new.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/";
const res = await scrapeUrl(pageUrl); // const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1); // expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl); // expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toContain( // expect(res.posts[0].contentUrl).toBe("https://i.redd.it/ln8ahz17kpa31.jpg");
".paheal.net/_images/15427ab5ed2cddbeef947e30f5c9cab6/2505982%20-%20Kai%27Sa%20League_of_Legends%20lawzilla.png" // });
);
}); });
// Twitter describe("shimmie2", () => {
test("get rule34.paheal.net post 2505982", async () => {
test("get twitter.com post", async () => { const pageUrl = "https://rule34.paheal.net/post/view/2505982"; // Somewhat sketchy
pending("JSDOM can't parse this page"); const res = await scrapeUrl(pageUrl);
// TODO: JSDOM sucks expect(res.posts.length).toBe(1);
// var pageUrl = "https://twitter.com/Dishwasher1910/status/1171162651461746688/photo/1"; expect(res.posts[0].pageUrl).toBe(pageUrl);
// const res = await scrapeUrl(pageUrl); expect(res.posts[0].contentUrl).toContain(
// expect(res.posts.length).toBe(1); ".paheal.net/_images/15427ab5ed2cddbeef947e30f5c9cab6/2505982%20-%20Kai%27Sa%20League_of_Legends%20lawzilla.png"
// expect(res.posts[0].pageUrl).toBe(pageUrl); );
// expect(res.posts[0].contentUrl).toBe("https://pbs.twimg.com/media/EEDOOCdU0AAHX00?format=jpg&name=orig"); });
});
describe("twitter", () => {
describe("get twitter.com post", () => {
const pageUrl = "https://twitter.com/Dishwasher1910/status/1171162651461746688/photo/1";
let res: ScrapeResults;
beforeAll(async () => {
res = await scrapeEvalUrl(pageUrl, "img[alt='Image']");
});
it("should grab original image url", async () => {
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe("https://pbs.twimg.com/media/EEDOOCdU0AAHX00?format=jpg&name=orig");
});
});
}); });

1
src/engines/AnimePictures.ts
View File

@ -12,6 +12,7 @@ export default class AnimePictures implements ScrapeEngine {
scrapeDocument(document: Document): ScrapeResult { scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name); let result = new ScrapeResult(this.name);
let post = new ScrapedPost(); let post = new ScrapedPost();
post.pageUrl = document.location.href;
post.source = document.location.href; post.source = document.location.href;
// Set image url // Set image url

2
tsconfig.json
View File

@ -19,4 +19,4 @@
"include": [ "include": [
"src/index.ts" "src/index.ts"
] ]
} }

4386
yarn.lock
View File

File diff suppressed because it is too large Load Diff