Nes
/
neo-scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix zerochan and new.reddit + add/cleanup tests

pull/16/head
neobooru 2021-11-22 00:28:32 +01:00
parent 32713f5ff4
commit b222782b21
22 changed files with 1605 additions and 703 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "neo-scraper", "name": "neo-scraper",
"version": "0.5.0", "version": "0.5.1",
"description": "Scrape a booru-like post object from the DOM.", "description": "Scrape a booru-like post object from the DOM.",
"main": "dist/index.js", "main": "dist/index.js",
"types": "dist/index.d.js", "types": "dist/index.d.js",
@ -14,16 +14,16 @@
"license": "MIT", "license": "MIT",
"devDependencies": { "devDependencies": {
"@types/expect-puppeteer": "^4.4.7", "@types/expect-puppeteer": "^4.4.7",
"@types/jest": "^27.0.3", "@types/jest": "^26.0.14",
"@types/jest-environment-puppeteer": "^4.4.1", "@types/jest-environment-puppeteer": "^4.4.1",
"@types/jsdom": "^12.2.4", "@types/jsdom": "^12.2.4",
"@types/puppeteer": "^5.4.4", "@types/puppeteer": "^5.4.4",
"@types/turndown": "^5.0.1", "@types/turndown": "^5.0.1",
"jest": "^27.3.1", "jest": "^26.6.3",
"jest-puppeteer": "^6.0.0", "jest-puppeteer": "^6.0.0",
"jsdom": "^15.2.1", "jsdom": "^15.2.1",
"puppeteer": "^11.0.0", "puppeteer": "^11.0.0",
"ts-jest": "^27.0.7", "ts-jest": "^26.5.6",
"typescript": "^4.5.2" "typescript": "^4.5.2"
}, },
"jest": { "jest": {
@ -35,7 +35,10 @@
"ts", "ts",
"js" "js"
], ],
"testRegex": "^.+\\.test\\.ts$" "testRegex": "^.+\\.test\\.ts$",
"setupFilesAfterEnv": [
"./src/test/jest.setup.ts"
]
}, },
"dependencies": { "dependencies": {
"turndown": "^7.1.1" "turndown": "^7.1.1"

191
src/NeoScraper.test.ts
View File

@ -1,41 +1,4 @@
import { ConstructorOptions, JSDOM } from "jsdom";
import NeoScraper from "./NeoScraper"; import NeoScraper from "./NeoScraper";
import { ScrapeResults } from "./ScrapeEngine";
async function scrapeUrl(url: string) {
const dom = await JSDOM.fromURL(url);
return scrapeDom(dom.window.document);
}
async function scrapeEvalUrl(url: string, waitForSelector: string) {
await page.goto(url);
await page.waitForSelector(waitForSelector, { visible: true });
const document = await page.content();
return scrapeHtml(url, document);
}
async function scrapeHtml(url: string, html: string) {
const dom = new JSDOM(html, <ConstructorOptions>{ url });
return scrapeDom(dom.window.document);
}
async function scrapeDom(document: Document) {
const scraper = new NeoScraper();
return scraper.scrapeDocument(document);
}
beforeAll(() => {
// Expose JSDOM Element constructor
// @ts-ignore
global.Element = new JSDOM().window.Element;
// 'Implement' innerText in JSDOM: https://github.com/jsdom/jsdom/issues/1245
// @ts-ignore
Object.defineProperty(global.Element.prototype, "innerText", {
get() {
return this.textContent.replace(/(\r\n|\n|\r|^\s+|\s+$)/gm, "");
},
});
});
test("create scraper with default engines", async () => { test("create scraper with default engines", async () => {
const scraper = new NeoScraper(); const scraper = new NeoScraper();
@ -53,157 +16,3 @@ test("all default scrapers have unique names", () => {
} }
} }
}); });
// All links are SFW unless stated otherwise
// Anime pictures
describe("anime-pictures.net", () => {
test("get anime-pictures.net post 729833", async () => {
const pageUrl = "https://anime-pictures.net/pictures/view_post/729833?lang=en";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://anime-pictures.net/pictures/download_image/729833-4000x6865-arknights-ch%26%2339%3Ben+%28arknights%29-ttk+%28kirinottk%29-single-long+hair-tall+image.jpeg"
);
});
});
describe("danbooru", () => {
test("get danbooru.donmai.us post 3555872", async () => {
const pageUrl = "https://danbooru.donmai.us/posts/3555872";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://cdn.donmai.us/original/90/33/__flint_girls_und_panzer_and_1_more_drawn_by_omachi_slabco__90332299216db81b8793084e4cb7c15d.jpg"
);
});
test("get danbooru.donmai.us post 4924490", async () => {
const pageUrl = "https://danbooru.donmai.us/posts/4924490";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://cdn.donmai.us/original/c5/fb/__yuuki_and_tamaki_princess_connect_drawn_by_yuureidoushi_yuurei6214__c5fbc62758d0f1d8c07aac524282434e.png"
);
expect(res.posts[0].notes).toHaveLength(17);
expect(res.posts[0].notes[0].text).toBe("Nya~nya ♪");
});
});
describe("gelbooru", () => {
test("get safebooru.org post 2871532", async () => {
const pageUrl = "https://safebooru.org/index.php?page=post&s=view&id=2871532";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://safebooru.org//images/2757/5f19c7744db918c3d4c4c6035bc65dfa9e18866f.jpg"
);
});
test("get gelbooru.com post 4925055", async () => {
const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=4925055&tags=kimetsu_no_yaiba";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
});
test("get rule34.xxx post 2740994", async () => {
const pageUrl = "https://rule34.xxx/index.php?page=post&s=view&id=2740994";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toContain(".rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg");
});
});
describe("moebooru", () => {
test("get yande.re post 438510", async () => {
const pageUrl = "https://yande.re/post/show/438510";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://files.yande.re/image/49b79bc372716a579d8525dcaccb56e2/yande.re%20438510%20landscape%20link%20nintendo%20princess_zelda%20sword%20the_legend_of_zelda%20the_legend_of_zelda%3A_breath_of_the_wild.jpg"
);
});
test("get konachan.com post 289968", async () => {
const pageUrl = "https://konachan.com/post/show/289968/ahri_-league_of_legends-akali-animal_ears-blonde_h";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://konachan.com/image/e40b38fb704434e4da3eae41e26bca39/Konachan.com%20-%20289968%20akali%20blue_eyes%20breasts%20choker%20cleavage%20evelynn%20foxgirl%20gloves%20group%20hat%20kai%27sa%20long_hair%20microphone%20ponytail%20saphirya%20short_hair%20signed%20watermark.jpg"
);
});
});
// Commented out because it doesn't work.
// describe("pixiv", () => {
// describe("get pixiv post 93964654", () => {
// const pageUrl = "https://www.pixiv.net/en/artworks/93964654";
// let res: ScrapeResults;
// beforeAll(async () => {
// res = await scrapeEvalUrl(pageUrl, ".gtm-expand-full-size-illust");
// });
// it("should grab original image url", async () => {
// expect(res.posts.length).toBe(1);
// expect(res.posts[0].pageUrl).toBe(pageUrl);
// expect(res.posts[0].contentUrl).toBe("https://i.pximg.net/img-original/img/2021/11/07/00/01/02/93964654_p0.jpg");
// });
// });
// });
describe("reddit", () => {
test("get old.reddit.com post", async () => {
const pageUrl = "https://old.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe("https://i.redd.it/ln8ahz17kpa31.jpg");
});
// test("get new.reddit.com post", async () => {
// const pageUrl = "https://new.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/";
// const res = await scrapeUrl(pageUrl);
// expect(res.posts.length).toBe(1);
// expect(res.posts[0].pageUrl).toBe(pageUrl);
// expect(res.posts[0].contentUrl).toBe("https://i.redd.it/ln8ahz17kpa31.jpg");
// });
});
describe("shimmie2", () => {
test("get rule34.paheal.net post 2505982", async () => {
const pageUrl = "https://rule34.paheal.net/post/view/2505982"; // Somewhat sketchy
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toContain(
".paheal.net/_images/15427ab5ed2cddbeef947e30f5c9cab6/2505982%20-%20Kai%27Sa%20League_of_Legends%20lawzilla.png"
);
});
});
describe("twitter", () => {
describe("get twitter.com post", () => {
const pageUrl = "https://twitter.com/Dishwasher1910/status/1171162651461746688/photo/1";
let res: ScrapeResults;
beforeAll(async () => {
res = await scrapeEvalUrl(pageUrl, "img[alt='Image']");
});
it("should grab original image url", async () => {
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe("https://pbs.twimg.com/media/EEDOOCdU0AAHX00?format=jpg&name=orig");
});
});
});

2
src/ScrapeEngine.ts
View File

@ -39,7 +39,7 @@ export class ScrapeResult {
tryAddPost(post: ScrapedPost) { tryAddPost(post: ScrapedPost) {
if (!post.contentUrl) { if (!post.contentUrl) {
console.log(`[${this.engine}] Not adding post because imageUrl is unset!`); console.log(`[${this.engine}] Not adding post because contentUrl is unset!`);
} else { } else {
this.posts.push(post); this.posts.push(post);
} }

11
src/engines/AnimePictures.test.ts Normal file
View File

@ -0,0 +1,11 @@
import { scrapeUrl } from "../test/engine";
test("get anime-pictures.net post 729833", async () => {
const pageUrl = "https://anime-pictures.net/pictures/view_post/729833?lang=en";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://anime-pictures.net/pictures/download_image/729833-4000x6865-arknights-ch%26%2339%3Ben+%28arknights%29-ttk+%28kirinottk%29-single-long+hair-tall+image.jpeg"
);
});

23
src/engines/Danbooru.test.ts Normal file
View File

@ -0,0 +1,23 @@
import { scrapeUrl } from "../test/engine";
test("get danbooru.donmai.us post 3555872", async () => {
const pageUrl = "https://danbooru.donmai.us/posts/3555872";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://cdn.donmai.us/original/90/33/__flint_girls_und_panzer_and_1_more_drawn_by_omachi_slabco__90332299216db81b8793084e4cb7c15d.jpg"
);
});
test("get danbooru.donmai.us post 4924490", async () => {
const pageUrl = "https://danbooru.donmai.us/posts/4924490";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://cdn.donmai.us/original/c5/fb/__yuuki_and_tamaki_princess_connect_drawn_by_yuureidoushi_yuurei6214__c5fbc62758d0f1d8c07aac524282434e.png"
);
expect(res.posts[0].notes).toHaveLength(17);
expect(res.posts[0].notes[0].text).toBe("Nya~nya ♪");
});

4
src/engines/FurAffinity.ts
View File

@ -19,10 +19,6 @@ export default class e621 implements ScrapeEngine {
const downloadEl = document.querySelector("#submissionImg") as HTMLImageElement; const downloadEl = document.querySelector("#submissionImg") as HTMLImageElement;
post.contentUrl = downloadEl?.src; post.contentUrl = downloadEl?.src;
if (post.contentUrl == undefined) {
return result;
}
// Set content type // Set content type
post.contentType = guessContentType(post.contentUrl); post.contentType = guessContentType(post.contentUrl);

27
src/engines/Gelbooru.test.ts Normal file
View File

@ -0,0 +1,27 @@
import { scrapeUrl } from "../test/engine";
test("get safebooru.org post 2871532", async () => {
const pageUrl = "https://safebooru.org/index.php?page=post&s=view&id=2871532";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://safebooru.org//images/2757/5f19c7744db918c3d4c4c6035bc65dfa9e18866f.jpg"
);
});
test("get gelbooru.com post 4925055", async () => {
const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=4925055&tags=kimetsu_no_yaiba";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
});
test("get rule34.xxx post 2740994", async () => {
const pageUrl = "https://rule34.xxx/index.php?page=post&s=view&id=2740994";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toContain(".rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg");
});

4
src/engines/Gelbooru.ts
View File

@ -53,10 +53,6 @@ export default class Gelbooru extends ScrapeEngineBase {
.filter((x) => x && x.innerText == "Original image") .filter((x) => x && x.innerText == "Original image")
.map((x) => (post.contentUrl = x.href)); .map((x) => (post.contentUrl = x.href));
if (post.contentUrl == undefined) {
return result;
}
// Set content type // Set content type
if (document.getElementById("gelcomVideoPlayer") != null) { if (document.getElementById("gelcomVideoPlayer") != null) {
post.contentType = "video"; post.contentType = "video";

21
src/engines/Moebooru.test.ts Normal file
View File

@ -0,0 +1,21 @@
import { scrapeUrl } from "../test/engine";
test("get yande.re post 438510", async () => {
const pageUrl = "https://yande.re/post/show/438510";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://files.yande.re/image/49b79bc372716a579d8525dcaccb56e2/yande.re%20438510%20landscape%20link%20nintendo%20princess_zelda%20sword%20the_legend_of_zelda%20the_legend_of_zelda%3A_breath_of_the_wild.jpg"
);
});
test("get konachan.com post 289968", async () => {
const pageUrl = "https://konachan.com/post/show/289968/ahri_-league_of_legends-akali-animal_ears-blonde_h";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://konachan.com/image/e40b38fb704434e4da3eae41e26bca39/Konachan.com%20-%20289968%20akali%20blue_eyes%20breasts%20choker%20cleavage%20evelynn%20foxgirl%20gloves%20group%20hat%20kai%27sa%20long_hair%20microphone%20ponytail%20saphirya%20short_hair%20signed%20watermark.jpg"
);
});

18
src/engines/Pixiv.test.ts Normal file
View File

@ -0,0 +1,18 @@
import { ScrapeResults } from "../ScrapeEngine";
import { scrapeEvalUrl } from "../test/engine";
describe("get pixiv post 93964654", () => {
const pageUrl = "https://www.pixiv.net/en/artworks/93964654";
let res: ScrapeResults;
beforeAll(async () => {
res = await scrapeEvalUrl(pageUrl, ".gtm-expand-full-size-illust");
});
it("should grab original image url", async () => {
pending("Test does not work.");
// expect(res.posts.length).toBe(1);
// expect(res.posts[0].pageUrl).toBe(pageUrl);
// expect(res.posts[0].contentUrl).toBe("https://i.pximg.net/img-original/img/2021/11/07/00/01/02/93964654_p0.jpg");
});
});

17
src/engines/Reddit.test.ts Normal file
View File

@ -0,0 +1,17 @@
import { scrapeUrl } from "../test/engine";
test("get old.reddit.com post", async () => {
const pageUrl = "https://old.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe("https://i.redd.it/ln8ahz17kpa31.jpg");
});
test("get new.reddit.com post", async () => {
const pageUrl = "https://new.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe("https://i.redd.it/ln8ahz17kpa31.jpg");
});

6
src/engines/Reddit.ts
View File

@ -21,9 +21,9 @@ export default class Reddit implements ScrapeEngine {
} }
// For new reddit // For new reddit
const newLinkElements = document.querySelectorAll("div[data-test-id='post-content'] > div > a"); const newLinkEl = document.querySelector("div[data-test-id='post-content'] img.ImageBox-image")?.parentElement;
if (newLinkElements.length > 0) { if (newLinkEl) {
post.contentUrl = (newLinkElements[0] as HTMLAnchorElement).href; post.contentUrl = (newLinkEl as HTMLAnchorElement).href;
} }
if (post.contentUrl != undefined) { if (post.contentUrl != undefined) {

11
src/engines/Shimmie2.test.ts Normal file
View File

@ -0,0 +1,11 @@
import { scrapeUrl } from "../test/engine";
test("get rule34.paheal.net post 2505982", async () => {
const pageUrl = "https://rule34.paheal.net/post/view/2505982"; // Somewhat sketchy
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toContain(
".paheal.net/_images/15427ab5ed2cddbeef947e30f5c9cab6/2505982%20-%20Kai%27Sa%20League_of_Legends%20lawzilla.png"
);
});

4
src/engines/Shimmie2.ts
View File

@ -27,10 +27,6 @@ export default class Shimmie2 implements ScrapeEngine {
} }
} }
if (post.contentUrl == undefined) {
return result;
}
// Set content type // Set content type
post.contentType = guessContentType(post.contentUrl); post.contentType = guessContentType(post.contentUrl);

9
src/engines/Shuushuu.test.ts Normal file
View File

@ -0,0 +1,9 @@
import { scrapeUrl } from "../test/engine";
test("get e-shuushuu.net post 1063586", async () => {
const pageUrl = "https://e-shuushuu.net/image/1063586/";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe("https://e-shuushuu.net/images/2021-08-28-1063586.jpeg");
});

17
src/engines/Twitter.test.ts Normal file
View File

@ -0,0 +1,17 @@
import { ScrapeResults } from "../ScrapeEngine";
import { scrapeEvalUrl } from "../test/engine";
describe("get twitter.com post", () => {
const pageUrl = "https://twitter.com/Dishwasher1910/status/1171162651461746688/photo/1";
let res: ScrapeResults;
beforeAll(async () => {
res = await scrapeEvalUrl(pageUrl, "img[alt='Image']");
});
it("should grab original image url", async () => {
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe("https://pbs.twimg.com/media/EEDOOCdU0AAHX00?format=jpg&name=orig");
});
});

19
src/engines/Zerochan.test.ts Normal file
View File

@ -0,0 +1,19 @@
import { scrapeUrl } from "../test/engine";
test("get zerochan.net post 2934998", async () => {
const pageUrl = "https://www.zerochan.net/2934998";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].tags.find((x) => x.name == "nintendo")).not.toBeUndefined();
expect(res.posts[0].contentUrl).toBe("https://static.zerochan.net/Shirona.%28Pok%C3%A9mon%29.full.2934998.png");
});
test("get zerochan.net gif post 3511809", async () => {
const pageUrl = "https://www.zerochan.net/3511809";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0]).toHaveTag("rwby");
expect(res.posts[0].contentUrl).toBe("https://static.zerochan.net/Ruby.Rose.full.3511809.gif");
});

20
src/engines/Zerochan.ts
View File

@ -27,11 +27,6 @@ export default class Zerochan implements ScrapeEngine {
} }
} }
if (post.contentUrl == undefined) {
// No point in continuing when we don't have an image.
return result;
}
// Set content type // Set content type
post.contentType = guessContentType(post.contentUrl); post.contentType = guessContentType(post.contentUrl);
@ -39,21 +34,20 @@ export default class Zerochan implements ScrapeEngine {
post.rating = "safe"; post.rating = "safe";
// Set tags // Set tags
const tagElements = Array.from(document.getElementById("tags")!.children); const tagElements = Array.from(document.querySelectorAll("#tags > li"));
for (const tagElement of tagElements) { for (const tagElement of tagElements) {
// Get tag name from url, because some longer tags are shortened. // Get tag name from url, because some longer tags are shortened.
// Eg. "Tate no Yuusha no Nariagari" -> "Tate no Yuusha no Nariaga..." // Eg. "Tate no Yuusha no Nariagari" -> "Tate no Yuusha no Nariaga..."
// (ironically this is longer, but you get the idea) // (ironically this is longer, but you get the idea)
const tagUrl = (tagElement.children[0] as HTMLAnchorElement).pathname.substr(1); let tagName: string | undefined = undefined;
const tagName = decodeURIComponent(tagUrl.replace(/\+/g, "_")).toLowerCase(); const tagUrl = (tagElement.querySelector("a") as HTMLAnchorElement | undefined)?.pathname?.substr(1);
if (tagUrl) {
let tagType = ""; tagName = decodeURIComponent(tagUrl.replace(/\+/g, "_")).toLowerCase();
if (tagElement.childNodes.length > 1 && tagElement.childNodes[1].textContent) }
tagType = tagElement.childNodes[1].textContent.trim().toLowerCase();
let category: TagCategory | undefined; let category: TagCategory | undefined;
switch (tagType) { switch (tagElement.className) {
case "game": case "game":
case "series": case "series":
category = "copyright"; category = "copyright";

4
src/engines/e621.ts
View File

@ -18,10 +18,6 @@ export default class e621 implements ScrapeEngine {
const downloadEl = document.querySelector("#image-download-link > a") as HTMLAnchorElement; const downloadEl = document.querySelector("#image-download-link > a") as HTMLAnchorElement;
post.contentUrl = downloadEl?.href; post.contentUrl = downloadEl?.href;
if (post.contentUrl == undefined) {
return result;
}
// Set content type // Set content type
if (document.querySelector("video#image") != null) { if (document.querySelector("video#image") != null) {
post.contentType = "video"; post.contentType = "video";

35
src/test/engine.ts Normal file
View File

@ -0,0 +1,35 @@
import { JSDOM, ConstructorOptions } from "jsdom";
import NeoScraper from "../NeoScraper";
/**
* Go to an URL and parse it's raw HTML without executing any client-side code.
* @param url
* @returns
*/
export async function scrapeUrl(url: string) {
const dom = await JSDOM.fromURL(url);
return scrapeDom(dom.window.document);
}
/**
* Go to an URL and evaluate the page using Puppeteer.
* @param url
* @param waitForSelector
* @returns
*/
export async function scrapeEvalUrl(url: string, waitForSelector: string) {
await page.goto(url);
await page.waitForSelector(waitForSelector, { visible: true });
const document = await page.content();
return scrapeHtml(url, document);
}
async function scrapeHtml(url: string, html: string) {
const dom = new JSDOM(html, <ConstructorOptions>{ url });
return scrapeDom(dom.window.document);
}
async function scrapeDom(document: Document) {
const scraper = new NeoScraper();
return scraper.scrapeDocument(document);
}

34
src/test/jest.setup.ts Normal file
View File

@ -0,0 +1,34 @@
import { JSDOM } from "jsdom";
import { ScrapedPost } from "../ScrapeEngine";
// Expose JSDOM Element constructor
// @ts-ignore
global.Element = new JSDOM().window.Element;
// 'Implement' innerText in JSDOM: https://github.com/jsdom/jsdom/issues/1245
// @ts-ignore
Object.defineProperty(global.Element.prototype, "innerText", {
get() {
return this.textContent.replace(/(\r\n|\n|\r|^\s+|\s+$)/gm, "");
},
});
declare global {
namespace jest {
interface Matchers<R> {
toHaveTag(tagName: string): CustomMatcherResult;
}
}
}
// Add our own functions
expect.extend({
toHaveTag(received: any, tagName: string): jest.CustomMatcherResult {
let pass: boolean = false;
const message = () => (pass ? "" : `post does not have the '${tagName}' tag.`);
if (received instanceof ScrapedPost) {
pass = received.tags.find((x) => x.name == tagName) != undefined;
}
return { pass, message };
},
});

1818
yarn.lock
View File

File diff suppressed because it is too large Load Diff