Nes
/
neo-scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

AnimePictures parse resolution + more tests

pull/16/head
neobooru 2021-11-23 13:44:42 +01:00
parent c847990b92
commit 905c36b2ff
10 changed files with 101 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/Utility.ts
View File

@ -179,3 +179,18 @@ export function createNotesFromMoebooruBoxes(document: Document, boxSize: [numbe
}
return notes;
}
export function parseResolutionString(str: string): [number, number] | undefined {
// Example string: 1600x2200
const res = str
.split("x") // Split on 'x' character
.map((v, _) => parseInt(v)) // Parse ints
.filter(Number); // Filter NaN. Also removes the number 0, but that's fine because the resolution can't be 0.
if (res.length == 2) {
// Kinda hacky, but this is currently the correct way to convert an array to a tuple.
return [res[0], res[1]];
} else {
console.log(`[parseResolutionString] Couldn't parse resolution from '${str}'.`);
}
}

8
src/engines/AnimePictures.test.ts
View File

@ -4,6 +4,14 @@ test("get anime-pictures.net post 729833", async () => {
const pageUrl = "https://anime-pictures.net/pictures/view_post/729833?lang=en";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("arknights", "copyright");
expect(res.posts[0]).toHaveTag("ch'en_(arknights)", "character");
expect(res.posts[0]).toHaveTag("ttk_(kirinottk)", "artist");
expect(res.posts[0]).toHaveTag("single");
expect(res.posts[0]).toHaveTag("holding");
expect(res.posts[0]).toHaveTag("gloves");
expect(res.posts[0]).toHaveTag("jacket");
expect(res.posts[0]).toHaveResolution([4000, 6865]);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://anime-pictures.net/pictures/download_image/729833-4000x6865-arknights-ch%26%2339%3Ben+%28arknights%29-ttk+%28kirinottk%29-single-long+hair-tall+image.jpeg"

16
src/engines/AnimePictures.ts
View File

@ -1,6 +1,6 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes";
import { guessContentType } from "../Utility";
import { guessContentType, parseResolutionString } from "../Utility";
export default class AnimePictures implements ScrapeEngine {
name = "animepictures";
@ -20,9 +20,6 @@ export default class AnimePictures implements ScrapeEngine {
if (originalImageElements.length > 0) {
post.contentUrl = (originalImageElements[0] as HTMLAnchorElement).href;
} else {
// No point in continuing when we don't have an image.
return result;
}
// Set content type
@ -31,8 +28,17 @@ export default class AnimePictures implements ScrapeEngine {
// Set rating
post.rating = "safe"; // Probably
// Set resolution
const postContentLinkEls = Array.from(
document.querySelectorAll<HTMLAnchorElement>("#content > div > .post_content > a")
);
const resEl = postContentLinkEls.find((x) => x.href.indexOf("res_x") != -1);
if (resEl) {
post.resolution = parseResolutionString(resEl.innerText);
}
// Set tags
const tagElements = Array.from(document.querySelectorAll("ul.tags > li > a")).map((x) => x as HTMLAnchorElement);
const tagElements = Array.from(document.querySelectorAll<HTMLAnchorElement>("ul.tags > li > a"));
for (const tagElement of tagElements) {
let tagName = tagElement.innerText;

7
src/engines/Danbooru.test.ts
View File

@ -4,11 +4,15 @@ test("get danbooru.donmai.us post 3555872", async () => {
const pageUrl = "https://danbooru.donmai.us/posts/3555872";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("omachi_(slabco)", "artist");
expect(res.posts[0]).toHaveTag("dr_pepper", "copyright");
expect(res.posts[0]).toHaveTag("flint_(girls_und_panzer)", "character");
expect(res.posts[0]).toHaveTag("bangs");
expect(res.posts[0]).toHaveTag("shirt");
expect(res.posts[0]).toHaveTag("skirt");
expect(res.posts[0]).toHaveTag("t-shirt");
expect(res.posts[0]).toHaveTag("long_hair");
expect(res.posts[0]).toHaveResolution([1142, 1600]);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://cdn.donmai.us/original/90/33/__flint_girls_und_panzer_and_1_more_drawn_by_omachi_slabco__90332299216db81b8793084e4cb7c15d.jpg"
@ -19,6 +23,9 @@ test("get danbooru.donmai.us post 4924490", async () => {
const pageUrl = "https://danbooru.donmai.us/posts/4924490";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("yuureidoushi_(yuurei6214)", "artist");
expect(res.posts[0]).toHaveTag("comic");
expect(res.posts[0]).toHaveResolution([1000, 2062]);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://cdn.donmai.us/original/c5/fb/__yuuki_and_tamaki_princess_connect_drawn_by_yuureidoushi_yuurei6214__c5fbc62758d0f1d8c07aac524282434e.png"

18
src/engines/Gelbooru.test.ts
View File

@ -4,6 +4,14 @@ test("get safebooru.org post 2871532", async () => {
const pageUrl = "https://safebooru.org/index.php?page=post&s=view&id=2871532";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("omachi_(slabco)", "artist");
expect(res.posts[0]).toHaveTag("flint_(girls_und_panzer)", "character");
expect(res.posts[0]).toHaveTag("bangs");
expect(res.posts[0]).toHaveTag("shirt");
expect(res.posts[0]).toHaveTag("skirt");
expect(res.posts[0]).toHaveTag("t-shirt");
expect(res.posts[0]).toHaveTag("long_hair");
expect(res.posts[0]).toHaveResolution([1142, 1600]);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://safebooru.org//images/2757/5f19c7744db918c3d4c4c6035bc65dfa9e18866f.jpg"
@ -14,6 +22,9 @@ test("get gelbooru.com post 4925055", async () => {
const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=4925055&tags=kimetsu_no_yaiba";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("maosen", "artist");
expect(res.posts[0]).toHaveTag("butterfly");
expect(res.posts[0]).toHaveResolution([720, 1103]);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
});
@ -33,6 +44,7 @@ test("get gelbooru.com post 6225495 with notes", async () => {
expect(res.posts[0]).toHaveNote("I'm fine, more or less.");
expect(res.posts[0]).toHaveNote("But no need to worry!");
expect(res.posts[0]).toHaveNote("You're always the best just by being yourself-!!!");
expect(res.posts[0]).toHaveResolution([1600, 2200]);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/af/1d/af1dd661fab6714dffc29cb38b6b2269.png");
});
@ -40,6 +52,12 @@ test("get gelbooru.com post 6225495 with notes", async () => {
test("get rule34.xxx post 2740994", async () => {
const pageUrl = "https://rule34.xxx/index.php?page=post&s=view&id=2740994";
const res = await scrapeUrl(pageUrl);
expect(res.posts[0]).toHaveTag("fire_emblem", "copyright");
expect(res.posts[0]).toHaveTag("nintendo", "copyright");
expect(res.posts[0]).toHaveTag("lucina", "character");
expect(res.posts[0]).toHaveTag("blue_hair");
expect(res.posts[0]).toHaveTag("solo");
expect(res.posts[0]).toHaveResolution([600, 800]);
expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toContain(".rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg");

15
src/engines/Gelbooru.ts
View File

@ -1,7 +1,7 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes";
import { CategoryMap } from "./Common";
import { createNoteFromDanbooruArticle, guessContentType } from "../Utility";
import { createNoteFromDanbooruArticle, guessContentType, parseResolutionString } from "../Utility";
// Small hack to avoid screwing with semver
enum Version {
@ -103,18 +103,7 @@ export default class Gelbooru extends ScrapeEngineBase {
}
// If size element
else if (matches[1] == "Size") {
// Example string: 1600x2200
const res = matches[2]
.split("x") // Split on 'x' character
.map((v, _) => parseInt(v)) // Parse ints
.filter(Number); // Filter NaN. Also removes the number 0, but that's fine because the resolution can't be 0.
if (res.length == 2) {
// Kinda hacky, but this is currently the correct way to convert an array to a tuple.
post.resolution = [res[0], res[1]];
} else {
this.log("Couldn't parse post resolution.");
}
post.resolution = parseResolutionString(matches[2]);
}
}
}

9
src/engines/Moebooru.test.ts
View File

@ -4,6 +4,10 @@ test("get yande.re post 438510", async () => {
const pageUrl = "https://yande.re/post/show/438510";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("the_legend_of_zelda", "copyright");
expect(res.posts[0]).toHaveTag("link", "character");
expect(res.posts[0]).toHaveTag("landscape");
expect(res.posts[0]).toHaveResolution([3000, 1688]);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://files.yande.re/image/49b79bc372716a579d8525dcaccb56e2/yande.re%20438510%20landscape%20link%20nintendo%20princess_zelda%20sword%20the_legend_of_zelda%20the_legend_of_zelda%3A_breath_of_the_wild.jpg"
@ -16,6 +20,7 @@ test("get yande.re post 684478 with notes", async () => {
expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("gym_uniform");
expect(res.posts[0]).toHaveNote("post #671879");
expect(res.posts[0]).toHaveResolution([2480, 3508]);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://files.yande.re/image/8bdf7c54b0a459acee080f0aeb2a259c/yande.re%20684478%20gym_uniform%20pal_lett.jpg"
@ -26,6 +31,8 @@ test("get konachan.com post 289968", async () => {
const pageUrl = "https://konachan.com/post/show/289968/ahri_-league_of_legends-akali-animal_ears-blonde_h";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("league_of_legends", "copyright");
expect(res.posts[0]).toHaveResolution([1200, 783]);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://konachan.com/image/e40b38fb704434e4da3eae41e26bca39/Konachan.com%20-%20289968%20akali%20blue_eyes%20breasts%20choker%20cleavage%20evelynn%20foxgirl%20gloves%20group%20hat%20kai%27sa%20long_hair%20microphone%20ponytail%20saphirya%20short_hair%20signed%20watermark.jpg"
@ -41,6 +48,7 @@ test("get konachan.com post 327454 with notes", async () => {
expect(res.posts[0]).toHaveTag("kexue");
expect(res.posts[0]).toHaveTag("water");
expect(res.posts[0]).toHaveNote("Under construction. \nBe careful of your safety");
expect(res.posts[0]).toHaveResolution([5629, 2505]);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://konachan.com/image/94a13c8a1fcb9c0f6c9a6c6c5c316259/Konachan.com%20-%20327454%20apron%20brown_hair%20building%20car%20gloves%20kexue%20long_hair%20original%20rain%20see_through%20short_hair%20twintails%20uniform%20water.jpg"
@ -58,6 +66,7 @@ test("get konachan.com post 285782 with notes", async () => {
expect(res.posts[0]).toHaveNote("I'M AWAKE!!");
expect(res.posts[0]).toHaveNote("Poi!");
expect(res.posts[0]).toHaveNote("Haaah!");
expect(res.posts[0]).toHaveResolution([1000, 830]);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://konachan.com/image/24f5974ad8ed725292ccb3f682181bc1/Konachan.com%20-%20285782%20anthropomorphism%20bed%20black_eyes%20blonde_hair%20cropped%20fang%20kantai_collection%20long_hair%20momoniku%20pajamas%20white%20yuudachi_%28kancolle%29.jpg"

15
src/engines/Moebooru.ts
View File

@ -1,6 +1,6 @@
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapedNote } from "../ScrapeEngine";
import { TagCategory } from "../BooruTypes";
import { createNotesFromMoebooruBoxes, guessContentType } from "../Utility";
import { createNotesFromMoebooruBoxes, guessContentType, parseResolutionString } from "../Utility";
export default class Moebooru extends ScrapeEngineBase {
name = "moebooru";
@ -54,18 +54,7 @@ export default class Moebooru extends ScrapeEngineBase {
}
// If size element
else if (matches[1] == "Size") {
// Example string: 1600x2200
const res = matches[2]
.split("x") // Split on 'x' character
.map((v, _) => parseInt(v)) // Parse ints
.filter(Number); // Filter NaN. Also removes the number 0, but that's fine because the resolution can't be 0.
if (res.length == 2) {
// Kinda hacky, but this is currently the correct way to convert an array to a tuple.
post.resolution = [res[0], res[1]];
} else {
this.log("Couldn't parse post resolution.");
}
post.resolution = parseResolutionString(matches[2]);
}
}
}

5
src/test/jest.d.ts vendored
View File

@ -1,10 +1,13 @@
import "jest-extended";
import { TagCategory } from "../BooruTypes";
declare global {
namespace jest {
interface Matchers<R> {
toHaveTag(tagName: string): CustomMatcherResult;
// The `= undefined` part is incorrect according to VSCode, but it is actually required.
toHaveTag(tagName: string, tagCategory: TagCategory | undefined = undefined): CustomMatcherResult;
toHaveNote(noteText: string): CustomMatcherResult;
toHaveResolution(resolution: [number, number]): CustomMatcherResult;
}
}
}

29
src/test/jest.setup.ts
View File

@ -1,4 +1,5 @@
import { JSDOM } from "jsdom";
import { TagCategory } from "../BooruTypes";
import { ScrapedPost } from "../ScrapeEngine";
// Expose JSDOM Element constructor
@ -14,22 +15,42 @@ Object.defineProperty(global.Element.prototype, "innerText", {
// Add our own functions
expect.extend({
toHaveTag(received: any, tagName: string): jest.CustomMatcherResult {
toHaveTag(
received: any,
tagName: string,
tagCategory: TagCategory | undefined = undefined
): jest.CustomMatcherResult {
let pass: boolean = false;
const message = () => (pass ? "" : `post does not have the '${tagName}' tag.`);
const message = () => (pass ? "" : `Post does not have the '${tagName}' tag, or the category is incorrect.`);
if (received instanceof ScrapedPost) {
pass = received.tags.find((x) => x.name == tagName) != undefined;
const found = received.tags.find((x) => x.name == tagName);
if (found) {
if (tagCategory) {
pass = found.category == tagCategory;
} else {
pass = true;
}
}
}
return { pass, message };
},
toHaveNote(received: any, noteText: string): jest.CustomMatcherResult {
let pass: boolean = false;
const message = () => (pass ? "" : `post does not have a note with '${noteText}'.`);
const message = () => (pass ? "" : `Post does not have a note with '${noteText}'.`);
if (received instanceof ScrapedPost) {
pass = received.notes.find((x) => x.text == noteText) != undefined;
}
return { pass, message };
},
toHaveResolution(received: any, resolution: [number, number]): jest.CustomMatcherResult {
let pass: boolean = false;
const message = () => (pass ? "" : `Expected: ${resolution}\nReceived: ${received?.resolution}`);
if (received instanceof ScrapedPost && received.resolution?.length == 2) {
pass = received.resolution[0] == resolution[0] && received.resolution[1] == resolution[1];
}
return { pass, message };
},
});