Add resolution scraping feature to various sites

pull/16/head
neobooru 2022-01-30 19:11:04 +01:00
parent cbeea1c665
commit 5bb418dae6
6 changed files with 54 additions and 2 deletions

View File

@ -0,0 +1,15 @@
import { scrapeUrl } from "../test/engine";
test("get furaffinity.net post 23281942", async () => {
const pageUrl = "https://www.furaffinity.net/view/23281942/";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("zac");
expect(res.posts[0]).toHaveTag("riven");
expect(res.posts[0]).toHaveTag("pool");
expect(res.posts[0]).toHaveResolution([1280, 1279]);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe(
"https://d.furaffinity.net/art/nielsdejong/1492781239/1492781239.nielsdejong_pool_party_fun_with_riven_and_zac_by_polkinart.jpg"
);
});

View File

@ -1,7 +1,7 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine"; import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
import { guessContentType } from "../Utility"; import { guessContentType, parseResolutionString } from "../Utility";
export default class e621 implements ScrapeEngine { export default class FurAffinity implements ScrapeEngine {
name = "furaffinity"; name = "furaffinity";
canImport(url: Location): boolean { canImport(url: Location): boolean {
@ -22,6 +22,14 @@ export default class e621 implements ScrapeEngine {
// Set content type // Set content type
post.contentType = guessContentType(post.contentUrl); post.contentType = guessContentType(post.contentUrl);
// Set resolution
const infoTexts = Array.from(document.querySelector("section.info.text")?.children ?? []);
const sizeDiv = infoTexts.find((x) => x.querySelector("strong")?.innerText == "Size");
if (sizeDiv) {
const resolution = parseResolutionString(sizeDiv.querySelector("span")?.innerText);
post.resolution = resolution;
}
// Set rating // Set rating
const ratingEls = Array.from(document.querySelectorAll("div.rating > span.rating-box")).map( const ratingEls = Array.from(document.querySelectorAll("div.rating > span.rating-box")).map(
(x) => x as HTMLSpanElement (x) => x as HTMLSpanElement

View File

@ -7,6 +7,7 @@ test("get inkbunny.net post 2430530", async () => {
expect(res.posts[0]).toHaveTag("comic"); expect(res.posts[0]).toHaveTag("comic");
expect(res.posts[0]).toHaveTag("flower"); expect(res.posts[0]).toHaveTag("flower");
expect(res.posts[0]).toHaveTag("medusa"); expect(res.posts[0]).toHaveTag("medusa");
expect(res.posts[0]).toHaveResolution([1062, 1521]);
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe("https://nl.ib.metapix.net/files/full/3562/3562592_vavacung_medusa.png"); expect(res.posts[0].contentUrl).toBe("https://nl.ib.metapix.net/files/full/3562/3562592_vavacung_medusa.png");
}); });

View File

@ -33,6 +33,13 @@ export default class Inkbunny implements ScrapeEngine {
return result; return result;
} }
// Set post resolution
const width = parseInt((<HTMLInputElement>document.getElementById("submission-origwidth"))?.value);
const height = parseInt((<HTMLInputElement>document.getElementById("submission-origheight"))?.value);
if (width != NaN && height != NaN) {
post.resolution = [width, height];
}
// Set rating // Set rating
const ratingExp = new RegExp("Rating: (.*)"); const ratingExp = new RegExp("Rating: (.*)");
const spanEls = Array.from(document.querySelectorAll("div > span")); const spanEls = Array.from(document.querySelectorAll("div > span"));

14
src/engines/e621.test.ts Normal file
View File

@ -0,0 +1,14 @@
import { scrapeUrl } from "../test/engine";
test("get e621.net post 380315", async () => {
const pageUrl = "https://e621.net/posts/380315?q=league_of_legends+rating%3Asafe+order%3Ascore";
const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("inkinesss", "artist");
expect(res.posts[0]).toHaveTag("nasus_(lol)", "character");
expect(res.posts[0]).toHaveTag("ambiguous_gender");
expect(res.posts[0]).toHaveTag("low_res", "meta");
expect(res.posts[0]).toHaveResolution([500, 500]);
expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].contentUrl).toBe("https://static1.e621.net/data/92/e7/92e71bd73ab219d80516c4d9adad3db5.jpg");
});

View File

@ -37,6 +37,13 @@ export default class e621 implements ScrapeEngine {
break; break;
} }
// Set resolution
const width = parseInt((<HTMLSpanElement>document.querySelector("span[itemprop='width']"))?.innerText);
const height = parseInt((<HTMLSpanElement>document.querySelector("span[itemprop='height']"))?.innerText);
if (width != NaN && height != NaN) {
post.resolution = [width, height];
}
// Set tags // Set tags
const tagEls = Array.from(document.querySelectorAll("#tag-list > ul > li")).map((x) => x as HTMLLIElement); const tagEls = Array.from(document.querySelectorAll("#tag-list > ul > li")).map((x) => x as HTMLLIElement);