Nes
/
neo-scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix + bump version

pull/20/head
neobooru 2025-04-12 01:03:38 +02:00
parent 4692fe757d
commit 51be6dfdcd
3 changed files with 87 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "neo-scraper", "name": "neo-scraper",
"version": "0.10.0", "version": "0.10.1",
"description": "Scrape a booru-like post object from the DOM.", "description": "Scrape a booru-like post object from the DOM.",
"type": "module", "type": "module",
"main": "dist/index.js", "main": "dist/index.js",

4
src/engines/FurAffinity.ts
View File

@ -47,12 +47,12 @@ export default class FurAffinity extends ScrapeEngineBase {
} }
// Set tags // Set tags
post.tags = Array.from(document.querySelectorAll("section.tags-row > span.tags")) post.tags = Array.from(document.querySelectorAll("section.tags-row > span.tags a"))
.map((x) => (x as HTMLSpanElement).innerText) .map((x) => (x as HTMLSpanElement).innerText)
.map((x) => new ScrapedTag(x)); .map((x) => new ScrapedTag(x));
// Set user who submitted the art as the artist // Set user who submitted the art as the artist
const submissionUser = document.querySelector<HTMLElement>(".submission-id-sub-container > a > strong")?.innerText; const submissionUser = document.querySelector<HTMLElement>(".submission-id-sub-container a")?.innerText;
if (submissionUser) { if (submissionUser) {
post.tags.push(new ScrapedTag(submissionUser, "artist")); post.tags.push(new ScrapedTag(submissionUser, "artist"));
} }

168
src/engines/Gelbooru.test.ts
View File

@ -1,84 +1,84 @@
import { scrapeUrl } from "../test/engine.js"; import { scrapeUrl } from "../test/engine.js";
test("get safebooru.org post 2871532", async () => { test("get safebooru.org post 2871532", async () => {
const pageUrl = "https://safebooru.org/index.php?page=post&s=view&id=2871532"; const pageUrl = "https://safebooru.org/index.php?page=post&s=view&id=2871532";
const res = await scrapeUrl(pageUrl); const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("omachi_(slabco)", "artist"); expect(res.posts[0]).toHaveTag("omachi_(slabco)", "artist");
expect(res.posts[0]).toHaveTag("flint_(girls_und_panzer)", "character"); expect(res.posts[0]).toHaveTag("flint_(girls_und_panzer)", "character");
expect(res.posts[0]).toHaveTag("bangs"); expect(res.posts[0]).toHaveTag("bangs");
expect(res.posts[0]).toHaveTag("shirt"); expect(res.posts[0]).toHaveTag("shirt");
expect(res.posts[0]).toHaveTag("skirt"); expect(res.posts[0]).toHaveTag("skirt");
expect(res.posts[0]).toHaveTag("t-shirt"); expect(res.posts[0]).toHaveTag("t-shirt");
expect(res.posts[0]).toHaveTag("long_hair"); expect(res.posts[0]).toHaveTag("long_hair");
expect(res.posts[0]).toHaveResolution([1142, 1600]); expect(res.posts[0]).toHaveResolution([1142, 1600]);
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("safe"); expect(res.posts[0].rating).toBe("safe");
expect(res.posts[0].contentUrl).toContain( expect(res.posts[0].contentUrl).toContain(
"https://safebooru.org//images/2757/5f19c7744db918c3d4c4c6035bc65dfa9e18866f.jpg" "https://safebooru.org//images/2757/5f19c7744db918c3d4c4c6035bc65dfa9e18866f.jpg"
); );
}); });
test("get gelbooru.com post 4925055", async () => { test("get gelbooru.com post 4925055", async () => {
const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=4925055&tags=kimetsu_no_yaiba"; const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=4925055&tags=kimetsu_no_yaiba";
const res = await scrapeUrl(pageUrl); const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("maosen", "artist"); expect(res.posts[0]).toHaveTag("maosen", "artist");
expect(res.posts[0]).toHaveTag("butterfly"); expect(res.posts[0]).toHaveTag("butterfly");
expect(res.posts[0]).toHaveResolution([720, 1103]); expect(res.posts[0]).toHaveResolution([720, 1103]);
expect(res.posts[0]).toHaveSource("https://www.pixiv.net/artworks/76853958"); expect(res.posts[0]).toHaveSource("https://www.pixiv.net/artworks/76853958");
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("sketchy"); expect(res.posts[0].rating).toBe("sketchy");
expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg"); expect(res.posts[0].contentUrl).toContain("gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
}); });
test("get gelbooru.com post 6225495 with notes", async () => { test("get gelbooru.com post 6225495 with notes", async () => {
const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=6225495"; const pageUrl = "https://gelbooru.com/index.php?page=post&s=view&id=6225495";
const res = await scrapeUrl(pageUrl); const res = await scrapeUrl(pageUrl);
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0]).toHaveTag("hair_up"); expect(res.posts[0]).toHaveTag("hair_up");
expect(res.posts[0]).toHaveTag("hair_up"); expect(res.posts[0]).toHaveTag("hair_up");
expect(res.posts[0]).toHaveTag("teeth"); expect(res.posts[0]).toHaveTag("teeth");
expect(res.posts[0]).toHaveTag("highres"); expect(res.posts[0]).toHaveTag("highres");
expect(res.posts[0]).toHaveTag("nintendo"); expect(res.posts[0]).toHaveTag("nintendo");
expect(res.posts[0]).toHaveTag("link"); expect(res.posts[0]).toHaveTag("link");
expect(res.posts[0]).toHaveTag("ttanuu."); expect(res.posts[0]).toHaveTag("ttanuu.");
expect(res.posts[0]).toHaveNote("I'm fine, more or less."); expect(res.posts[0]).toHaveNote("I'm fine, more or less.");
expect(res.posts[0]).toHaveNote("But no need to worry!"); expect(res.posts[0]).toHaveNote("But no need to worry!");
expect(res.posts[0]).toHaveNote("You're always the best just by being yourself-!!!"); expect(res.posts[0]).toHaveNote("You're always the best just by being yourself-!!!");
expect(res.posts[0]).toHaveResolution([1600, 2200]); expect(res.posts[0]).toHaveResolution([1600, 2200]);
expect(res.posts[0]).toHaveSource("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=88721816"); expect(res.posts[0]).toHaveSource("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=88721816");
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("sketchy"); expect(res.posts[0].rating).toBe("sketchy");
expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/af/1d/af1dd661fab6714dffc29cb38b6b2269.png"); expect(res.posts[0].contentUrl).toContain("gelbooru.com/images/af/1d/af1dd661fab6714dffc29cb38b6b2269.png");
}); });
test("get rule34.xxx post 2740994", async () => { test("get rule34.xxx post 2740994", async () => {
const pageUrl = "https://rule34.xxx/index.php?page=post&s=view&id=2740994"; const pageUrl = "https://rule34.xxx/index.php?page=post&s=view&id=2740994";
const res = await scrapeUrl(pageUrl); const res = await scrapeUrl(pageUrl);
expect(res.posts[0]).toHaveTag("fire_emblem", "copyright"); expect(res.posts[0]).toHaveTag("fire_emblem", "copyright");
expect(res.posts[0]).toHaveTag("nintendo", "copyright"); expect(res.posts[0]).toHaveTag("nintendo", "copyright");
expect(res.posts[0]).toHaveTag("lucina_(fire_emblem)", "character"); expect(res.posts[0]).toHaveTag("lucina_(fire_emblem)", "character");
expect(res.posts[0]).toHaveTag("blue_hair"); expect(res.posts[0]).toHaveTag("blue_hair");
expect(res.posts[0]).toHaveTag("solo"); expect(res.posts[0]).toHaveTag("solo");
expect(res.posts[0]).toHaveResolution([600, 800]); expect(res.posts[0]).toHaveResolution([600, 800]);
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("safe"); expect(res.posts[0].rating).toBe("safe");
expect(res.posts[0].contentUrl).toContain(".rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg"); expect(res.posts[0].contentUrl).toContain(".rule34.xxx//images/2472/35bf31c98f53a369c047baf9bae496fb.jpeg");
}); });
test("get xbooru.com post 834083", async () => { test("get xbooru.com post 834083", async () => {
// Don't worry this isn't a cursed image. // Don't worry this isn't a cursed image.
const pageUrl = "https://xbooru.com/index.php?page=post&s=view&id=834083"; const pageUrl = "https://xbooru.com/index.php?page=post&s=view&id=834083";
const res = await scrapeUrl(pageUrl); const res = await scrapeUrl(pageUrl);
expect(res.posts[0]).toHaveTag("friendship_is_magic", "copyright"); expect(res.posts[0]).toHaveTag("friendship_is_magic", "copyright");
expect(res.posts[0]).toHaveTag("nurse_redheart", "character"); expect(res.posts[0]).toHaveTag("nurse_redheart", "character");
expect(res.posts[0]).toHaveTag("nurse"); expect(res.posts[0]).toHaveTag("nurse");
expect(res.posts[0]).toHaveTag("standing"); expect(res.posts[0]).toHaveTag("standing");
expect(res.posts[0]).toHaveResolution([1024, 1878]); expect(res.posts[0]).toHaveResolution([1024, 1878]);
expect(res.posts.length).toBe(1); expect(res.posts.length).toBe(1);
expect(res.posts[0].pageUrl).toBe(pageUrl); expect(res.posts[0].pageUrl).toBe(pageUrl);
expect(res.posts[0].rating).toBe("safe"); expect(res.posts[0].rating).toBe("safe");
expect(res.posts[0].contentUrl).toContain("https://img.xbooru.com//images/584/040dc1625c094a7d5d7b0918aa0d8324.png"); expect(res.posts[0].contentUrl).toContain("https://img.xbooru.com//images/584/040dc1625c094a7d5d7b0918aa0d8324.png");
}); });