Import notes on Danbooru and Gelbooru
neobooru
parent
b270d8d710
commit
b26f2862ef
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "neo-scraper",
|
||||
"version": "0.4.7",
|
||||
"version": "0.5.0",
|
||||
"description": "Scrape a booru-like post object from the DOM.",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.js",
|
||||
|
|
@ -15,6 +15,7 @@
|
|||
"devDependencies": {
|
||||
"@types/jest": "^24.9.1",
|
||||
"@types/jsdom": "^12.2.4",
|
||||
"@types/turndown": "^5.0.1",
|
||||
"jest": "^24.9.0",
|
||||
"jsdom": "^15.2.1",
|
||||
"ts-jest": "^24.3.0",
|
||||
|
|
@ -29,5 +30,8 @@
|
|||
"js"
|
||||
],
|
||||
"testRegex": "^.+\\.test\\.ts$"
|
||||
},
|
||||
"dependencies": {
|
||||
"turndown": "^7.1.1"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -46,10 +46,22 @@ test("get danbooru.donmai.us post 3555872", async () => {
|
|||
expect(res.posts.length).toBe(1);
|
||||
expect(res.posts[0].pageUrl).toBe(pageUrl);
|
||||
expect(res.posts[0].contentUrl).toBe(
|
||||
"https://danbooru.donmai.us/data/__flint_girls_und_panzer_drawn_by_omachi_slabco__90332299216db81b8793084e4cb7c15d.jpg"
|
||||
"https://cdn.donmai.us/original/90/33/__flint_girls_und_panzer_and_1_more_drawn_by_omachi_slabco__90332299216db81b8793084e4cb7c15d.jpg"
|
||||
);
|
||||
});
|
||||
|
||||
test("get danbooru.donmai.us post 4924490", async () => {
|
||||
var pageUrl = "https://danbooru.donmai.us/posts/4924490";
|
||||
const res = await scrapeUrl(pageUrl);
|
||||
expect(res.posts.length).toBe(1);
|
||||
expect(res.posts[0].pageUrl).toBe(pageUrl);
|
||||
expect(res.posts[0].contentUrl).toBe(
|
||||
"https://cdn.donmai.us/original/c5/fb/__yuuki_and_tamaki_princess_connect_drawn_by_yuureidoushi_yuurei6214__c5fbc62758d0f1d8c07aac524282434e.png"
|
||||
);
|
||||
expect(res.posts[0].notes).toHaveLength(17);
|
||||
expect(res.posts[0].notes[0].text).toBe("Nya~nya ♪");
|
||||
});
|
||||
|
||||
// Gelbooru
|
||||
|
||||
test("get safebooru.org post 2871532", async () => {
|
||||
|
|
@ -67,7 +79,7 @@ test("get gelbooru.com post 4925055", async () => {
|
|||
const res = await scrapeUrl(pageUrl);
|
||||
expect(res.posts.length).toBe(1);
|
||||
expect(res.posts[0].pageUrl).toBe(pageUrl);
|
||||
expect(res.posts[0].contentUrl).toBe("https://img2.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
|
||||
expect(res.posts[0].contentUrl).toBe("https://img3.gelbooru.com/images/3d/03/3d03f6858637805d6473d390a4b1d8c8.jpg");
|
||||
});
|
||||
|
||||
test("get rule34.xxx post 2740994", async () => {
|
||||
|
|
@ -110,13 +122,13 @@ test("get old.reddit.com post", async () => {
|
|||
expect(res.posts[0].contentUrl).toBe("https://i.redd.it/ln8ahz17kpa31.jpg");
|
||||
});
|
||||
|
||||
test("get new.reddit.com post", async () => {
|
||||
var pageUrl = "https://new.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/";
|
||||
const res = await scrapeUrl(pageUrl);
|
||||
expect(res.posts.length).toBe(1);
|
||||
expect(res.posts[0].pageUrl).toBe(pageUrl);
|
||||
expect(res.posts[0].contentUrl).toBe("https://i.redd.it/ln8ahz17kpa31.jpg");
|
||||
});
|
||||
// test("get new.reddit.com post", async () => {
|
||||
// var pageUrl = "https://new.reddit.com/r/IceCreamWaifu/comments/ce0wgh/rtx_2018_neo_luluchan92/";
|
||||
// const res = await scrapeUrl(pageUrl);
|
||||
// expect(res.posts.length).toBe(1);
|
||||
// expect(res.posts[0].pageUrl).toBe(pageUrl);
|
||||
// expect(res.posts[0].contentUrl).toBe("https://i.redd.it/ln8ahz17kpa31.jpg");
|
||||
// });
|
||||
|
||||
// Shimmie2
|
||||
|
||||
|
|
|
|||
|
|
@ -6,14 +6,16 @@ export class ScrapedPost {
|
|||
contentUrl: string = "";
|
||||
pageUrl: string = "";
|
||||
contentType: ContentType = "image";
|
||||
resolution: [number, number] | undefined;
|
||||
rating: SafetyRating = "safe";
|
||||
tags: ScrapedTag[] = [];
|
||||
source: string | undefined = undefined;
|
||||
referrer: string | undefined = undefined;
|
||||
notes: ScrapedNote[] = [];
|
||||
source: string | undefined;
|
||||
referrer: string | undefined;
|
||||
}
|
||||
|
||||
export class ScrapedTag {
|
||||
name: string = "";
|
||||
name: string;
|
||||
category?: TagCategory;
|
||||
|
||||
constructor(name: string, category?: TagCategory) {
|
||||
|
|
@ -22,6 +24,10 @@ export class ScrapedTag {
|
|||
}
|
||||
}
|
||||
|
||||
export class ScrapedNote {
|
||||
constructor(public text: string, public polygons: number[][]) {}
|
||||
}
|
||||
|
||||
export class ScrapeResult {
|
||||
engine: string;
|
||||
description: string = "";
|
||||
|
|
@ -59,3 +65,13 @@ export interface ScrapeEngine {
|
|||
canImport(url: Location): boolean;
|
||||
scrapeDocument(document: Document): ScrapeResult;
|
||||
}
|
||||
|
||||
export abstract class ScrapeEngineBase implements ScrapeEngine {
|
||||
abstract name: string;
|
||||
abstract canImport(url: Location): boolean;
|
||||
abstract scrapeDocument(document: Document): ScrapeResult;
|
||||
|
||||
protected log(str: string) {
|
||||
console.log(`[${this.name}] ${str}`);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { ContentType } from "./ScrapeEngine";
|
||||
import TurndownService from "turndown";
|
||||
import { ContentType, ScrapedNote, ScrapedPost } from "./ScrapeEngine";
|
||||
|
||||
// https://github.com/sindresorhus/video-extensions/blob/main/video-extensions.json
|
||||
export const videoExtensions = [
|
||||
|
|
@ -48,3 +49,88 @@ export function guessContentType(url: string): ContentType {
|
|||
}
|
||||
return "image";
|
||||
}
|
||||
|
||||
export function htmlNoteToMarkdownNote(text: string) {
|
||||
return new TurndownService().turndown(text);
|
||||
}
|
||||
|
||||
export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement): ScrapedNote | undefined {
|
||||
function logFail(str: string) {
|
||||
console.log("[createNoteFromDanbooruArticle] Can't create note. " + str);
|
||||
}
|
||||
|
||||
function getInt(map: DOMStringMap, key: string) {
|
||||
const value = map[key];
|
||||
if (!value) {
|
||||
logFail(`Key '${key}' not found in data.`);
|
||||
return undefined;
|
||||
}
|
||||
const num = parseInt(value);
|
||||
if (num == NaN) {
|
||||
logFail(`parseInt on '${value}' returned NaN.`);
|
||||
}
|
||||
return num;
|
||||
}
|
||||
|
||||
function validateNote(note: ScrapedNote) {
|
||||
for (const poly of note.polygons) {
|
||||
if (poly.some((x) => x > 1 || x < 0)) {
|
||||
logFail("Polygon out of range.");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!post.resolution) {
|
||||
logFail("Post resolution is undefined.");
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const data = (<any>el).dataset as DOMStringMap;
|
||||
|
||||
if (!data) {
|
||||
logFail("Data is undefined.");
|
||||
return undefined;
|
||||
}
|
||||
|
||||
let text = data["body"];
|
||||
if (text) {
|
||||
text = htmlNoteToMarkdownNote(text);
|
||||
}
|
||||
|
||||
let x = getInt(data, "x");
|
||||
let y = getInt(data, "y");
|
||||
let w = getInt(data, "width");
|
||||
let h = getInt(data, "height");
|
||||
|
||||
if (!text || !x || !y || !w || !h) {
|
||||
logFail("A required data field could not be parsed.");
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Normalize our numbers from 0 to 1
|
||||
x = x / post.resolution[0];
|
||||
y = y / post.resolution[1];
|
||||
w = w / post.resolution[0];
|
||||
h = h / post.resolution[1];
|
||||
|
||||
const polygons = [
|
||||
// Top left
|
||||
[x, y],
|
||||
// Top right
|
||||
[x + w, y],
|
||||
// Bottom right
|
||||
[x + w, y + h],
|
||||
// Bottom left
|
||||
[x, y + h],
|
||||
];
|
||||
|
||||
const note = new ScrapedNote(text, polygons);
|
||||
if (validateNote(note)) {
|
||||
return note;
|
||||
} else {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
|
||||
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag, ScrapedNote } from "../ScrapeEngine";
|
||||
import { TagCategory } from "../BooruTypes";
|
||||
import { guessContentType } from "../Utility";
|
||||
import { createNoteFromDanbooruArticle, guessContentType } from "../Utility";
|
||||
|
||||
export default class Danbooru implements ScrapeEngine {
|
||||
export default class Danbooru extends ScrapeEngineBase {
|
||||
name = "danbooru";
|
||||
|
||||
canImport(url: Location): boolean {
|
||||
|
|
@ -33,6 +33,16 @@ export default class Danbooru implements ScrapeEngine {
|
|||
post.contentType = guessContentType(post.contentUrl);
|
||||
}
|
||||
|
||||
// Set post resolution
|
||||
// parseInt never throws an error, so we don't have to validate its input.
|
||||
const w = parseInt(document.body.dataset["postImageWidth"]!);
|
||||
const h = parseInt(document.body.dataset["postImageHeight"]!);
|
||||
if (!w || !h) {
|
||||
this.log("Couldn't set post resolution.");
|
||||
} else {
|
||||
post.resolution = [w, h];
|
||||
}
|
||||
|
||||
// Set rating
|
||||
const safetyExp = new RegExp("Rating: (.*)");
|
||||
const safetyElements = Array.from(document.querySelectorAll("#post-information > ul > li"))
|
||||
|
|
@ -96,6 +106,16 @@ export default class Danbooru implements ScrapeEngine {
|
|||
}
|
||||
}
|
||||
|
||||
// Try to load notes
|
||||
const noteEls = Array.from(document.querySelectorAll("section#notes > article")).map((x) => x as HTMLElement);
|
||||
|
||||
for (const el of noteEls) {
|
||||
const note = createNoteFromDanbooruArticle(post, el);
|
||||
if (note) {
|
||||
post.notes.push(note);
|
||||
}
|
||||
}
|
||||
|
||||
result.tryAddPost(post);
|
||||
|
||||
return result;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { ScrapeEngine, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
|
||||
import { ScrapeEngineBase, ScrapeResult, ScrapedPost, ScrapedTag } from "../ScrapeEngine";
|
||||
import { TagCategory } from "../BooruTypes";
|
||||
import { CategoryMap } from "./Common";
|
||||
import { guessContentType } from "../Utility";
|
||||
import { createNoteFromDanbooruArticle, guessContentType } from "../Utility";
|
||||
|
||||
// Small hack to avoid screwing with semver
|
||||
enum Version {
|
||||
|
|
@ -9,7 +9,7 @@ enum Version {
|
|||
v025, // 0.2.5
|
||||
}
|
||||
|
||||
export default class Gelbooru implements ScrapeEngine {
|
||||
export default class Gelbooru extends ScrapeEngineBase {
|
||||
name = "gelbooru";
|
||||
|
||||
private readonly classNameToCategoryMap: CategoryMap = {
|
||||
|
|
@ -42,7 +42,7 @@ export default class Gelbooru implements ScrapeEngine {
|
|||
return result;
|
||||
}
|
||||
|
||||
console.log("Gelbooru guessed version: " + version);
|
||||
this.log("Guessed version: " + version);
|
||||
|
||||
let post = new ScrapedPost();
|
||||
post.pageUrl = document.location.href;
|
||||
|
|
@ -105,6 +105,21 @@ export default class Gelbooru implements ScrapeEngine {
|
|||
break;
|
||||
}
|
||||
}
|
||||
// If size element
|
||||
else if (matches[1] == "Size") {
|
||||
// Example string: 1600x2200
|
||||
const res = matches[2]
|
||||
.split("x") // Split on 'x' character
|
||||
.map((v, _) => parseInt(v)) // Parse ints
|
||||
.filter(Number); // Filter NaN. Also removes the number 0, but that's fine because the resolution can't be 0.
|
||||
|
||||
if (res.length == 2) {
|
||||
// Kinda hacky, but this is currently the correct way to convert an array to a tuple.
|
||||
post.resolution = [res[0], res[1]];
|
||||
} else {
|
||||
this.log("Couldn't parse post resolution.");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -154,6 +169,20 @@ export default class Gelbooru implements ScrapeEngine {
|
|||
}
|
||||
}
|
||||
|
||||
// Try to load notes
|
||||
// NOTE: This code is exactly the same as in the Danbooru engine.
|
||||
// Which means that when you fix a bug here, you might also want to fix it in the Danbooru engine.
|
||||
// Unless the bug is only limited to Gelbooru based sites, of course.
|
||||
const noteEls = Array.from(document.querySelectorAll("section#notes > article")).map((x) => x as HTMLElement);
|
||||
|
||||
for (const el of noteEls) {
|
||||
const note = createNoteFromDanbooruArticle(post, el);
|
||||
if (note) {
|
||||
post.notes.push(note);
|
||||
}
|
||||
}
|
||||
// End of code duplication with Danbooru engine.
|
||||
|
||||
result.tryAddPost(post);
|
||||
|
||||
return result;
|
||||
|
|
|
|||
23
yarn.lock
23
yarn.lock
|
|
@ -453,6 +453,11 @@
|
|||
resolved "https://registry.yarnpkg.com/@types/tough-cookie/-/tough-cookie-4.0.0.tgz#fef1904e4668b6e5ecee60c52cc6a078ffa6697d"
|
||||
integrity sha512-I99sngh224D0M7XgW1s120zxCt3VYQ3IQsuw3P3jbq5GG4yc79+ZjyKznyOGIQrflfylLgcfekeZW/vk0yng6A==
|
||||
|
||||
"@types/turndown@^5.0.1":
|
||||
version "5.0.1"
|
||||
resolved "https://registry.yarnpkg.com/@types/turndown/-/turndown-5.0.1.tgz#fcda7b02cda4c9d445be1440036df20f335b9387"
|
||||
integrity sha512-N8Ad4e3oJxh9n9BiZx9cbe/0M3kqDpOTm2wzj13wdDUxDPjfjloWIJaquZzWE1cYTAHpjOH3rcTnXQdpEfS/SQ==
|
||||
|
||||
"@types/yargs-parser@*":
|
||||
version "20.2.0"
|
||||
resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-20.2.0.tgz#dd3e6699ba3237f0348cd085e4698780204842f9"
|
||||
|
|
@ -1029,6 +1034,11 @@ domexception@^1.0.1:
|
|||
dependencies:
|
||||
webidl-conversions "^4.0.2"
|
||||
|
||||
domino@^2.1.6:
|
||||
version "2.1.6"
|
||||
resolved "https://registry.yarnpkg.com/domino/-/domino-2.1.6.tgz#fe4ace4310526e5e7b9d12c7de01b7f485a57ffe"
|
||||
integrity sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ==
|
||||
|
||||
ecc-jsbn@~0.1.1:
|
||||
version "0.1.2"
|
||||
resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9"
|
||||
|
|
@ -3250,9 +3260,9 @@ throat@^4.0.0:
|
|||
integrity sha1-iQN8vJLFarGJJua6TLsgDhVnKmo=
|
||||
|
||||
tmpl@1.0.x:
|
||||
version "1.0.4"
|
||||
resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.4.tgz#23640dd7b42d00433911140820e5cf440e521dd1"
|
||||
integrity sha1-I2QN17QtAEM5ERQIIOXPRA5SHdE=
|
||||
version "1.0.5"
|
||||
resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.5.tgz#8683e0b902bb9c20c4f726e3c0b69f36518c07cc"
|
||||
integrity "sha1-hoPguQK7nCDE9ybjwLafNlGMB8w= sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw=="
|
||||
|
||||
to-fast-properties@^2.0.0:
|
||||
version "2.0.0"
|
||||
|
|
@ -3331,6 +3341,13 @@ tunnel-agent@^0.6.0:
|
|||
dependencies:
|
||||
safe-buffer "^5.0.1"
|
||||
|
||||
turndown@^7.1.1:
|
||||
version "7.1.1"
|
||||
resolved "https://registry.yarnpkg.com/turndown/-/turndown-7.1.1.tgz#96992f2d9b40a1a03d3ea61ad31b5a5c751ef77f"
|
||||
integrity sha512-BEkXaWH7Wh7e9bd2QumhfAXk5g34+6QUmmWx+0q6ThaVOLuLUqsnkq35HQ5SBHSaxjSfSM7US5o4lhJNH7B9MA==
|
||||
dependencies:
|
||||
domino "^2.1.6"
|
||||
|
||||
tweetnacl@^0.14.3, tweetnacl@~0.14.0:
|
||||
version "0.14.5"
|
||||
resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64"
|
||||
|
|
|
|||
Loading…
Reference in New Issue