Move scraper code to a separate library

feature-show_image_resolution v0.1.2
neobooru 2019-09-23 19:28:02 +02:00
parent 4b48e686bc
commit 90bb7475a7
16 changed files with 47 additions and 552 deletions

21
LICENSE Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2019 neobooru
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

7
package-lock.json generated
View File

@ -1,6 +1,6 @@
{
"name": "szuru-chrome",
"version": "0.1.1",
"version": "0.1.2",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
@ -4035,6 +4035,11 @@
"integrity": "sha512-iyam8fBuCUpWeKPGpaNMetEocMt364qkCsfL9JuhjXX6dRnguRVOfk2GZaDpPjcOKiiXCPINZC1GczQ7iTq3Zw==",
"dev": true
},
"neo-scraper": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/neo-scraper/-/neo-scraper-0.1.0.tgz",
"integrity": "sha512-XfSFRKwB9vVldWGPecOxWeMfBfNkhOklPbb4qYknnsyW7gkbudY5L3VkqM8DFnsvXyXEGW2rFTsc1JTfigypKg=="
},
"nice-try": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz",

View File

@ -1,6 +1,6 @@
{
"name": "szuru-chrome",
"version": "0.1.1",
"version": "0.1.2",
"description": "Import media from various sites into szurubooru",
"author": "neobooru",
"license": "MIT",
@ -13,6 +13,7 @@
},
"dependencies": {
"axios": "^0.19.0",
"neo-scraper": "^0.1.0",
"ts-optchain": "^0.1.8",
"vue": "^2.6.10",
"webextension-polyfill": "^0.3.1"

View File

@ -1,25 +0,0 @@
/**
* These types are NOT compatible with szurubooru, they are only used in the WebExtension.
*/
import { Safety } from "./SzuruTypes";
export type BooruCategory = "copyright" | "character" | "artist" | "meta";
export class ScrapedTag {
name: string = "";
category?: string;
constructor(name: string, category?: string) {
this.name = name.replace(/\s/g, "_"); // Replace all spaces with underscores
this.category = category;
}
}
// FIXME: LocalPost -> valid szurubooru type
export class ScrapedPost {
safety: Safety = "safe";
source: string = "";
imageUrl: string = "";
tags: ScrapedTag[] = [];
}

View File

@ -1,7 +1,7 @@
import axios, { AxiosRequestConfig } from 'axios';
import { ScrapedPost } from './LocalTypes';
import { SzuruSiteConfig } from './Config';
import { TagsResult, TagCategoriesResult, Post, Tag, SzuruError } from './SzuruTypes';
import axios, { AxiosRequestConfig } from "axios";
import { ScrapedPost } from "neo-scraper";
import { SzuruSiteConfig } from "./Config";
import { TagsResult, TagCategoriesResult, Post, Tag, SzuruError } from "./SzuruTypes";
/**
* A 1:1 wrapper around the szurubooru API.
@ -55,7 +55,7 @@ export default class SzuruWrapper {
async createPost(post: ScrapedPost): Promise<Post> {
var obj = {
tags: post.tags.map(x => x.name),
safety: post.safety,
safety: post.rating,
source: post.source,
contentUrl: post.imageUrl
};

View File

@ -1,88 +0,0 @@
import ILoader from "./ILoader"
import { ScrapedPost, ScrapedTag, BooruCategory } from "../LocalTypes";
export default class Danbooru implements ILoader {
canImport(location: Location): boolean {
return (
location.host == "danbooru.donmai.us" ||
location.host == "safebooru.donmai.us"
);
}
async grabPost(dom: Document): Promise<ScrapedPost | null> {
let post = new ScrapedPost();
post.source = document.location.href;
// Set image url
const originalImageElements = Array.from(document.querySelectorAll("#post-information > ul > li > a"))
.map(x => x as HTMLAnchorElement)
.filter(x => x.parentElement && x.parentElement.innerText.startsWith("Size"));
if (originalImageElements.length > 0) {
post.imageUrl = originalImageElements[0].href;
}
// Set safety
const safetyExp = new RegExp("Rating: (.*)");
const safetyElements = Array.from(document.querySelectorAll("#post-information > ul > li"))
.map(x => x as HTMLLIElement)
.filter(x => x.innerText.startsWith("Rating"));
if (safetyElements.length > 0) {
const matches = safetyElements[0].innerText.match(safetyExp);
if (matches && matches.length > 0) {
switch (matches[1].toLowerCase()) {
case "safe":
post.safety = "safe";
break;
case "questionable":
post.safety = "sketchy";
break;
case "explicit":
post.safety = "unsafe";
break;
}
}
}
// Set tags
const tagElements = Array.from(document.querySelectorAll("#tag-list > ul > li")).map(x => x as HTMLLIElement);
for (const tagElement of tagElements) {
let tagName: string | undefined;
const tagNameElement = tagElement.querySelector(".search-tag") as HTMLAnchorElement;
if (tagNameElement) {
tagName = tagNameElement.innerText;
}
let category: BooruCategory | undefined;
switch (tagElement.className) {
case "category-3":
category = "copyright";
break;
case "category-4":
category = "character";
break;
case "category-1":
category = "artist";
break;
case "category-5":
category = "meta";
break;
}
if (tagName) {
let tag = new ScrapedTag(tagName, category);
post.tags.push(tag);
}
}
// console.dir(post);
// If post has enough filled fields to be considered a valid post
if (post.imageUrl) {
return post;
} else {
return null;
}
}
}

View File

@ -1,27 +0,0 @@
import ILoader from "./ILoader";
import { ScrapedPost } from "../LocalTypes";
export default class Fallback implements ILoader {
canImport(location: Location): boolean {
return false;
}
async grabPost(document: Document): Promise<ScrapedPost | null> {
function getSize(el: HTMLImageElement) {
return el.width * el.height;
}
// Get largest img element on this page
const largestImgElement = Array.from(document.getElementsByTagName('img'))
.reduce((a, b) => getSize(a) > getSize(b) ? a : b);
if (largestImgElement) {
let post = new ScrapedPost();
post.source = document.location.href;
post.imageUrl = largestImgElement.src;
return post;
}
return null;
}
}

View File

@ -1,149 +0,0 @@
import ILoader from "./ILoader"
import { ScrapedPost, ScrapedTag, BooruCategory } from "../LocalTypes";
// Small hack to avoid screwing with semver
enum Version {
v020 = 1, // 0.2.0
v025, // 0.2.5
}
export default class Gelbooru implements ILoader {
canImport(location: Location): boolean {
return (
location.host == "safebooru.org" ||
location.host == "gelbooru.com" ||
location.host == "rule34.xxx"
);
}
async grabPost(dom: Document): Promise<ScrapedPost | null> {
/**
* Make sure to use the `dom` variable and NOT `document`!
*/
let version: Version;
switch (document.location.host) {
case "safebooru.org":
case "rule34.xxx":
version = Version.v020;
break;
case "gelbooru.com":
version = Version.v025;
break;
default:
console.error("Couldn't guess version");
return null;
}
console.log("Gelbooru guessed version: " + version);
let post = new ScrapedPost();
post.source = document.location.href;
// Set image url (direct url to image)
const originalImageElements = document.querySelectorAll("li > a");
for (const idx in originalImageElements) {
const el = originalImageElements[idx] as HTMLAnchorElement;
if (el && el.innerText == "Original image") {
post.imageUrl = el.href;
break;
}
}
// Set safety
const safetyExp = new RegExp("Rating: (.*)");
let safetyElements: NodeListOf<Element>;
if (version == Version.v020) {
safetyElements = document.querySelectorAll("#stats > ul > li");
} else {
safetyElements = document.querySelectorAll("#tag-list > div > li");
}
for (const idx in safetyElements) {
const el = safetyElements[idx] as HTMLLIElement;
if (el && el.innerText) {
// matches is an array where (index)
// 0: full string
// 1: capture group 1 (rating, so 'Safe', 'Questionable', 'Explicit')
const matches = el.innerText.match(safetyExp);
if (matches && matches.length > 0) {
switch (matches[1].toLowerCase()) {
case "safe":
post.safety = "safe";
break;
case "questionable":
post.safety = "sketchy";
break;
case "explicit":
post.safety = "unsafe";
break;
}
}
}
}
// Set tags
let tagElements: NodeListOf<Element>;
if (version == Version.v020) {
tagElements = document.querySelectorAll("#tag-sidebar > li");
} else {
tagElements = document.querySelectorAll("#tag-list > div > li[class^='tag-type']");
}
for (const idx in tagElements) {
const el = tagElements[idx] as HTMLLIElement;
if (el && el.innerText) {
let tagName: string;
if (version == Version.v020) {
const nameElements = el.getElementsByTagName("a");
// Some <li/> don't actually contain a name.
// E.g. the sub-headers ("Copyright", "Character", "General", "Meta") on rule34.xxx
if (nameElements.length > 0) {
tagName = el.getElementsByTagName("a")[0].innerText;
}
else {
continue;
}
} else {
// First <a> is "?"
// Second <a> is "actual_tag_name"
tagName = el.getElementsByTagName("a")[1].innerText;
}
let category: BooruCategory | undefined;
switch (el.className) {
case "tag-type-copyright":
category = "copyright";
break;
case "tag-type-character":
category = "character";
break;
case "tag-type-artist":
category = "artist";
break;
case "tag-type-metadata":
category = "meta";
break;
}
let tag = new ScrapedTag(tagName, category);
post.tags.push(tag);
}
}
// console.dir(post);
// If post has enough filled fields to be considered a valid post
if (post.imageUrl) {
return post;
} else {
return null;
}
}
};

View File

@ -1,6 +0,0 @@
import { ScrapedPost } from "../LocalTypes";
export default interface ILoader {
canImport(location: Location): boolean;
grabPost(dom: Document): Promise<ScrapedPost | null>;
}

View File

@ -1,97 +0,0 @@
import ILoader from "./ILoader"
import { ScrapedPost, ScrapedTag, BooruCategory } from "../LocalTypes";
export default class Moebooru implements ILoader {
canImport(location: Location): boolean {
return (
location.host == "yande.re" ||
location.host == "konachan.com"
);
}
async grabPost(dom: Document): Promise<ScrapedPost | null> {
let post = new ScrapedPost();
post.source = document.location.href;
// Set image url
const originalImageElement = document.querySelector("#highres") as HTMLAnchorElement;
if (originalImageElement) {
post.imageUrl = originalImageElement.href;
}
// Set safety
// Same method as Gelbooru but cleaner code
const safetyExp = new RegExp("Rating: (.*)");
const safetyElements = Array.from(document.querySelectorAll("#stats > ul > li"))
.map(x => x as HTMLLIElement)
.filter(x => x.innerText.startsWith("Rating:"));
if (safetyElements.length > 0) {
const matches = safetyElements[0].innerText.match(safetyExp);
if (matches && matches.length > 0) {
switch (matches[1].toLowerCase()) {
case "safe":
post.safety = "safe";
break;
case "questionable":
post.safety = "sketchy";
break;
case "explicit":
post.safety = "unsafe";
break;
}
}
}
// Set tags
const tagElements = Array.from(document.querySelectorAll("#tag-sidebar > li"))
.map(x => x as HTMLLIElement);
for (const el of tagElements) {
let tagName: string;
let tagNameElements = el.getElementsByTagName("a");
if (tagNameElements.length > 1) {
// First <a> is "?"
// Second <a> is "actual_tag_name"
tagName = tagNameElements[1].innerText;
} else {
// Skip if no tag name is found
continue;
}
let category: BooruCategory | undefined;
// Loop over all classes because some sites (konachan)
// have mutliple classes, e.g. "tag-link tag-type-copyright"
for (const className of Array.from(el.classList)) {
switch (className) {
case "tag-type-copyright":
category = "copyright";
break;
case "tag-type-character":
category = "character";
break;
case "tag-type-artist":
category = "artist";
break;
case "tag-type-metadata":
category = "meta";
break;
}
}
let tag = new ScrapedTag(tagName, category);
post.tags.push(tag);
}
// console.dir(post);
// If post has enough filled fields to be considered a valid post
if (post.imageUrl) {
return post;
} else {
return null;
}
}
}

View File

@ -1,37 +0,0 @@
import ILoader from "./ILoader"
import { ScrapedPost, ScrapedTag } from "../LocalTypes";
export default class Reddit implements ILoader {
canImport(location: Location): boolean {
return (
location.host == "reddit.com" ||
location.host == "www.reddit.com" ||
location.host == "old.reddit.com"
);
}
async grabPost(dom: Document): Promise<ScrapedPost | null> {
let post = new ScrapedPost();
post.source = document.location.href;
// Set image url
// For old.reddit.com
const oldLinkElements = document.querySelectorAll("a.title");
if (oldLinkElements.length > 0) {
post.imageUrl = (oldLinkElements[0] as HTMLAnchorElement).href;
}
// For new reddit
const newLinkElements = document.querySelectorAll("div[data-test-id='post-content'] > div > a");
if (newLinkElements.length > 0) {
post.imageUrl = (newLinkElements[0] as HTMLAnchorElement).href;
}
// If post has enough filled fields to be considered a valid post
if (post.imageUrl) {
return post;
} else {
return null;
}
}
}

View File

@ -1,43 +0,0 @@
import ILoader from "./ILoader"
import { ScrapedPost, ScrapedTag } from "../LocalTypes";
export default class Shimmie2 implements ILoader {
canImport(location: Location): boolean {
return location.host == "rule34.paheal.net";
}
async grabPost(dom: Document): Promise<ScrapedPost | null> {
let post = new ScrapedPost();
post.source = document.location.href;
// Set image url
const originalImageElements = Array.from(document.querySelectorAll(".image_info > tbody > tr > td > a"))
.map(x => x as HTMLAnchorElement)
.filter(x => x.innerText == "Image Only");
if (originalImageElements.length > 0) {
post.imageUrl = originalImageElements[0].href;
}
// Set safety
post.safety = "unsafe"; // Usually it is unsafe
// Set tags
// Shimmie2 doesn't support categories so we don't need to worry about those.
// They also use uppercase tags, so we toLowerCase() those.
// Tags might not match with other booru-likes, but that is for the user to figure out.
// For example by deleting 'wrong' tags or setting up some aliases in szurubooru.
post.tags = Array.from(document.querySelectorAll("a.tag_name"))
.map(x => (x as HTMLAnchorElement).innerText.toLowerCase())
.map(x => new ScrapedTag(x));
// console.dir(post);
// If post has enough filled fields to be considered a valid post
if (post.imageUrl) {
return post;
} else {
return null;
}
}
}

View File

@ -1,41 +0,0 @@
import ILoader from "./ILoader"
import { ScrapedPost, ScrapedTag } from "../LocalTypes";
function getOriginalImage(imageUrl: string) {
const x = imageUrl.replace(/:large|:med|:small|:orig$/, ""); // Remove `:large`, `:small` etc from url
return x + ":orig"; // Add :orig to url (to get original image)
}
export default class Twitter implements ILoader {
canImport(location: Location): boolean {
return location.host == "twitter.com";
}
async grabPost(dom: Document): Promise<ScrapedPost | null> {
let post = new ScrapedPost();
post.source = document.location.href;
// Set image url
// When clicked on post (not fullscreen, with post text etc)
const overlayElement = document.querySelector("#permalink-overlay-dialog");
if (overlayElement) {
const cardImageElement = overlayElement.querySelector("div[data-element-context='platform_photo_card'] > img");
if (cardImageElement) {
post.imageUrl = getOriginalImage((cardImageElement as HTMLImageElement).src);
}
}
// When clicked on image (fullscreen)
const mediaImageElements = document.querySelectorAll("div.Gallery-media > img.media-image");
if (mediaImageElements.length > 0) {
post.imageUrl = getOriginalImage((mediaImageElements[0] as HTMLImageElement).src);
}
// If post has enough filled fields to be considered a valid post
if (post.imageUrl) {
return post;
} else {
return null;
}
}
}

View File

@ -1,7 +0,0 @@
export { default as Gelbooru } from "./Gelbooru";
export { default as Danbooru } from "./Danbooru";
export { default as Shimmie2 } from "./Shimmie2";
export { default as Moebooru } from "./Moebooru";
export { default as Reddit } from "./Reddit";
export { default as Twitter } from "./Twitter";
export { default as Fallback } from "./Fallback";

View File

@ -25,19 +25,19 @@
<span class="block-title">Safety</span>
<label class="container">
<input type="radio" value="safe" v-model="post.safety" />
<input type="radio" value="safe" v-model="post.rating" />
<span class="checkmark"></span>
Safe
</label>
<label class="container">
<input type="radio" value="sketchy" v-model="post.safety" />
<input type="radio" value="sketchy" v-model="post.rating" />
<span class="checkmark"></span>
Sketchy
</label>
<label class="container">
<input type="radio" value="unsafe" v-model="post.safety" />
<input type="radio" value="unsafe" v-model="post.rating" />
<span class="checkmark"></span>
Unsafe
</label>
@ -88,7 +88,7 @@
<script lang="ts">
import Vue from "vue";
import { browser } from "webextension-polyfill-ts";
import { ScrapedPost, ScrapedTag } from "../LocalTypes";
import { ScrapedPost, ScrapedTag, ScrapeResults } from "neo-scraper";
import SzuruWrapper from "../SzuruWrapper";
import { Post, SzuruError } from "../SzuruTypes";
import { Config, SzuruSiteConfig } from "../Config";
@ -123,7 +123,11 @@ export default Vue.extend({
}))[0];
// Send 'grab_post' to the content script on the active tab
const post = (await browser.tabs.sendMessage(activeTab.id!, "grab_post")) as ScrapedPost;
const x = (await browser.tabs.sendMessage(activeTab.id!, "grab_post"));
// We need to create a new ScrapeResults object and fill it with our data, because the get_posts()
// method gets 'lost' when sent from the contentscript to the popup (it gets JSON.stringified and any prototype defines are lost there)
const res = Object.assign(new ScrapeResults(), x);
const post = res.posts.length > 0 ? res.posts[0] : null;
if (post) {
this.post = post;

View File

@ -1,25 +1,9 @@
import { NeoScraper, ScrapeResults } from "neo-scraper";
import { browser, Runtime } from "webextension-polyfill-ts";
import * as loaders from "../loaders";
import { ScrapedPost } from "../LocalTypes";
import ILoader from "../loaders/ILoader";
async function grabPost(): Promise<ScrapedPost | null> {
for (const loaderName in loaders) {
// HACK:
const loader = new (loaders as any)[loaderName]() as ILoader;
if (loader.canImport(window.location)) {
console.log(`Using ${loaderName} importer to scrape this page`)
return await loader.grabPost(document);
}
}
// Try using the Fallback loader if no other loader can scrape this page
const post = new loaders.Fallback().grabPost(document);
if (post) {
return post;
}
return null;
function grabPost(): ScrapeResults {
var scraper = new NeoScraper();
return scraper.scrapeDocument(document, undefined, true);
}
async function messageHandler(message: any, sender: Runtime.MessageSender): Promise<any> {