Cleanup + add fallback loader (grabs largest image on page)

feature-show_image_resolution
neobooru 2019-09-19 23:21:52 +02:00
parent e3286bc7e8
commit e2867bb3ff
17 changed files with 119 additions and 81 deletions

2
package-lock.json generated
View File

@ -1,6 +1,6 @@
{ {
"name": "szuru-chrome", "name": "szuru-chrome",
"version": "0.1.0", "version": "0.1.1",
"lockfileVersion": 1, "lockfileVersion": 1,
"requires": true, "requires": true,
"dependencies": { "dependencies": {

View File

@ -4,9 +4,9 @@
import { Safety } from "./SzuruTypes"; import { Safety } from "./SzuruTypes";
export type LocalCategory = "copyright" | "character" | "artist" | "meta"; export type BooruCategory = "copyright" | "character" | "artist" | "meta";
export class LocalTag { export class ScrapedTag {
name: string = ""; name: string = "";
category?: string; category?: string;
@ -17,15 +17,9 @@ export class LocalTag {
} }
// FIXME: LocalPost -> valid szurubooru type // FIXME: LocalPost -> valid szurubooru type
export class LocalPost { export class ScrapedPost {
safety: Safety = "safe"; safety: Safety = "safe";
source: string = ""; source: string = "";
imageUrl: string = ""; imageUrl: string = "";
tags: LocalTag[] = []; tags: ScrapedTag[] = [];
}
export interface LocalError {
name: string;
title: string;
description: string;
} }

View File

@ -4,6 +4,15 @@
export type Safety = "safe" | "sketchy" | "unsafe"; export type Safety = "safe" | "sketchy" | "unsafe";
export type TagsResult = PagedSearchResult<Tag>
export type TagCategoriesResult = UnpagedSearchResult<TagCategory>
export interface SzuruError {
name: string;
title: string;
description: string;
}
export interface PagedSearchResult<T> { export interface PagedSearchResult<T> {
query: string; query: string;
offset: number; offset: number;
@ -81,6 +90,3 @@ export interface Post {
notes: any[]; // Note resource notes: any[]; // Note resource
comments: any[]; // Comment resource comments: any[]; // Comment resource
} }
export type TagsResult = PagedSearchResult<Tag>
export type TagCategoriesResult = UnpagedSearchResult<TagCategory>

View File

@ -1,7 +1,7 @@
import axios, { AxiosRequestConfig } from 'axios'; import axios, { AxiosRequestConfig } from 'axios';
import { LocalPost, LocalError } from './LocalTypes'; import { ScrapedPost } from './LocalTypes';
import { SzuruSiteConfig } from './Config'; import { SzuruSiteConfig } from './Config';
import { TagsResult, TagCategoriesResult, Post, Tag } from './SzuruTypes'; import { TagsResult, TagCategoriesResult, Post, Tag, SzuruError } from './SzuruTypes';
/** /**
* A 1:1 wrapper around the szurubooru API. * A 1:1 wrapper around the szurubooru API.
@ -52,7 +52,7 @@ export default class SzuruWrapper {
return (await this.apiGet("tag-categories")).data; return (await this.apiGet("tag-categories")).data;
} }
async createPost(post: LocalPost): Promise<Post> { async createPost(post: ScrapedPost): Promise<Post> {
var obj = { var obj = {
tags: post.tags.map(x => x.name), tags: post.tags.map(x => x.name),
safety: post.safety, safety: post.safety,
@ -111,7 +111,7 @@ export default class SzuruWrapper {
try { try {
return await axios(config); return await axios(config);
} catch (ex) { } catch (ex) {
const error = ex.response.data as LocalError; const error = ex.response.data as SzuruError;
throw error ? error : ex; throw error ? error : ex;
} }
} }

View File

@ -101,6 +101,10 @@ a.remove-tag {
text-decoration: none; text-decoration: none;
} }
.messages > li:first-child {
margin-top: 10px;
}
.messages > li:not(:last-child) { .messages > li:not(:last-child) {
margin-bottom: 5px; margin-bottom: 5px;
} }

View File

@ -1,5 +1,5 @@
import ILoader from "./ILoader" import ILoader from "./ILoader"
import { LocalPost, LocalTag, LocalCategory } from "../LocalTypes"; import { ScrapedPost, ScrapedTag, BooruCategory } from "../LocalTypes";
export default class Danbooru implements ILoader { export default class Danbooru implements ILoader {
canImport(location: Location): boolean { canImport(location: Location): boolean {
@ -9,12 +9,12 @@ export default class Danbooru implements ILoader {
); );
} }
async grabPost(dom: Document): Promise<LocalPost | null> { async grabPost(dom: Document): Promise<ScrapedPost | null> {
let post = new LocalPost(); let post = new ScrapedPost();
post.source = dom.location.href; post.source = document.location.href;
// Set image url // Set image url
const originalImageElements = Array.from(dom.querySelectorAll("#post-information > ul > li > a")) const originalImageElements = Array.from(document.querySelectorAll("#post-information > ul > li > a"))
.map(x => x as HTMLAnchorElement) .map(x => x as HTMLAnchorElement)
.filter(x => x.parentElement && x.parentElement.innerText.startsWith("Size")); .filter(x => x.parentElement && x.parentElement.innerText.startsWith("Size"));
@ -46,7 +46,7 @@ export default class Danbooru implements ILoader {
} }
// Set tags // Set tags
const tagElements = Array.from(dom.querySelectorAll("#tag-list > ul > li")).map(x => x as HTMLLIElement); const tagElements = Array.from(document.querySelectorAll("#tag-list > ul > li")).map(x => x as HTMLLIElement);
for (const tagElement of tagElements) { for (const tagElement of tagElements) {
let tagName: string | undefined; let tagName: string | undefined;
const tagNameElement = tagElement.querySelector(".search-tag") as HTMLAnchorElement; const tagNameElement = tagElement.querySelector(".search-tag") as HTMLAnchorElement;
@ -54,7 +54,7 @@ export default class Danbooru implements ILoader {
tagName = tagNameElement.innerText; tagName = tagNameElement.innerText;
} }
let category: LocalCategory | undefined; let category: BooruCategory | undefined;
switch (tagElement.className) { switch (tagElement.className) {
case "category-3": case "category-3":
category = "copyright"; category = "copyright";
@ -71,7 +71,7 @@ export default class Danbooru implements ILoader {
} }
if (tagName) { if (tagName) {
let tag = new LocalTag(tagName, category); let tag = new ScrapedTag(tagName, category);
post.tags.push(tag); post.tags.push(tag);
} }
} }

27
src/loaders/Fallback.ts Normal file
View File

@ -0,0 +1,27 @@
import ILoader from "./ILoader";
import { ScrapedPost } from "../LocalTypes";
export default class Fallback implements ILoader {
canImport(location: Location): boolean {
return false;
}
async grabPost(document: Document): Promise<ScrapedPost | null> {
function getSize(el: HTMLImageElement) {
return el.width * el.height;
}
// Get largest img element on this page
const largestImgElement = Array.from(document.getElementsByTagName('img'))
.reduce((a, b) => getSize(a) > getSize(b) ? a : b);
if (largestImgElement) {
let post = new ScrapedPost();
post.source = document.location.href;
post.imageUrl = largestImgElement.src;
return post;
}
return null;
}
}

View File

@ -1,5 +1,5 @@
import ILoader from "./ILoader" import ILoader from "./ILoader"
import { LocalPost, LocalTag, LocalCategory } from "../LocalTypes"; import { ScrapedPost, ScrapedTag, BooruCategory } from "../LocalTypes";
// Small hack to avoid screwing with semver // Small hack to avoid screwing with semver
enum Version { enum Version {
@ -16,13 +16,13 @@ export default class Gelbooru implements ILoader {
); );
} }
async grabPost(dom: Document): Promise<LocalPost | null> { async grabPost(dom: Document): Promise<ScrapedPost | null> {
/** /**
* Make sure to use the `dom` variable and NOT `document`! * Make sure to use the `dom` variable and NOT `document`!
*/ */
let version: Version; let version: Version;
switch (dom.location.host) { switch (document.location.host) {
case "safebooru.org": case "safebooru.org":
case "rule34.xxx": case "rule34.xxx":
version = Version.v020; version = Version.v020;
@ -37,11 +37,11 @@ export default class Gelbooru implements ILoader {
console.log("Gelbooru guessed version: " + version); console.log("Gelbooru guessed version: " + version);
let post = new LocalPost(); let post = new ScrapedPost();
post.source = dom.location.href; post.source = document.location.href;
// Set image url (direct url to image) // Set image url (direct url to image)
const originalImageElements = dom.querySelectorAll("li > a"); const originalImageElements = document.querySelectorAll("li > a");
for (const idx in originalImageElements) { for (const idx in originalImageElements) {
const el = originalImageElements[idx] as HTMLAnchorElement; const el = originalImageElements[idx] as HTMLAnchorElement;
@ -56,9 +56,9 @@ export default class Gelbooru implements ILoader {
let safetyElements: NodeListOf<Element>; let safetyElements: NodeListOf<Element>;
if (version == Version.v020) { if (version == Version.v020) {
safetyElements = dom.querySelectorAll("#stats > ul > li"); safetyElements = document.querySelectorAll("#stats > ul > li");
} else { } else {
safetyElements = dom.querySelectorAll("#tag-list > div > li"); safetyElements = document.querySelectorAll("#tag-list > div > li");
} }
for (const idx in safetyElements) { for (const idx in safetyElements) {
@ -88,9 +88,9 @@ export default class Gelbooru implements ILoader {
let tagElements: NodeListOf<Element>; let tagElements: NodeListOf<Element>;
if (version == Version.v020) { if (version == Version.v020) {
tagElements = dom.querySelectorAll("#tag-sidebar > li"); tagElements = document.querySelectorAll("#tag-sidebar > li");
} else { } else {
tagElements = dom.querySelectorAll("#tag-list > div > li[class^='tag-type']"); tagElements = document.querySelectorAll("#tag-list > div > li[class^='tag-type']");
} }
for (const idx in tagElements) { for (const idx in tagElements) {
@ -115,7 +115,7 @@ export default class Gelbooru implements ILoader {
tagName = el.getElementsByTagName("a")[1].innerText; tagName = el.getElementsByTagName("a")[1].innerText;
} }
let category: LocalCategory | undefined; let category: BooruCategory | undefined;
switch (el.className) { switch (el.className) {
case "tag-type-copyright": case "tag-type-copyright":
@ -132,7 +132,7 @@ export default class Gelbooru implements ILoader {
break; break;
} }
let tag = new LocalTag(tagName, category); let tag = new ScrapedTag(tagName, category);
post.tags.push(tag); post.tags.push(tag);
} }
} }

View File

@ -1,6 +1,6 @@
import { LocalPost } from "../LocalTypes"; import { ScrapedPost } from "../LocalTypes";
export default interface ILoader { export default interface ILoader {
canImport(location: Location): boolean; canImport(location: Location): boolean;
grabPost(dom: Document): Promise<LocalPost | null>; grabPost(dom: Document): Promise<ScrapedPost | null>;
} }

View File

@ -1,5 +1,5 @@
import ILoader from "./ILoader" import ILoader from "./ILoader"
import { LocalPost, LocalTag, LocalCategory } from "../LocalTypes"; import { ScrapedPost, ScrapedTag, BooruCategory } from "../LocalTypes";
export default class Moebooru implements ILoader { export default class Moebooru implements ILoader {
canImport(location: Location): boolean { canImport(location: Location): boolean {
@ -9,12 +9,12 @@ export default class Moebooru implements ILoader {
); );
} }
async grabPost(dom: Document): Promise<LocalPost | null> { async grabPost(dom: Document): Promise<ScrapedPost | null> {
let post = new LocalPost(); let post = new ScrapedPost();
post.source = dom.location.href; post.source = document.location.href;
// Set image url // Set image url
const originalImageElement = dom.querySelector("#highres") as HTMLAnchorElement; const originalImageElement = document.querySelector("#highres") as HTMLAnchorElement;
if (originalImageElement) { if (originalImageElement) {
post.imageUrl = originalImageElement.href; post.imageUrl = originalImageElement.href;
} }
@ -22,7 +22,7 @@ export default class Moebooru implements ILoader {
// Set safety // Set safety
// Same method as Gelbooru but cleaner code // Same method as Gelbooru but cleaner code
const safetyExp = new RegExp("Rating: (.*)"); const safetyExp = new RegExp("Rating: (.*)");
const safetyElements = Array.from(dom.querySelectorAll("#stats > ul > li")) const safetyElements = Array.from(document.querySelectorAll("#stats > ul > li"))
.map(x => x as HTMLLIElement) .map(x => x as HTMLLIElement)
.filter(x => x.innerText.startsWith("Rating:")); .filter(x => x.innerText.startsWith("Rating:"));
@ -44,7 +44,7 @@ export default class Moebooru implements ILoader {
} }
// Set tags // Set tags
const tagElements = Array.from(dom.querySelectorAll("#tag-sidebar > li")) const tagElements = Array.from(document.querySelectorAll("#tag-sidebar > li"))
.map(x => x as HTMLLIElement); .map(x => x as HTMLLIElement);
for (const el of tagElements) { for (const el of tagElements) {
@ -60,7 +60,7 @@ export default class Moebooru implements ILoader {
continue; continue;
} }
let category: LocalCategory | undefined; let category: BooruCategory | undefined;
// Loop over all classes because some sites (konachan) // Loop over all classes because some sites (konachan)
// have mutliple classes, e.g. "tag-link tag-type-copyright" // have mutliple classes, e.g. "tag-link tag-type-copyright"
@ -81,7 +81,7 @@ export default class Moebooru implements ILoader {
} }
} }
let tag = new LocalTag(tagName, category); let tag = new ScrapedTag(tagName, category);
post.tags.push(tag); post.tags.push(tag);
} }

View File

@ -1,5 +1,5 @@
import ILoader from "./ILoader" import ILoader from "./ILoader"
import { LocalPost, LocalTag } from "../LocalTypes"; import { ScrapedPost, ScrapedTag } from "../LocalTypes";
export default class Reddit implements ILoader { export default class Reddit implements ILoader {
canImport(location: Location): boolean { canImport(location: Location): boolean {
@ -10,19 +10,19 @@ export default class Reddit implements ILoader {
); );
} }
async grabPost(dom: Document): Promise<LocalPost | null> { async grabPost(dom: Document): Promise<ScrapedPost | null> {
let post = new LocalPost(); let post = new ScrapedPost();
post.source = dom.location.href; post.source = document.location.href;
// Set image url // Set image url
// For old.reddit.com // For old.reddit.com
const oldLinkElements = dom.querySelectorAll("a.title"); const oldLinkElements = document.querySelectorAll("a.title");
if (oldLinkElements.length > 0) { if (oldLinkElements.length > 0) {
post.imageUrl = (oldLinkElements[0] as HTMLAnchorElement).href; post.imageUrl = (oldLinkElements[0] as HTMLAnchorElement).href;
} }
// For new reddit // For new reddit
const newLinkElements = dom.querySelectorAll("div[data-test-id='post-content'] > div > a"); const newLinkElements = document.querySelectorAll("div[data-test-id='post-content'] > div > a");
if (newLinkElements.length > 0) { if (newLinkElements.length > 0) {
post.imageUrl = (newLinkElements[0] as HTMLAnchorElement).href; post.imageUrl = (newLinkElements[0] as HTMLAnchorElement).href;
} }

View File

@ -1,17 +1,17 @@
import ILoader from "./ILoader" import ILoader from "./ILoader"
import { LocalPost, LocalTag } from "../LocalTypes"; import { ScrapedPost, ScrapedTag } from "../LocalTypes";
export default class Shimmie2 implements ILoader { export default class Shimmie2 implements ILoader {
canImport(location: Location): boolean { canImport(location: Location): boolean {
return location.host == "rule34.paheal.net"; return location.host == "rule34.paheal.net";
} }
async grabPost(dom: Document): Promise<LocalPost | null> { async grabPost(dom: Document): Promise<ScrapedPost | null> {
let post = new LocalPost(); let post = new ScrapedPost();
post.source = dom.location.href; post.source = document.location.href;
// Set image url // Set image url
const originalImageElements = Array.from(dom.querySelectorAll(".image_info > tbody > tr > td > a")) const originalImageElements = Array.from(document.querySelectorAll(".image_info > tbody > tr > td > a"))
.map(x => x as HTMLAnchorElement) .map(x => x as HTMLAnchorElement)
.filter(x => x.innerText == "Image Only"); .filter(x => x.innerText == "Image Only");
@ -27,9 +27,9 @@ export default class Shimmie2 implements ILoader {
// They also use uppercase tags, so we toLowerCase() those. // They also use uppercase tags, so we toLowerCase() those.
// Tags might not match with other booru-likes, but that is for the user to figure out. // Tags might not match with other booru-likes, but that is for the user to figure out.
// For example by deleting 'wrong' tags or setting up some aliases in szurubooru. // For example by deleting 'wrong' tags or setting up some aliases in szurubooru.
post.tags = Array.from(dom.querySelectorAll("a.tag_name")) post.tags = Array.from(document.querySelectorAll("a.tag_name"))
.map(x => (x as HTMLAnchorElement).innerText.toLowerCase()) .map(x => (x as HTMLAnchorElement).innerText.toLowerCase())
.map(x => new LocalTag(x)); .map(x => new ScrapedTag(x));
// console.dir(post); // console.dir(post);

View File

@ -1,5 +1,5 @@
import ILoader from "./ILoader" import ILoader from "./ILoader"
import { LocalPost, LocalTag } from "../LocalTypes"; import { ScrapedPost, ScrapedTag } from "../LocalTypes";
function getOriginalImage(imageUrl: string) { function getOriginalImage(imageUrl: string) {
const x = imageUrl.replace(/:large|:med|:small|:orig$/, ""); // Remove `:large`, `:small` etc from url const x = imageUrl.replace(/:large|:med|:small|:orig$/, ""); // Remove `:large`, `:small` etc from url
@ -11,13 +11,13 @@ export default class Twitter implements ILoader {
return location.host == "twitter.com"; return location.host == "twitter.com";
} }
async grabPost(dom: Document): Promise<LocalPost | null> { async grabPost(dom: Document): Promise<ScrapedPost | null> {
let post = new LocalPost(); let post = new ScrapedPost();
post.source = dom.location.href; post.source = document.location.href;
// Set image url // Set image url
// When clicked on post (not fullscreen, with post text etc) // When clicked on post (not fullscreen, with post text etc)
const overlayElement = dom.querySelector("#permalink-overlay-dialog"); const overlayElement = document.querySelector("#permalink-overlay-dialog");
if (overlayElement) { if (overlayElement) {
const cardImageElement = overlayElement.querySelector("div[data-element-context='platform_photo_card'] > img"); const cardImageElement = overlayElement.querySelector("div[data-element-context='platform_photo_card'] > img");
if (cardImageElement) { if (cardImageElement) {
@ -26,7 +26,7 @@ export default class Twitter implements ILoader {
} }
// When clicked on image (fullscreen) // When clicked on image (fullscreen)
const mediaImageElements = dom.querySelectorAll("div.Gallery-media > img.media-image"); const mediaImageElements = document.querySelectorAll("div.Gallery-media > img.media-image");
if (mediaImageElements.length > 0) { if (mediaImageElements.length > 0) {
post.imageUrl = getOriginalImage((mediaImageElements[0] as HTMLImageElement).src); post.imageUrl = getOriginalImage((mediaImageElements[0] as HTMLImageElement).src);
} }

View File

@ -3,4 +3,5 @@ export { default as Danbooru } from "./Danbooru";
export { default as Shimmie2 } from "./Shimmie2"; export { default as Shimmie2 } from "./Shimmie2";
export { default as Moebooru } from "./Moebooru"; export { default as Moebooru } from "./Moebooru";
export { default as Reddit } from "./Reddit"; export { default as Reddit } from "./Reddit";
export { default as Twitter } from "./Twitter" export { default as Twitter } from "./Twitter";
export { default as Fallback } from "./Fallback";

View File

@ -1,7 +1,7 @@
<template> <template>
<div class="popup-container"> <div class="popup-container">
<div class="popup-row"> <div class="popup-row">
<div class="popup-column1" style="margin: 10px 10px 0px 10px;"> <div class="popup-column1" style="margin: 0px 10px;">
<ul class="messages"> <ul class="messages">
<li <li
v-for="msg in messages" v-for="msg in messages"
@ -88,9 +88,9 @@
<script lang="ts"> <script lang="ts">
import Vue from "vue"; import Vue from "vue";
import { browser } from "webextension-polyfill-ts"; import { browser } from "webextension-polyfill-ts";
import { LocalPost, LocalTag, LocalError } from "../LocalTypes"; import { ScrapedPost, ScrapedTag } from "../LocalTypes";
import SzuruWrapper from "../SzuruWrapper"; import SzuruWrapper from "../SzuruWrapper";
import { Post } from "../SzuruTypes"; import { Post, SzuruError } from "../SzuruTypes";
import { Config, SzuruSiteConfig } from "../Config"; import { Config, SzuruSiteConfig } from "../Config";
type MessageType = "error" | "info" | "success"; type MessageType = "error" | "info" | "success";
@ -110,7 +110,7 @@ export default Vue.extend({
return { return {
activeSite: null as SzuruSiteConfig | null, activeSite: null as SzuruSiteConfig | null,
szuru: null as SzuruWrapper | null, szuru: null as SzuruWrapper | null,
post: new LocalPost(), post: new ScrapedPost(),
messages: [] as Message[] messages: [] as Message[]
}; };
}, },
@ -123,7 +123,7 @@ export default Vue.extend({
}))[0]; }))[0];
// Send 'grab_post' to the content script on the active tab // Send 'grab_post' to the content script on the active tab
const post = (await browser.tabs.sendMessage(activeTab.id!, "grab_post")) as LocalPost; const post = (await browser.tabs.sendMessage(activeTab.id!, "grab_post")) as ScrapedPost;
if (post) { if (post) {
this.post = post; this.post = post;
@ -181,7 +181,7 @@ export default Vue.extend({
tagsMsg.type = "success"; tagsMsg.type = "success";
} }
} catch (ex) { } catch (ex) {
const error = ex as LocalError; const error = ex as SzuruError;
if (error) { if (error) {
this.clearMessages(); this.clearMessages();
@ -200,7 +200,7 @@ export default Vue.extend({
const url = browser.extension.getURL("options/options.html"); const url = browser.extension.getURL("options/options.html");
window.open(url); window.open(url);
}, },
getTagClasses(tag: LocalTag): string[] { getTagClasses(tag: ScrapedTag): string[] {
let classes: string[] = []; let classes: string[] = [];
if (tag.category) { if (tag.category) {
@ -211,7 +211,7 @@ export default Vue.extend({
return classes; return classes;
}, },
removeTag(tag: LocalTag) { removeTag(tag: ScrapedTag) {
if (this.post) { if (this.post) {
const idx = this.post.tags.indexOf(tag); const idx = this.post.tags.indexOf(tag);
if (idx != -1) { if (idx != -1) {

View File

@ -1,9 +1,9 @@
import { browser, Runtime } from "webextension-polyfill-ts"; import { browser, Runtime } from "webextension-polyfill-ts";
import * as loaders from "../loaders"; import * as loaders from "../loaders";
import { LocalPost } from "../LocalTypes"; import { ScrapedPost } from "../LocalTypes";
import ILoader from "../loaders/ILoader"; import ILoader from "../loaders/ILoader";
async function grabPost(): Promise<LocalPost | null> { async function grabPost(): Promise<ScrapedPost | null> {
for (const loaderName in loaders) { for (const loaderName in loaders) {
// HACK: // HACK:
const loader = new (loaders as any)[loaderName]() as ILoader; const loader = new (loaders as any)[loaderName]() as ILoader;
@ -11,6 +11,12 @@ async function grabPost(): Promise<LocalPost | null> {
console.log(`Using ${loaderName} importer to scrape this page`) console.log(`Using ${loaderName} importer to scrape this page`)
return await loader.grabPost(document); return await loader.grabPost(document);
} }
// Try using the Fallback loader if no other loader can scrape this page
const post = new loaders.Fallback().grabPost(document);
if (post) {
return post;
}
} }
return null; return null;

View File

@ -16,10 +16,10 @@
"plugins": [ "plugins": [
{ {
"transform": "ts-optchain/transform" "transform": "ts-optchain/transform"
}, }
] ]
}, },
"include": [ "include": [
"./src/**/*" "./src/**/*"
], ]
} }