Cleanup + add fallback loader (grabs largest image on page)

feature-show_image_resolution
neobooru 2019-09-19 23:21:52 +02:00
parent e3286bc7e8
commit e2867bb3ff
17 changed files with 119 additions and 81 deletions

2
package-lock.json generated
View File

@ -1,6 +1,6 @@
{
"name": "szuru-chrome",
"version": "0.1.0",
"version": "0.1.1",
"lockfileVersion": 1,
"requires": true,
"dependencies": {

View File

@ -4,9 +4,9 @@
import { Safety } from "./SzuruTypes";
export type LocalCategory = "copyright" | "character" | "artist" | "meta";
export type BooruCategory = "copyright" | "character" | "artist" | "meta";
export class LocalTag {
export class ScrapedTag {
name: string = "";
category?: string;
@ -17,15 +17,9 @@ export class LocalTag {
}
// FIXME: LocalPost -> valid szurubooru type
export class LocalPost {
export class ScrapedPost {
safety: Safety = "safe";
source: string = "";
imageUrl: string = "";
tags: LocalTag[] = [];
}
export interface LocalError {
name: string;
title: string;
description: string;
tags: ScrapedTag[] = [];
}

View File

@ -4,6 +4,15 @@
export type Safety = "safe" | "sketchy" | "unsafe";
export type TagsResult = PagedSearchResult<Tag>
export type TagCategoriesResult = UnpagedSearchResult<TagCategory>
export interface SzuruError {
name: string;
title: string;
description: string;
}
export interface PagedSearchResult<T> {
query: string;
offset: number;
@ -81,6 +90,3 @@ export interface Post {
notes: any[]; // Note resource
comments: any[]; // Comment resource
}
export type TagsResult = PagedSearchResult<Tag>
export type TagCategoriesResult = UnpagedSearchResult<TagCategory>

View File

@ -1,7 +1,7 @@
import axios, { AxiosRequestConfig } from 'axios';
import { LocalPost, LocalError } from './LocalTypes';
import { ScrapedPost } from './LocalTypes';
import { SzuruSiteConfig } from './Config';
import { TagsResult, TagCategoriesResult, Post, Tag } from './SzuruTypes';
import { TagsResult, TagCategoriesResult, Post, Tag, SzuruError } from './SzuruTypes';
/**
* A 1:1 wrapper around the szurubooru API.
@ -52,7 +52,7 @@ export default class SzuruWrapper {
return (await this.apiGet("tag-categories")).data;
}
async createPost(post: LocalPost): Promise<Post> {
async createPost(post: ScrapedPost): Promise<Post> {
var obj = {
tags: post.tags.map(x => x.name),
safety: post.safety,
@ -111,7 +111,7 @@ export default class SzuruWrapper {
try {
return await axios(config);
} catch (ex) {
const error = ex.response.data as LocalError;
const error = ex.response.data as SzuruError;
throw error ? error : ex;
}
}

View File

@ -101,6 +101,10 @@ a.remove-tag {
text-decoration: none;
}
.messages > li:first-child {
margin-top: 10px;
}
.messages > li:not(:last-child) {
margin-bottom: 5px;
}

View File

@ -1,5 +1,5 @@
import ILoader from "./ILoader"
import { LocalPost, LocalTag, LocalCategory } from "../LocalTypes";
import { ScrapedPost, ScrapedTag, BooruCategory } from "../LocalTypes";
export default class Danbooru implements ILoader {
canImport(location: Location): boolean {
@ -9,12 +9,12 @@ export default class Danbooru implements ILoader {
);
}
async grabPost(dom: Document): Promise<LocalPost | null> {
let post = new LocalPost();
post.source = dom.location.href;
async grabPost(dom: Document): Promise<ScrapedPost | null> {
let post = new ScrapedPost();
post.source = document.location.href;
// Set image url
const originalImageElements = Array.from(dom.querySelectorAll("#post-information > ul > li > a"))
const originalImageElements = Array.from(document.querySelectorAll("#post-information > ul > li > a"))
.map(x => x as HTMLAnchorElement)
.filter(x => x.parentElement && x.parentElement.innerText.startsWith("Size"));
@ -46,7 +46,7 @@ export default class Danbooru implements ILoader {
}
// Set tags
const tagElements = Array.from(dom.querySelectorAll("#tag-list > ul > li")).map(x => x as HTMLLIElement);
const tagElements = Array.from(document.querySelectorAll("#tag-list > ul > li")).map(x => x as HTMLLIElement);
for (const tagElement of tagElements) {
let tagName: string | undefined;
const tagNameElement = tagElement.querySelector(".search-tag") as HTMLAnchorElement;
@ -54,7 +54,7 @@ export default class Danbooru implements ILoader {
tagName = tagNameElement.innerText;
}
let category: LocalCategory | undefined;
let category: BooruCategory | undefined;
switch (tagElement.className) {
case "category-3":
category = "copyright";
@ -71,7 +71,7 @@ export default class Danbooru implements ILoader {
}
if (tagName) {
let tag = new LocalTag(tagName, category);
let tag = new ScrapedTag(tagName, category);
post.tags.push(tag);
}
}

27
src/loaders/Fallback.ts Normal file
View File

@ -0,0 +1,27 @@
import ILoader from "./ILoader";
import { ScrapedPost } from "../LocalTypes";
export default class Fallback implements ILoader {
canImport(location: Location): boolean {
return false;
}
async grabPost(document: Document): Promise<ScrapedPost | null> {
function getSize(el: HTMLImageElement) {
return el.width * el.height;
}
// Get largest img element on this page
const largestImgElement = Array.from(document.getElementsByTagName('img'))
.reduce((a, b) => getSize(a) > getSize(b) ? a : b);
if (largestImgElement) {
let post = new ScrapedPost();
post.source = document.location.href;
post.imageUrl = largestImgElement.src;
return post;
}
return null;
}
}

View File

@ -1,5 +1,5 @@
import ILoader from "./ILoader"
import { LocalPost, LocalTag, LocalCategory } from "../LocalTypes";
import { ScrapedPost, ScrapedTag, BooruCategory } from "../LocalTypes";
// Small hack to avoid screwing with semver
enum Version {
@ -16,13 +16,13 @@ export default class Gelbooru implements ILoader {
);
}
async grabPost(dom: Document): Promise<LocalPost | null> {
async grabPost(dom: Document): Promise<ScrapedPost | null> {
/**
* Make sure to use the `dom` variable and NOT `document`!
*/
let version: Version;
switch (dom.location.host) {
switch (document.location.host) {
case "safebooru.org":
case "rule34.xxx":
version = Version.v020;
@ -37,11 +37,11 @@ export default class Gelbooru implements ILoader {
console.log("Gelbooru guessed version: " + version);
let post = new LocalPost();
post.source = dom.location.href;
let post = new ScrapedPost();
post.source = document.location.href;
// Set image url (direct url to image)
const originalImageElements = dom.querySelectorAll("li > a");
const originalImageElements = document.querySelectorAll("li > a");
for (const idx in originalImageElements) {
const el = originalImageElements[idx] as HTMLAnchorElement;
@ -56,9 +56,9 @@ export default class Gelbooru implements ILoader {
let safetyElements: NodeListOf<Element>;
if (version == Version.v020) {
safetyElements = dom.querySelectorAll("#stats > ul > li");
safetyElements = document.querySelectorAll("#stats > ul > li");
} else {
safetyElements = dom.querySelectorAll("#tag-list > div > li");
safetyElements = document.querySelectorAll("#tag-list > div > li");
}
for (const idx in safetyElements) {
@ -88,9 +88,9 @@ export default class Gelbooru implements ILoader {
let tagElements: NodeListOf<Element>;
if (version == Version.v020) {
tagElements = dom.querySelectorAll("#tag-sidebar > li");
tagElements = document.querySelectorAll("#tag-sidebar > li");
} else {
tagElements = dom.querySelectorAll("#tag-list > div > li[class^='tag-type']");
tagElements = document.querySelectorAll("#tag-list > div > li[class^='tag-type']");
}
for (const idx in tagElements) {
@ -115,7 +115,7 @@ export default class Gelbooru implements ILoader {
tagName = el.getElementsByTagName("a")[1].innerText;
}
let category: LocalCategory | undefined;
let category: BooruCategory | undefined;
switch (el.className) {
case "tag-type-copyright":
@ -132,7 +132,7 @@ export default class Gelbooru implements ILoader {
break;
}
let tag = new LocalTag(tagName, category);
let tag = new ScrapedTag(tagName, category);
post.tags.push(tag);
}
}

View File

@ -1,6 +1,6 @@
import { LocalPost } from "../LocalTypes";
import { ScrapedPost } from "../LocalTypes";
export default interface ILoader {
canImport(location: Location): boolean;
grabPost(dom: Document): Promise<LocalPost | null>;
grabPost(dom: Document): Promise<ScrapedPost | null>;
}

View File

@ -1,5 +1,5 @@
import ILoader from "./ILoader"
import { LocalPost, LocalTag, LocalCategory } from "../LocalTypes";
import { ScrapedPost, ScrapedTag, BooruCategory } from "../LocalTypes";
export default class Moebooru implements ILoader {
canImport(location: Location): boolean {
@ -9,12 +9,12 @@ export default class Moebooru implements ILoader {
);
}
async grabPost(dom: Document): Promise<LocalPost | null> {
let post = new LocalPost();
post.source = dom.location.href;
async grabPost(dom: Document): Promise<ScrapedPost | null> {
let post = new ScrapedPost();
post.source = document.location.href;
// Set image url
const originalImageElement = dom.querySelector("#highres") as HTMLAnchorElement;
const originalImageElement = document.querySelector("#highres") as HTMLAnchorElement;
if (originalImageElement) {
post.imageUrl = originalImageElement.href;
}
@ -22,7 +22,7 @@ export default class Moebooru implements ILoader {
// Set safety
// Same method as Gelbooru but cleaner code
const safetyExp = new RegExp("Rating: (.*)");
const safetyElements = Array.from(dom.querySelectorAll("#stats > ul > li"))
const safetyElements = Array.from(document.querySelectorAll("#stats > ul > li"))
.map(x => x as HTMLLIElement)
.filter(x => x.innerText.startsWith("Rating:"));
@ -44,7 +44,7 @@ export default class Moebooru implements ILoader {
}
// Set tags
const tagElements = Array.from(dom.querySelectorAll("#tag-sidebar > li"))
const tagElements = Array.from(document.querySelectorAll("#tag-sidebar > li"))
.map(x => x as HTMLLIElement);
for (const el of tagElements) {
@ -60,7 +60,7 @@ export default class Moebooru implements ILoader {
continue;
}
let category: LocalCategory | undefined;
let category: BooruCategory | undefined;
// Loop over all classes because some sites (konachan)
// have mutliple classes, e.g. "tag-link tag-type-copyright"
@ -81,7 +81,7 @@ export default class Moebooru implements ILoader {
}
}
let tag = new LocalTag(tagName, category);
let tag = new ScrapedTag(tagName, category);
post.tags.push(tag);
}

View File

@ -1,5 +1,5 @@
import ILoader from "./ILoader"
import { LocalPost, LocalTag } from "../LocalTypes";
import { ScrapedPost, ScrapedTag } from "../LocalTypes";
export default class Reddit implements ILoader {
canImport(location: Location): boolean {
@ -10,19 +10,19 @@ export default class Reddit implements ILoader {
);
}
async grabPost(dom: Document): Promise<LocalPost | null> {
let post = new LocalPost();
post.source = dom.location.href;
async grabPost(dom: Document): Promise<ScrapedPost | null> {
let post = new ScrapedPost();
post.source = document.location.href;
// Set image url
// For old.reddit.com
const oldLinkElements = dom.querySelectorAll("a.title");
const oldLinkElements = document.querySelectorAll("a.title");
if (oldLinkElements.length > 0) {
post.imageUrl = (oldLinkElements[0] as HTMLAnchorElement).href;
}
// For new reddit
const newLinkElements = dom.querySelectorAll("div[data-test-id='post-content'] > div > a");
const newLinkElements = document.querySelectorAll("div[data-test-id='post-content'] > div > a");
if (newLinkElements.length > 0) {
post.imageUrl = (newLinkElements[0] as HTMLAnchorElement).href;
}

View File

@ -1,17 +1,17 @@
import ILoader from "./ILoader"
import { LocalPost, LocalTag } from "../LocalTypes";
import { ScrapedPost, ScrapedTag } from "../LocalTypes";
export default class Shimmie2 implements ILoader {
canImport(location: Location): boolean {
return location.host == "rule34.paheal.net";
}
async grabPost(dom: Document): Promise<LocalPost | null> {
let post = new LocalPost();
post.source = dom.location.href;
async grabPost(dom: Document): Promise<ScrapedPost | null> {
let post = new ScrapedPost();
post.source = document.location.href;
// Set image url
const originalImageElements = Array.from(dom.querySelectorAll(".image_info > tbody > tr > td > a"))
const originalImageElements = Array.from(document.querySelectorAll(".image_info > tbody > tr > td > a"))
.map(x => x as HTMLAnchorElement)
.filter(x => x.innerText == "Image Only");
@ -27,9 +27,9 @@ export default class Shimmie2 implements ILoader {
// They also use uppercase tags, so we toLowerCase() those.
// Tags might not match with other booru-likes, but that is for the user to figure out.
// For example by deleting 'wrong' tags or setting up some aliases in szurubooru.
post.tags = Array.from(dom.querySelectorAll("a.tag_name"))
post.tags = Array.from(document.querySelectorAll("a.tag_name"))
.map(x => (x as HTMLAnchorElement).innerText.toLowerCase())
.map(x => new LocalTag(x));
.map(x => new ScrapedTag(x));
// console.dir(post);

View File

@ -1,5 +1,5 @@
import ILoader from "./ILoader"
import { LocalPost, LocalTag } from "../LocalTypes";
import { ScrapedPost, ScrapedTag } from "../LocalTypes";
function getOriginalImage(imageUrl: string) {
const x = imageUrl.replace(/:large|:med|:small|:orig$/, ""); // Remove `:large`, `:small` etc from url
@ -11,13 +11,13 @@ export default class Twitter implements ILoader {
return location.host == "twitter.com";
}
async grabPost(dom: Document): Promise<LocalPost | null> {
let post = new LocalPost();
post.source = dom.location.href;
async grabPost(dom: Document): Promise<ScrapedPost | null> {
let post = new ScrapedPost();
post.source = document.location.href;
// Set image url
// When clicked on post (not fullscreen, with post text etc)
const overlayElement = dom.querySelector("#permalink-overlay-dialog");
const overlayElement = document.querySelector("#permalink-overlay-dialog");
if (overlayElement) {
const cardImageElement = overlayElement.querySelector("div[data-element-context='platform_photo_card'] > img");
if (cardImageElement) {
@ -26,7 +26,7 @@ export default class Twitter implements ILoader {
}
// When clicked on image (fullscreen)
const mediaImageElements = dom.querySelectorAll("div.Gallery-media > img.media-image");
const mediaImageElements = document.querySelectorAll("div.Gallery-media > img.media-image");
if (mediaImageElements.length > 0) {
post.imageUrl = getOriginalImage((mediaImageElements[0] as HTMLImageElement).src);
}

View File

@ -3,4 +3,5 @@ export { default as Danbooru } from "./Danbooru";
export { default as Shimmie2 } from "./Shimmie2";
export { default as Moebooru } from "./Moebooru";
export { default as Reddit } from "./Reddit";
export { default as Twitter } from "./Twitter"
export { default as Twitter } from "./Twitter";
export { default as Fallback } from "./Fallback";

View File

@ -1,7 +1,7 @@
<template>
<div class="popup-container">
<div class="popup-row">
<div class="popup-column1" style="margin: 10px 10px 0px 10px;">
<div class="popup-column1" style="margin: 0px 10px;">
<ul class="messages">
<li
v-for="msg in messages"
@ -88,9 +88,9 @@
<script lang="ts">
import Vue from "vue";
import { browser } from "webextension-polyfill-ts";
import { LocalPost, LocalTag, LocalError } from "../LocalTypes";
import { ScrapedPost, ScrapedTag } from "../LocalTypes";
import SzuruWrapper from "../SzuruWrapper";
import { Post } from "../SzuruTypes";
import { Post, SzuruError } from "../SzuruTypes";
import { Config, SzuruSiteConfig } from "../Config";
type MessageType = "error" | "info" | "success";
@ -110,7 +110,7 @@ export default Vue.extend({
return {
activeSite: null as SzuruSiteConfig | null,
szuru: null as SzuruWrapper | null,
post: new LocalPost(),
post: new ScrapedPost(),
messages: [] as Message[]
};
},
@ -123,7 +123,7 @@ export default Vue.extend({
}))[0];
// Send 'grab_post' to the content script on the active tab
const post = (await browser.tabs.sendMessage(activeTab.id!, "grab_post")) as LocalPost;
const post = (await browser.tabs.sendMessage(activeTab.id!, "grab_post")) as ScrapedPost;
if (post) {
this.post = post;
@ -181,7 +181,7 @@ export default Vue.extend({
tagsMsg.type = "success";
}
} catch (ex) {
const error = ex as LocalError;
const error = ex as SzuruError;
if (error) {
this.clearMessages();
@ -200,7 +200,7 @@ export default Vue.extend({
const url = browser.extension.getURL("options/options.html");
window.open(url);
},
getTagClasses(tag: LocalTag): string[] {
getTagClasses(tag: ScrapedTag): string[] {
let classes: string[] = [];
if (tag.category) {
@ -211,7 +211,7 @@ export default Vue.extend({
return classes;
},
removeTag(tag: LocalTag) {
removeTag(tag: ScrapedTag) {
if (this.post) {
const idx = this.post.tags.indexOf(tag);
if (idx != -1) {

View File

@ -1,9 +1,9 @@
import { browser, Runtime } from "webextension-polyfill-ts";
import * as loaders from "../loaders";
import { LocalPost } from "../LocalTypes";
import { ScrapedPost } from "../LocalTypes";
import ILoader from "../loaders/ILoader";
async function grabPost(): Promise<LocalPost | null> {
async function grabPost(): Promise<ScrapedPost | null> {
for (const loaderName in loaders) {
// HACK:
const loader = new (loaders as any)[loaderName]() as ILoader;
@ -11,6 +11,12 @@ async function grabPost(): Promise<LocalPost | null> {
console.log(`Using ${loaderName} importer to scrape this page`)
return await loader.grabPost(document);
}
// Try using the Fallback loader if no other loader can scrape this page
const post = new loaders.Fallback().grabPost(document);
if (post) {
return post;
}
}
return null;

View File

@ -16,10 +16,10 @@
"plugins": [
{
"transform": "ts-optchain/transform"
},
}
]
},
"include": [
"./src/**/*"
],
]
}