diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c006211 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 neobooru + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/package-lock.json b/package-lock.json index 2075fe3..1f72e04 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "szuru-chrome", - "version": "0.1.1", + "version": "0.1.2", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -4035,6 +4035,11 @@ "integrity": "sha512-iyam8fBuCUpWeKPGpaNMetEocMt364qkCsfL9JuhjXX6dRnguRVOfk2GZaDpPjcOKiiXCPINZC1GczQ7iTq3Zw==", "dev": true }, + "neo-scraper": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/neo-scraper/-/neo-scraper-0.1.0.tgz", + "integrity": "sha512-XfSFRKwB9vVldWGPecOxWeMfBfNkhOklPbb4qYknnsyW7gkbudY5L3VkqM8DFnsvXyXEGW2rFTsc1JTfigypKg==" + }, "nice-try": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz", diff --git a/package.json b/package.json index 30ee7ca..f5088c4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "szuru-chrome", - "version": "0.1.1", + "version": "0.1.2", "description": "Import media from various sites into szurubooru", "author": "neobooru", "license": "MIT", @@ -13,6 +13,7 @@ }, "dependencies": { "axios": "^0.19.0", + "neo-scraper": "^0.1.0", "ts-optchain": "^0.1.8", "vue": "^2.6.10", "webextension-polyfill": "^0.3.1" diff --git a/src/LocalTypes.ts b/src/LocalTypes.ts deleted file mode 100644 index 578f057..0000000 --- a/src/LocalTypes.ts +++ /dev/null @@ -1,25 +0,0 @@ -/** - * These types are NOT compatible with szurubooru, they are only used in the WebExtension. - */ - -import { Safety } from "./SzuruTypes"; - -export type BooruCategory = "copyright" | "character" | "artist" | "meta"; - -export class ScrapedTag { - name: string = ""; - category?: string; - - constructor(name: string, category?: string) { - this.name = name.replace(/\s/g, "_"); // Replace all spaces with underscores - this.category = category; - } -} - -// FIXME: LocalPost -> valid szurubooru type -export class ScrapedPost { - safety: Safety = "safe"; - source: string = ""; - imageUrl: string = ""; - tags: ScrapedTag[] = []; -} diff --git a/src/SzuruWrapper.ts b/src/SzuruWrapper.ts index 039019c..b3bf663 100644 --- a/src/SzuruWrapper.ts +++ b/src/SzuruWrapper.ts @@ -1,7 +1,7 @@ -import axios, { AxiosRequestConfig } from 'axios'; -import { ScrapedPost } from './LocalTypes'; -import { SzuruSiteConfig } from './Config'; -import { TagsResult, TagCategoriesResult, Post, Tag, SzuruError } from './SzuruTypes'; +import axios, { AxiosRequestConfig } from "axios"; +import { ScrapedPost } from "neo-scraper"; +import { SzuruSiteConfig } from "./Config"; +import { TagsResult, TagCategoriesResult, Post, Tag, SzuruError } from "./SzuruTypes"; /** * A 1:1 wrapper around the szurubooru API. @@ -55,7 +55,7 @@ export default class SzuruWrapper { async createPost(post: ScrapedPost): Promise { var obj = { tags: post.tags.map(x => x.name), - safety: post.safety, + safety: post.rating, source: post.source, contentUrl: post.imageUrl }; diff --git a/src/loaders/Danbooru.ts b/src/loaders/Danbooru.ts deleted file mode 100644 index bde2cc3..0000000 --- a/src/loaders/Danbooru.ts +++ /dev/null @@ -1,88 +0,0 @@ -import ILoader from "./ILoader" -import { ScrapedPost, ScrapedTag, BooruCategory } from "../LocalTypes"; - -export default class Danbooru implements ILoader { - canImport(location: Location): boolean { - return ( - location.host == "danbooru.donmai.us" || - location.host == "safebooru.donmai.us" - ); - } - - async grabPost(dom: Document): Promise { - let post = new ScrapedPost(); - post.source = document.location.href; - - // Set image url - const originalImageElements = Array.from(document.querySelectorAll("#post-information > ul > li > a")) - .map(x => x as HTMLAnchorElement) - .filter(x => x.parentElement && x.parentElement.innerText.startsWith("Size")); - - if (originalImageElements.length > 0) { - post.imageUrl = originalImageElements[0].href; - } - - // Set safety - const safetyExp = new RegExp("Rating: (.*)"); - const safetyElements = Array.from(document.querySelectorAll("#post-information > ul > li")) - .map(x => x as HTMLLIElement) - .filter(x => x.innerText.startsWith("Rating")); - - if (safetyElements.length > 0) { - const matches = safetyElements[0].innerText.match(safetyExp); - if (matches && matches.length > 0) { - switch (matches[1].toLowerCase()) { - case "safe": - post.safety = "safe"; - break; - case "questionable": - post.safety = "sketchy"; - break; - case "explicit": - post.safety = "unsafe"; - break; - } - } - } - - // Set tags - const tagElements = Array.from(document.querySelectorAll("#tag-list > ul > li")).map(x => x as HTMLLIElement); - for (const tagElement of tagElements) { - let tagName: string | undefined; - const tagNameElement = tagElement.querySelector(".search-tag") as HTMLAnchorElement; - if (tagNameElement) { - tagName = tagNameElement.innerText; - } - - let category: BooruCategory | undefined; - switch (tagElement.className) { - case "category-3": - category = "copyright"; - break; - case "category-4": - category = "character"; - break; - case "category-1": - category = "artist"; - break; - case "category-5": - category = "meta"; - break; - } - - if (tagName) { - let tag = new ScrapedTag(tagName, category); - post.tags.push(tag); - } - } - - // console.dir(post); - - // If post has enough filled fields to be considered a valid post - if (post.imageUrl) { - return post; - } else { - return null; - } - } -} diff --git a/src/loaders/Fallback.ts b/src/loaders/Fallback.ts deleted file mode 100644 index f586140..0000000 --- a/src/loaders/Fallback.ts +++ /dev/null @@ -1,27 +0,0 @@ -import ILoader from "./ILoader"; -import { ScrapedPost } from "../LocalTypes"; - -export default class Fallback implements ILoader { - canImport(location: Location): boolean { - return false; - } - - async grabPost(document: Document): Promise { - function getSize(el: HTMLImageElement) { - return el.width * el.height; - } - - // Get largest img element on this page - const largestImgElement = Array.from(document.getElementsByTagName('img')) - .reduce((a, b) => getSize(a) > getSize(b) ? a : b); - - if (largestImgElement) { - let post = new ScrapedPost(); - post.source = document.location.href; - post.imageUrl = largestImgElement.src; - return post; - } - - return null; - } -} diff --git a/src/loaders/Gelbooru.ts b/src/loaders/Gelbooru.ts deleted file mode 100644 index b14bd01..0000000 --- a/src/loaders/Gelbooru.ts +++ /dev/null @@ -1,149 +0,0 @@ -import ILoader from "./ILoader" -import { ScrapedPost, ScrapedTag, BooruCategory } from "../LocalTypes"; - -// Small hack to avoid screwing with semver -enum Version { - v020 = 1, // 0.2.0 - v025, // 0.2.5 -} - -export default class Gelbooru implements ILoader { - canImport(location: Location): boolean { - return ( - location.host == "safebooru.org" || - location.host == "gelbooru.com" || - location.host == "rule34.xxx" - ); - } - - async grabPost(dom: Document): Promise { - /** - * Make sure to use the `dom` variable and NOT `document`! - */ - - let version: Version; - switch (document.location.host) { - case "safebooru.org": - case "rule34.xxx": - version = Version.v020; - break; - case "gelbooru.com": - version = Version.v025; - break; - default: - console.error("Couldn't guess version"); - return null; - } - - console.log("Gelbooru guessed version: " + version); - - let post = new ScrapedPost(); - post.source = document.location.href; - - // Set image url (direct url to image) - const originalImageElements = document.querySelectorAll("li > a"); - for (const idx in originalImageElements) { - const el = originalImageElements[idx] as HTMLAnchorElement; - - if (el && el.innerText == "Original image") { - post.imageUrl = el.href; - break; - } - } - - // Set safety - const safetyExp = new RegExp("Rating: (.*)"); - let safetyElements: NodeListOf; - - if (version == Version.v020) { - safetyElements = document.querySelectorAll("#stats > ul > li"); - } else { - safetyElements = document.querySelectorAll("#tag-list > div > li"); - } - - for (const idx in safetyElements) { - const el = safetyElements[idx] as HTMLLIElement; - if (el && el.innerText) { - // matches is an array where (index) - // 0: full string - // 1: capture group 1 (rating, so 'Safe', 'Questionable', 'Explicit') - const matches = el.innerText.match(safetyExp); - if (matches && matches.length > 0) { - switch (matches[1].toLowerCase()) { - case "safe": - post.safety = "safe"; - break; - case "questionable": - post.safety = "sketchy"; - break; - case "explicit": - post.safety = "unsafe"; - break; - } - } - } - } - - // Set tags - let tagElements: NodeListOf; - - if (version == Version.v020) { - tagElements = document.querySelectorAll("#tag-sidebar > li"); - } else { - tagElements = document.querySelectorAll("#tag-list > div > li[class^='tag-type']"); - } - - for (const idx in tagElements) { - const el = tagElements[idx] as HTMLLIElement; - if (el && el.innerText) { - let tagName: string; - - if (version == Version.v020) { - const nameElements = el.getElementsByTagName("a"); - - // Some
  • don't actually contain a name. - // E.g. the sub-headers ("Copyright", "Character", "General", "Meta") on rule34.xxx - if (nameElements.length > 0) { - tagName = el.getElementsByTagName("a")[0].innerText; - } - else { - continue; - } - } else { - // First is "?" - // Second is "actual_tag_name" - tagName = el.getElementsByTagName("a")[1].innerText; - } - - let category: BooruCategory | undefined; - - switch (el.className) { - case "tag-type-copyright": - category = "copyright"; - break; - case "tag-type-character": - category = "character"; - break; - case "tag-type-artist": - category = "artist"; - break; - case "tag-type-metadata": - category = "meta"; - break; - } - - let tag = new ScrapedTag(tagName, category); - post.tags.push(tag); - } - } - - // console.dir(post); - - // If post has enough filled fields to be considered a valid post - if (post.imageUrl) { - return post; - } else { - return null; - } - } -}; diff --git a/src/loaders/ILoader.ts b/src/loaders/ILoader.ts deleted file mode 100644 index 83452f7..0000000 --- a/src/loaders/ILoader.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { ScrapedPost } from "../LocalTypes"; - -export default interface ILoader { - canImport(location: Location): boolean; - grabPost(dom: Document): Promise; -} diff --git a/src/loaders/Moebooru.ts b/src/loaders/Moebooru.ts deleted file mode 100644 index 1a5ca35..0000000 --- a/src/loaders/Moebooru.ts +++ /dev/null @@ -1,97 +0,0 @@ -import ILoader from "./ILoader" -import { ScrapedPost, ScrapedTag, BooruCategory } from "../LocalTypes"; - -export default class Moebooru implements ILoader { - canImport(location: Location): boolean { - return ( - location.host == "yande.re" || - location.host == "konachan.com" - ); - } - - async grabPost(dom: Document): Promise { - let post = new ScrapedPost(); - post.source = document.location.href; - - // Set image url - const originalImageElement = document.querySelector("#highres") as HTMLAnchorElement; - if (originalImageElement) { - post.imageUrl = originalImageElement.href; - } - - // Set safety - // Same method as Gelbooru but cleaner code - const safetyExp = new RegExp("Rating: (.*)"); - const safetyElements = Array.from(document.querySelectorAll("#stats > ul > li")) - .map(x => x as HTMLLIElement) - .filter(x => x.innerText.startsWith("Rating:")); - - if (safetyElements.length > 0) { - const matches = safetyElements[0].innerText.match(safetyExp); - if (matches && matches.length > 0) { - switch (matches[1].toLowerCase()) { - case "safe": - post.safety = "safe"; - break; - case "questionable": - post.safety = "sketchy"; - break; - case "explicit": - post.safety = "unsafe"; - break; - } - } - } - - // Set tags - const tagElements = Array.from(document.querySelectorAll("#tag-sidebar > li")) - .map(x => x as HTMLLIElement); - - for (const el of tagElements) { - let tagName: string; - let tagNameElements = el.getElementsByTagName("a"); - - if (tagNameElements.length > 1) { - // First is "?" - // Second is "actual_tag_name" - tagName = tagNameElements[1].innerText; - } else { - // Skip if no tag name is found - continue; - } - - let category: BooruCategory | undefined; - - // Loop over all classes because some sites (konachan) - // have mutliple classes, e.g. "tag-link tag-type-copyright" - for (const className of Array.from(el.classList)) { - switch (className) { - case "tag-type-copyright": - category = "copyright"; - break; - case "tag-type-character": - category = "character"; - break; - case "tag-type-artist": - category = "artist"; - break; - case "tag-type-metadata": - category = "meta"; - break; - } - } - - let tag = new ScrapedTag(tagName, category); - post.tags.push(tag); - } - - // console.dir(post); - - // If post has enough filled fields to be considered a valid post - if (post.imageUrl) { - return post; - } else { - return null; - } - } -} diff --git a/src/loaders/Reddit.ts b/src/loaders/Reddit.ts deleted file mode 100644 index 7222856..0000000 --- a/src/loaders/Reddit.ts +++ /dev/null @@ -1,37 +0,0 @@ -import ILoader from "./ILoader" -import { ScrapedPost, ScrapedTag } from "../LocalTypes"; - -export default class Reddit implements ILoader { - canImport(location: Location): boolean { - return ( - location.host == "reddit.com" || - location.host == "www.reddit.com" || - location.host == "old.reddit.com" - ); - } - - async grabPost(dom: Document): Promise { - let post = new ScrapedPost(); - post.source = document.location.href; - - // Set image url - // For old.reddit.com - const oldLinkElements = document.querySelectorAll("a.title"); - if (oldLinkElements.length > 0) { - post.imageUrl = (oldLinkElements[0] as HTMLAnchorElement).href; - } - - // For new reddit - const newLinkElements = document.querySelectorAll("div[data-test-id='post-content'] > div > a"); - if (newLinkElements.length > 0) { - post.imageUrl = (newLinkElements[0] as HTMLAnchorElement).href; - } - - // If post has enough filled fields to be considered a valid post - if (post.imageUrl) { - return post; - } else { - return null; - } - } -} diff --git a/src/loaders/Shimmie2.ts b/src/loaders/Shimmie2.ts deleted file mode 100644 index e105324..0000000 --- a/src/loaders/Shimmie2.ts +++ /dev/null @@ -1,43 +0,0 @@ -import ILoader from "./ILoader" -import { ScrapedPost, ScrapedTag } from "../LocalTypes"; - -export default class Shimmie2 implements ILoader { - canImport(location: Location): boolean { - return location.host == "rule34.paheal.net"; - } - - async grabPost(dom: Document): Promise { - let post = new ScrapedPost(); - post.source = document.location.href; - - // Set image url - const originalImageElements = Array.from(document.querySelectorAll(".image_info > tbody > tr > td > a")) - .map(x => x as HTMLAnchorElement) - .filter(x => x.innerText == "Image Only"); - - if (originalImageElements.length > 0) { - post.imageUrl = originalImageElements[0].href; - } - - // Set safety - post.safety = "unsafe"; // Usually it is unsafe - - // Set tags - // Shimmie2 doesn't support categories so we don't need to worry about those. - // They also use uppercase tags, so we toLowerCase() those. - // Tags might not match with other booru-likes, but that is for the user to figure out. - // For example by deleting 'wrong' tags or setting up some aliases in szurubooru. - post.tags = Array.from(document.querySelectorAll("a.tag_name")) - .map(x => (x as HTMLAnchorElement).innerText.toLowerCase()) - .map(x => new ScrapedTag(x)); - - // console.dir(post); - - // If post has enough filled fields to be considered a valid post - if (post.imageUrl) { - return post; - } else { - return null; - } - } -} diff --git a/src/loaders/Twitter.ts b/src/loaders/Twitter.ts deleted file mode 100644 index 22e34c7..0000000 --- a/src/loaders/Twitter.ts +++ /dev/null @@ -1,41 +0,0 @@ -import ILoader from "./ILoader" -import { ScrapedPost, ScrapedTag } from "../LocalTypes"; - -function getOriginalImage(imageUrl: string) { - const x = imageUrl.replace(/:large|:med|:small|:orig$/, ""); // Remove `:large`, `:small` etc from url - return x + ":orig"; // Add :orig to url (to get original image) -} - -export default class Twitter implements ILoader { - canImport(location: Location): boolean { - return location.host == "twitter.com"; - } - - async grabPost(dom: Document): Promise { - let post = new ScrapedPost(); - post.source = document.location.href; - - // Set image url - // When clicked on post (not fullscreen, with post text etc) - const overlayElement = document.querySelector("#permalink-overlay-dialog"); - if (overlayElement) { - const cardImageElement = overlayElement.querySelector("div[data-element-context='platform_photo_card'] > img"); - if (cardImageElement) { - post.imageUrl = getOriginalImage((cardImageElement as HTMLImageElement).src); - } - } - - // When clicked on image (fullscreen) - const mediaImageElements = document.querySelectorAll("div.Gallery-media > img.media-image"); - if (mediaImageElements.length > 0) { - post.imageUrl = getOriginalImage((mediaImageElements[0] as HTMLImageElement).src); - } - - // If post has enough filled fields to be considered a valid post - if (post.imageUrl) { - return post; - } else { - return null; - } - } -} diff --git a/src/loaders/index.ts b/src/loaders/index.ts deleted file mode 100644 index cc926bb..0000000 --- a/src/loaders/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -export { default as Gelbooru } from "./Gelbooru"; -export { default as Danbooru } from "./Danbooru"; -export { default as Shimmie2 } from "./Shimmie2"; -export { default as Moebooru } from "./Moebooru"; -export { default as Reddit } from "./Reddit"; -export { default as Twitter } from "./Twitter"; -export { default as Fallback } from "./Fallback"; diff --git a/src/popup/App.vue b/src/popup/App.vue index 3dc1793..e15c431 100644 --- a/src/popup/App.vue +++ b/src/popup/App.vue @@ -25,19 +25,19 @@ Safety @@ -88,7 +88,7 @@