Add basic note import support for Danbooru and Gelbooru
There UI currently doesn't display which notes will be imported, nor is there an option to not import notes. This will be added in a future version.next
parent
120ffb123e
commit
349894d4f9
|
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
"name": "szuru-chrome",
|
"name": "szuru-chrome",
|
||||||
"version": "0.7.6",
|
"version": "0.7.7",
|
||||||
"description": "Import media from various sites into szurubooru.",
|
"description": "Import media from various sites into szurubooru.",
|
||||||
"author": "neobooru",
|
"author": "neobooru",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
|
|
@ -16,7 +16,7 @@
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"axios": "^0.23.0",
|
"axios": "^0.23.0",
|
||||||
"neo-scraper": "^0.4.7",
|
"neo-scraper": "^0.5.0",
|
||||||
"vue": "^2.6.14",
|
"vue": "^2.6.14",
|
||||||
"webextension-polyfill": "^0.8.0"
|
"webextension-polyfill": "^0.8.0"
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -97,6 +97,11 @@ export interface Post {
|
||||||
comments: any[]; // Comment resource
|
comments: any[]; // Comment resource
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface SzuruNote {
|
||||||
|
polygon: number[][];
|
||||||
|
text: string;
|
||||||
|
}
|
||||||
|
|
||||||
export interface SimilarPost {
|
export interface SimilarPost {
|
||||||
distance: number;
|
distance: number;
|
||||||
post: Post;
|
post: Post;
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,16 @@
|
||||||
import axios, { AxiosRequestConfig, CancelToken } from "axios";
|
import axios, { AxiosRequestConfig, CancelToken } from "axios";
|
||||||
import { ScrapedPost } from "neo-scraper";
|
|
||||||
import { SzuruSiteConfig } from "./Config";
|
import { SzuruSiteConfig } from "./Config";
|
||||||
import { TagsResult, TagCategoriesResult, Post, Tag, SzuruError, ImageSearchResult, TagFields } from "./SzuruTypes";
|
import {
|
||||||
|
TagsResult,
|
||||||
|
TagCategoriesResult,
|
||||||
|
Post,
|
||||||
|
Tag,
|
||||||
|
SzuruError,
|
||||||
|
ImageSearchResult,
|
||||||
|
TagFields,
|
||||||
|
SzuruNote
|
||||||
|
} from "./SzuruTypes";
|
||||||
|
import { PostViewModel } from "./ViewModels";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A 1:1 wrapper around the szurubooru API.
|
* A 1:1 wrapper around the szurubooru API.
|
||||||
|
|
@ -61,11 +70,12 @@ export default class SzuruWrapper {
|
||||||
return (await this.apiGet("tag-categories")).data;
|
return (await this.apiGet("tag-categories")).data;
|
||||||
}
|
}
|
||||||
|
|
||||||
async createPost(post: ScrapedPost): Promise<Post> {
|
async createPost(post: PostViewModel): Promise<Post> {
|
||||||
var obj = {
|
var obj = {
|
||||||
tags: post.tags.map(x => x.name),
|
tags: post.tags.map(x => x.name),
|
||||||
safety: post.rating,
|
safety: post.rating,
|
||||||
source: post.source,
|
source: post.source,
|
||||||
|
notes: post.notes.map(note => <SzuruNote>{ polygon: note.polygons, text: note.text }),
|
||||||
contentUrl: post.contentUrl
|
contentUrl: post.contentUrl
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -83,6 +93,10 @@ export default class SzuruWrapper {
|
||||||
return (await this.apiPost("posts/reverse-search", obj)).data;
|
return (await this.apiPost("posts/reverse-search", obj)).data;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static async createFromConfig(siteConfig: SzuruSiteConfig): Promise<SzuruWrapper | null> {
|
||||||
|
return new SzuruWrapper(siteConfig.domain, siteConfig.username, siteConfig.authToken);
|
||||||
|
}
|
||||||
|
|
||||||
private async apiGet(
|
private async apiGet(
|
||||||
url: string,
|
url: string,
|
||||||
additionalHeaders: any = {},
|
additionalHeaders: any = {},
|
||||||
|
|
@ -137,8 +151,4 @@ export default class SzuruWrapper {
|
||||||
throw error ? error : ex;
|
throw error ? error : ex;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static async createFromConfig(siteConfig: SzuruSiteConfig): Promise<SzuruWrapper | null> {
|
|
||||||
return new SzuruWrapper(siteConfig.domain, siteConfig.username, siteConfig.authToken);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,49 @@
|
||||||
|
import { BooruTypes, ContentType, ScrapedNote, ScrapedPost, ScrapedTag } from "neo-scraper";
|
||||||
|
import { MicroTag, Tag } from "./SzuruTypes";
|
||||||
|
|
||||||
|
export class TagViewModel {
|
||||||
|
public implications: TagViewModel[] = [];
|
||||||
|
|
||||||
|
constructor(public name: string, public category?: BooruTypes.TagCategory, public usages?: number | undefined) {}
|
||||||
|
|
||||||
|
static fromTag(tag: Tag) {
|
||||||
|
let x = new TagViewModel(tag.names[0], <BooruTypes.TagCategory>tag.category, tag.usages);
|
||||||
|
|
||||||
|
if (tag.implications) {
|
||||||
|
x.implications = tag.implications.map(y => TagViewModel.fromMicroTag(y));
|
||||||
|
}
|
||||||
|
|
||||||
|
return x;
|
||||||
|
}
|
||||||
|
|
||||||
|
static fromMicroTag(tag: MicroTag) {
|
||||||
|
return new TagViewModel(tag.names[0], <BooruTypes.TagCategory>tag.category, tag.usages);
|
||||||
|
}
|
||||||
|
|
||||||
|
static fromScapedTag(tag: ScrapedTag): TagViewModel {
|
||||||
|
return new TagViewModel(tag.name, tag.category);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class PostViewModel {
|
||||||
|
name: string = "";
|
||||||
|
tags: TagViewModel[] = [];
|
||||||
|
notes: ScrapedNote[];
|
||||||
|
contentUrl: string;
|
||||||
|
pageUrl: string;
|
||||||
|
contentType: ContentType;
|
||||||
|
rating: BooruTypes.SafetyRating;
|
||||||
|
source: string | undefined;
|
||||||
|
referrer: string | undefined;
|
||||||
|
|
||||||
|
constructor(post: ScrapedPost) {
|
||||||
|
this.contentUrl = post.contentUrl;
|
||||||
|
this.pageUrl = post.pageUrl;
|
||||||
|
this.contentType = post.contentType;
|
||||||
|
this.rating = post.rating;
|
||||||
|
this.source = post.source;
|
||||||
|
this.referrer = post.referrer;
|
||||||
|
this.tags = post.tags.map(x => TagViewModel.fromScapedTag(x));
|
||||||
|
this.notes = post.notes;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -127,58 +127,14 @@
|
||||||
|
|
||||||
<script lang="ts">
|
<script lang="ts">
|
||||||
import Vue from "vue";
|
import Vue from "vue";
|
||||||
|
import axios, { CancelTokenSource } from "axios";
|
||||||
import { browser, Runtime, WebRequest } from "webextension-polyfill-ts";
|
import { browser, Runtime, WebRequest } from "webextension-polyfill-ts";
|
||||||
import { ContentType, ScrapedPost, ScrapedTag, ScrapeResults, BooruTypes } from "neo-scraper";
|
import { ScrapedPost, ScrapeResults } from "neo-scraper";
|
||||||
import SzuruWrapper from "../SzuruWrapper";
|
import SzuruWrapper from "../SzuruWrapper";
|
||||||
import { MicroTag, Post, Tag } from "../SzuruTypes";
|
import { Post, } from "../SzuruTypes";
|
||||||
import { Config, SzuruSiteConfig } from "../Config";
|
import { Config, SzuruSiteConfig } from "../Config";
|
||||||
import { BrowserCommand, Message, getUrl, isChrome, encodeTagName } from "../Common";
|
import { BrowserCommand, Message, getUrl, isChrome, encodeTagName } from "../Common";
|
||||||
import axios, { CancelTokenSource } from "axios";
|
import { PostViewModel, TagViewModel } from "../ViewModels";
|
||||||
|
|
||||||
class TagViewModel {
|
|
||||||
public implications: TagViewModel[] = [];
|
|
||||||
|
|
||||||
constructor(public name: string, public category?: BooruTypes.TagCategory, public usages?: number | undefined) { }
|
|
||||||
|
|
||||||
static fromTag(tag: Tag) {
|
|
||||||
let x = new TagViewModel(tag.names[0], <BooruTypes.TagCategory>tag.category, tag.usages);
|
|
||||||
|
|
||||||
if (tag.implications) {
|
|
||||||
x.implications = tag.implications.map((y) => TagViewModel.fromMicroTag(y));
|
|
||||||
}
|
|
||||||
|
|
||||||
return x;
|
|
||||||
}
|
|
||||||
|
|
||||||
static fromMicroTag(tag: MicroTag) {
|
|
||||||
return new TagViewModel(tag.names[0], <BooruTypes.TagCategory>tag.category, tag.usages);
|
|
||||||
}
|
|
||||||
|
|
||||||
static fromScapedTag(tag: ScrapedTag): TagViewModel {
|
|
||||||
return new TagViewModel(tag.name, tag.category);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
class PostViewModel {
|
|
||||||
name: string = "";
|
|
||||||
tags: TagViewModel[] = [];
|
|
||||||
contentUrl: string;
|
|
||||||
pageUrl: string;
|
|
||||||
contentType: ContentType;
|
|
||||||
rating: BooruTypes.SafetyRating;
|
|
||||||
source: string | undefined;
|
|
||||||
referrer: string | undefined;
|
|
||||||
|
|
||||||
constructor(post: ScrapedPost) {
|
|
||||||
this.contentUrl = post.contentUrl;
|
|
||||||
this.pageUrl = post.pageUrl;
|
|
||||||
this.contentType = post.contentType;
|
|
||||||
this.rating = post.rating;
|
|
||||||
this.source = post.source;
|
|
||||||
this.referrer = post.referrer;
|
|
||||||
this.tags = post.tags.map((x) => TagViewModel.fromScapedTag(x));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export default Vue.extend({
|
export default Vue.extend({
|
||||||
data() {
|
data() {
|
||||||
|
|
@ -391,7 +347,6 @@ export default Vue.extend({
|
||||||
},
|
},
|
||||||
async loadTagCounts() {
|
async loadTagCounts() {
|
||||||
const allTags = this.posts.flatMap((x) => x.tags);
|
const allTags = this.posts.flatMap((x) => x.tags);
|
||||||
console.log(allTags);
|
|
||||||
|
|
||||||
for (let i = 0; i < allTags.length; i += 100) {
|
for (let i = 0; i < allTags.length; i += 100) {
|
||||||
const query = allTags
|
const query = allTags
|
||||||
|
|
|
||||||
|
|
@ -1,11 +1,11 @@
|
||||||
import { ScrapedPost } from "neo-scraper";
|
|
||||||
import { browser, Runtime } from "webextension-polyfill-ts";
|
import { browser, Runtime } from "webextension-polyfill-ts";
|
||||||
import { BrowserCommand, Message, encodeTagName, getUrl } from "../Common";
|
import { BrowserCommand, Message, encodeTagName, getUrl } from "../Common";
|
||||||
import { Config } from "../Config";
|
import { Config } from "../Config";
|
||||||
import SzuruWrapper from "../SzuruWrapper";
|
import SzuruWrapper from "../SzuruWrapper";
|
||||||
import { PostAlreadyUploadedError, SzuruError } from "../SzuruTypes";
|
import { PostAlreadyUploadedError, SzuruError } from "../SzuruTypes";
|
||||||
|
import { PostViewModel } from "../ViewModels";
|
||||||
|
|
||||||
async function uploadPost(post: ScrapedPost) {
|
async function uploadPost(post: PostViewModel) {
|
||||||
console.dir(post);
|
console.dir(post);
|
||||||
|
|
||||||
const cfg = await Config.load();
|
const cfg = await Config.load();
|
||||||
|
|
@ -82,12 +82,12 @@ async function uploadPost(post: ScrapedPost) {
|
||||||
browser.runtime.sendMessage(new BrowserCommand("push_message", new Message(msg, "error")));
|
browser.runtime.sendMessage(new BrowserCommand("push_message", new Message(msg, "error")));
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
browser.runtime.sendMessage(new BrowserCommand("push_message", new Message(ex, "error")));
|
browser.runtime.sendMessage(new BrowserCommand("push_message", new Message(ex.message, "error")));
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
console.error(ex);
|
console.error(ex);
|
||||||
browser.runtime.sendMessage(new BrowserCommand("push_message", new Message(ex, "error")));
|
browser.runtime.sendMessage(new BrowserCommand("push_message", new Message(ex.message, "error")));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
22
yarn.lock
22
yarn.lock
|
|
@ -688,6 +688,11 @@ dir-glob@^3.0.1:
|
||||||
dependencies:
|
dependencies:
|
||||||
path-type "^4.0.0"
|
path-type "^4.0.0"
|
||||||
|
|
||||||
|
domino@^2.1.6:
|
||||||
|
version "2.1.6"
|
||||||
|
resolved "https://registry.yarnpkg.com/domino/-/domino-2.1.6.tgz#fe4ace4310526e5e7b9d12c7de01b7f485a57ffe"
|
||||||
|
integrity sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ==
|
||||||
|
|
||||||
ecc-jsbn@~0.1.1:
|
ecc-jsbn@~0.1.1:
|
||||||
version "0.1.2"
|
version "0.1.2"
|
||||||
resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9"
|
resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9"
|
||||||
|
|
@ -1417,10 +1422,12 @@ neo-async@^2.6.2:
|
||||||
resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f"
|
resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f"
|
||||||
integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==
|
integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==
|
||||||
|
|
||||||
neo-scraper@^0.4.7:
|
neo-scraper@^0.5.0:
|
||||||
version "0.4.7"
|
version "0.5.0"
|
||||||
resolved "https://registry.yarnpkg.com/neo-scraper/-/neo-scraper-0.4.7.tgz#bee74ce7a80e794b7d1dd3e3eed52d91931dbfcf"
|
resolved "https://registry.yarnpkg.com/neo-scraper/-/neo-scraper-0.5.0.tgz#8af59253a1dead386a906dc7281c024ae7938e25"
|
||||||
integrity sha512-Tmv+3xrFTk1c2/vnY2qLmft3KT0IXIOglsSC7ai3+lgmC+1LCVjSqlnWBlIPqTIHCOFnIFhb06H00znN/uAptw==
|
integrity sha512-SUh9kQC/uI2rTSity+cG7kTwDdlQ9XjEEdqw7hkh0AIjB2yi2X9jHe8mVq0VFC/VLaoBU8QlWUstobmfspipGA==
|
||||||
|
dependencies:
|
||||||
|
turndown "^7.1.1"
|
||||||
|
|
||||||
node-releases@^2.0.0:
|
node-releases@^2.0.0:
|
||||||
version "2.0.0"
|
version "2.0.0"
|
||||||
|
|
@ -2018,6 +2025,13 @@ tunnel-agent@^0.6.0:
|
||||||
dependencies:
|
dependencies:
|
||||||
safe-buffer "^5.0.1"
|
safe-buffer "^5.0.1"
|
||||||
|
|
||||||
|
turndown@^7.1.1:
|
||||||
|
version "7.1.1"
|
||||||
|
resolved "https://registry.yarnpkg.com/turndown/-/turndown-7.1.1.tgz#96992f2d9b40a1a03d3ea61ad31b5a5c751ef77f"
|
||||||
|
integrity sha512-BEkXaWH7Wh7e9bd2QumhfAXk5g34+6QUmmWx+0q6ThaVOLuLUqsnkq35HQ5SBHSaxjSfSM7US5o4lhJNH7B9MA==
|
||||||
|
dependencies:
|
||||||
|
domino "^2.1.6"
|
||||||
|
|
||||||
tweetnacl@^0.14.3, tweetnacl@~0.14.0:
|
tweetnacl@^0.14.3, tweetnacl@~0.14.0:
|
||||||
version "0.14.5"
|
version "0.14.5"
|
||||||
resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64"
|
resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64"
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue