Add basic note import support for Danbooru and Gelbooru

There UI currently doesn't display which notes will be imported, nor is there an option to not import notes. This will be added in a future version.
next
neobooru 2021-11-20 01:39:18 +01:00
parent 120ffb123e
commit 349894d4f9
7 changed files with 99 additions and 66 deletions

View File

@ -1,6 +1,6 @@
{ {
"name": "szuru-chrome", "name": "szuru-chrome",
"version": "0.7.6", "version": "0.7.7",
"description": "Import media from various sites into szurubooru.", "description": "Import media from various sites into szurubooru.",
"author": "neobooru", "author": "neobooru",
"license": "MIT", "license": "MIT",
@ -16,7 +16,7 @@
}, },
"dependencies": { "dependencies": {
"axios": "^0.23.0", "axios": "^0.23.0",
"neo-scraper": "^0.4.7", "neo-scraper": "^0.5.0",
"vue": "^2.6.14", "vue": "^2.6.14",
"webextension-polyfill": "^0.8.0" "webextension-polyfill": "^0.8.0"
}, },

View File

@ -97,6 +97,11 @@ export interface Post {
comments: any[]; // Comment resource comments: any[]; // Comment resource
} }
export interface SzuruNote {
polygon: number[][];
text: string;
}
export interface SimilarPost { export interface SimilarPost {
distance: number; distance: number;
post: Post; post: Post;

View File

@ -1,7 +1,16 @@
import axios, { AxiosRequestConfig, CancelToken } from "axios"; import axios, { AxiosRequestConfig, CancelToken } from "axios";
import { ScrapedPost } from "neo-scraper";
import { SzuruSiteConfig } from "./Config"; import { SzuruSiteConfig } from "./Config";
import { TagsResult, TagCategoriesResult, Post, Tag, SzuruError, ImageSearchResult, TagFields } from "./SzuruTypes"; import {
TagsResult,
TagCategoriesResult,
Post,
Tag,
SzuruError,
ImageSearchResult,
TagFields,
SzuruNote
} from "./SzuruTypes";
import { PostViewModel } from "./ViewModels";
/** /**
* A 1:1 wrapper around the szurubooru API. * A 1:1 wrapper around the szurubooru API.
@ -61,11 +70,12 @@ export default class SzuruWrapper {
return (await this.apiGet("tag-categories")).data; return (await this.apiGet("tag-categories")).data;
} }
async createPost(post: ScrapedPost): Promise<Post> { async createPost(post: PostViewModel): Promise<Post> {
var obj = { var obj = {
tags: post.tags.map(x => x.name), tags: post.tags.map(x => x.name),
safety: post.rating, safety: post.rating,
source: post.source, source: post.source,
notes: post.notes.map(note => <SzuruNote>{ polygon: note.polygons, text: note.text }),
contentUrl: post.contentUrl contentUrl: post.contentUrl
}; };
@ -83,6 +93,10 @@ export default class SzuruWrapper {
return (await this.apiPost("posts/reverse-search", obj)).data; return (await this.apiPost("posts/reverse-search", obj)).data;
} }
static async createFromConfig(siteConfig: SzuruSiteConfig): Promise<SzuruWrapper | null> {
return new SzuruWrapper(siteConfig.domain, siteConfig.username, siteConfig.authToken);
}
private async apiGet( private async apiGet(
url: string, url: string,
additionalHeaders: any = {}, additionalHeaders: any = {},
@ -137,8 +151,4 @@ export default class SzuruWrapper {
throw error ? error : ex; throw error ? error : ex;
} }
} }
static async createFromConfig(siteConfig: SzuruSiteConfig): Promise<SzuruWrapper | null> {
return new SzuruWrapper(siteConfig.domain, siteConfig.username, siteConfig.authToken);
}
} }

49
src/ViewModels.ts Normal file
View File

@ -0,0 +1,49 @@
import { BooruTypes, ContentType, ScrapedNote, ScrapedPost, ScrapedTag } from "neo-scraper";
import { MicroTag, Tag } from "./SzuruTypes";
export class TagViewModel {
public implications: TagViewModel[] = [];
constructor(public name: string, public category?: BooruTypes.TagCategory, public usages?: number | undefined) {}
static fromTag(tag: Tag) {
let x = new TagViewModel(tag.names[0], <BooruTypes.TagCategory>tag.category, tag.usages);
if (tag.implications) {
x.implications = tag.implications.map(y => TagViewModel.fromMicroTag(y));
}
return x;
}
static fromMicroTag(tag: MicroTag) {
return new TagViewModel(tag.names[0], <BooruTypes.TagCategory>tag.category, tag.usages);
}
static fromScapedTag(tag: ScrapedTag): TagViewModel {
return new TagViewModel(tag.name, tag.category);
}
}
export class PostViewModel {
name: string = "";
tags: TagViewModel[] = [];
notes: ScrapedNote[];
contentUrl: string;
pageUrl: string;
contentType: ContentType;
rating: BooruTypes.SafetyRating;
source: string | undefined;
referrer: string | undefined;
constructor(post: ScrapedPost) {
this.contentUrl = post.contentUrl;
this.pageUrl = post.pageUrl;
this.contentType = post.contentType;
this.rating = post.rating;
this.source = post.source;
this.referrer = post.referrer;
this.tags = post.tags.map(x => TagViewModel.fromScapedTag(x));
this.notes = post.notes;
}
}

View File

@ -127,58 +127,14 @@
<script lang="ts"> <script lang="ts">
import Vue from "vue"; import Vue from "vue";
import axios, { CancelTokenSource } from "axios";
import { browser, Runtime, WebRequest } from "webextension-polyfill-ts"; import { browser, Runtime, WebRequest } from "webextension-polyfill-ts";
import { ContentType, ScrapedPost, ScrapedTag, ScrapeResults, BooruTypes } from "neo-scraper"; import { ScrapedPost, ScrapeResults } from "neo-scraper";
import SzuruWrapper from "../SzuruWrapper"; import SzuruWrapper from "../SzuruWrapper";
import { MicroTag, Post, Tag } from "../SzuruTypes"; import { Post, } from "../SzuruTypes";
import { Config, SzuruSiteConfig } from "../Config"; import { Config, SzuruSiteConfig } from "../Config";
import { BrowserCommand, Message, getUrl, isChrome, encodeTagName } from "../Common"; import { BrowserCommand, Message, getUrl, isChrome, encodeTagName } from "../Common";
import axios, { CancelTokenSource } from "axios"; import { PostViewModel, TagViewModel } from "../ViewModels";
class TagViewModel {
public implications: TagViewModel[] = [];
constructor(public name: string, public category?: BooruTypes.TagCategory, public usages?: number | undefined) { }
static fromTag(tag: Tag) {
let x = new TagViewModel(tag.names[0], <BooruTypes.TagCategory>tag.category, tag.usages);
if (tag.implications) {
x.implications = tag.implications.map((y) => TagViewModel.fromMicroTag(y));
}
return x;
}
static fromMicroTag(tag: MicroTag) {
return new TagViewModel(tag.names[0], <BooruTypes.TagCategory>tag.category, tag.usages);
}
static fromScapedTag(tag: ScrapedTag): TagViewModel {
return new TagViewModel(tag.name, tag.category);
}
}
class PostViewModel {
name: string = "";
tags: TagViewModel[] = [];
contentUrl: string;
pageUrl: string;
contentType: ContentType;
rating: BooruTypes.SafetyRating;
source: string | undefined;
referrer: string | undefined;
constructor(post: ScrapedPost) {
this.contentUrl = post.contentUrl;
this.pageUrl = post.pageUrl;
this.contentType = post.contentType;
this.rating = post.rating;
this.source = post.source;
this.referrer = post.referrer;
this.tags = post.tags.map((x) => TagViewModel.fromScapedTag(x));
}
}
export default Vue.extend({ export default Vue.extend({
data() { data() {
@ -391,7 +347,6 @@ export default Vue.extend({
}, },
async loadTagCounts() { async loadTagCounts() {
const allTags = this.posts.flatMap((x) => x.tags); const allTags = this.posts.flatMap((x) => x.tags);
console.log(allTags);
for (let i = 0; i < allTags.length; i += 100) { for (let i = 0; i < allTags.length; i += 100) {
const query = allTags const query = allTags

View File

@ -1,11 +1,11 @@
import { ScrapedPost } from "neo-scraper";
import { browser, Runtime } from "webextension-polyfill-ts"; import { browser, Runtime } from "webextension-polyfill-ts";
import { BrowserCommand, Message, encodeTagName, getUrl } from "../Common"; import { BrowserCommand, Message, encodeTagName, getUrl } from "../Common";
import { Config } from "../Config"; import { Config } from "../Config";
import SzuruWrapper from "../SzuruWrapper"; import SzuruWrapper from "../SzuruWrapper";
import { PostAlreadyUploadedError, SzuruError } from "../SzuruTypes"; import { PostAlreadyUploadedError, SzuruError } from "../SzuruTypes";
import { PostViewModel } from "../ViewModels";
async function uploadPost(post: ScrapedPost) { async function uploadPost(post: PostViewModel) {
console.dir(post); console.dir(post);
const cfg = await Config.load(); const cfg = await Config.load();
@ -82,12 +82,12 @@ async function uploadPost(post: ScrapedPost) {
browser.runtime.sendMessage(new BrowserCommand("push_message", new Message(msg, "error"))); browser.runtime.sendMessage(new BrowserCommand("push_message", new Message(msg, "error")));
break; break;
default: default:
browser.runtime.sendMessage(new BrowserCommand("push_message", new Message(ex, "error"))); browser.runtime.sendMessage(new BrowserCommand("push_message", new Message(ex.message, "error")));
break; break;
} }
} else { } else {
console.error(ex); console.error(ex);
browser.runtime.sendMessage(new BrowserCommand("push_message", new Message(ex, "error"))); browser.runtime.sendMessage(new BrowserCommand("push_message", new Message(ex.message, "error")));
} }
} }
} }

View File

@ -688,6 +688,11 @@ dir-glob@^3.0.1:
dependencies: dependencies:
path-type "^4.0.0" path-type "^4.0.0"
domino@^2.1.6:
version "2.1.6"
resolved "https://registry.yarnpkg.com/domino/-/domino-2.1.6.tgz#fe4ace4310526e5e7b9d12c7de01b7f485a57ffe"
integrity sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ==
ecc-jsbn@~0.1.1: ecc-jsbn@~0.1.1:
version "0.1.2" version "0.1.2"
resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9" resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9"
@ -1417,10 +1422,12 @@ neo-async@^2.6.2:
resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f" resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f"
integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw== integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==
neo-scraper@^0.4.7: neo-scraper@^0.5.0:
version "0.4.7" version "0.5.0"
resolved "https://registry.yarnpkg.com/neo-scraper/-/neo-scraper-0.4.7.tgz#bee74ce7a80e794b7d1dd3e3eed52d91931dbfcf" resolved "https://registry.yarnpkg.com/neo-scraper/-/neo-scraper-0.5.0.tgz#8af59253a1dead386a906dc7281c024ae7938e25"
integrity sha512-Tmv+3xrFTk1c2/vnY2qLmft3KT0IXIOglsSC7ai3+lgmC+1LCVjSqlnWBlIPqTIHCOFnIFhb06H00znN/uAptw== integrity sha512-SUh9kQC/uI2rTSity+cG7kTwDdlQ9XjEEdqw7hkh0AIjB2yi2X9jHe8mVq0VFC/VLaoBU8QlWUstobmfspipGA==
dependencies:
turndown "^7.1.1"
node-releases@^2.0.0: node-releases@^2.0.0:
version "2.0.0" version "2.0.0"
@ -2018,6 +2025,13 @@ tunnel-agent@^0.6.0:
dependencies: dependencies:
safe-buffer "^5.0.1" safe-buffer "^5.0.1"
turndown@^7.1.1:
version "7.1.1"
resolved "https://registry.yarnpkg.com/turndown/-/turndown-7.1.1.tgz#96992f2d9b40a1a03d3ea61ad31b5a5c751ef77f"
integrity sha512-BEkXaWH7Wh7e9bd2QumhfAXk5g34+6QUmmWx+0q6ThaVOLuLUqsnkq35HQ5SBHSaxjSfSM7US5o4lhJNH7B9MA==
dependencies:
domino "^2.1.6"
tweetnacl@^0.14.3, tweetnacl@~0.14.0: tweetnacl@^0.14.3, tweetnacl@~0.14.0:
version "0.14.5" version "0.14.5"
resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64"