Show how often a tag is used in your szurubooru instance + cleanup

next
neobooru 2021-06-13 17:05:10 +02:00
parent 3f195327ea
commit 1f719dc0cb
9 changed files with 124 additions and 37 deletions

View File

@ -1,6 +1,6 @@
{ {
"name": "szuru-chrome", "name": "szuru-chrome",
"version": "0.6.4", "version": "0.7.0",
"description": "Import media from various sites into szurubooru.", "description": "Import media from various sites into szurubooru.",
"author": "neobooru", "author": "neobooru",
"license": "MIT", "license": "MIT",
@ -16,7 +16,7 @@
}, },
"dependencies": { "dependencies": {
"axios": "^0.21.1", "axios": "^0.21.1",
"neo-scraper": "^0.4.3", "neo-scraper": "^0.4.4",
"vue": "^2.6.11", "vue": "^2.6.11",
"webextension-polyfill": "^0.3.1" "webextension-polyfill": "^0.3.1"
}, },

View File

@ -34,3 +34,9 @@ export function getUrl(root: string, ...parts: string[]): string {
export function isChrome() { export function isChrome() {
return /Chrome/.test(navigator.userAgent); return /Chrome/.test(navigator.userAgent);
} }
export function encodeTagName(tagName: string) {
// Searching for posts with re:zero will show an error message about unknown named token.
// Searching for posts with re\:zero will show posts tagged with re:zero.
return tagName.replace(/\:/g, "\\:");
}

View File

@ -12,6 +12,7 @@ export class Config {
importCommentary: boolean = false; importCommentary: boolean = false;
useOriginalSource: boolean = true; useOriginalSource: boolean = true;
autoSearchSimilar: boolean = false; autoSearchSimilar: boolean = false;
loadTagCounts: boolean = true;
sites: Array<SzuruSiteConfig> = []; sites: Array<SzuruSiteConfig> = [];
public async save() { public async save() {

View File

@ -13,6 +13,11 @@ export interface SzuruError {
description: string; description: string;
} }
export interface PostAlreadyUploadedError extends SzuruError {
otherPostId: number;
otherPostUrl: string;
}
export interface PagedSearchResult<T> { export interface PagedSearchResult<T> {
query: string; query: string;
offset: number; offset: number;

View File

@ -40,12 +40,13 @@ export default class SzuruWrapper {
return (await this.apiPut("tag/" + tag.names[0], tag)).data; return (await this.apiPut("tag/" + tag.names[0], tag)).data;
} }
async getTags(query: string): Promise<TagsResult> { async getTags(query: string, offset: number = 0): Promise<TagsResult> {
let url = "tags"; const params = new URLSearchParams();
params.append("offset", offset.toString());
if (query) { if (query) {
url += query; params.append("query", query);
} }
return (await this.apiGet(url)).data; return (await this.apiGet("tags?" + params.toString())).data;
} }
async getTagCategories(): Promise<TagCategoriesResult> { async getTagCategories(): Promise<TagCategoriesResult> {
@ -82,7 +83,7 @@ export default class SzuruWrapper {
}; };
config.headers = { ...this.baseHeaders, ...additionalHeaders }; config.headers = { ...this.baseHeaders, ...additionalHeaders };
return await this.execute(config); return this.execute(config);
} }
private async apiPost(url: string, data: any, additionalHeaders: any = {}): Promise<any> { private async apiPost(url: string, data: any, additionalHeaders: any = {}): Promise<any> {
@ -94,7 +95,7 @@ export default class SzuruWrapper {
}; };
config.headers = { ...this.baseHeaders, ...additionalHeaders }; config.headers = { ...this.baseHeaders, ...additionalHeaders };
return await this.execute(config); return this.execute(config);
} }
private async apiPut(url: string, data: any, additionalHeaders: any = {}): Promise<any> { private async apiPut(url: string, data: any, additionalHeaders: any = {}): Promise<any> {
@ -106,14 +107,13 @@ export default class SzuruWrapper {
}; };
config.headers = { ...this.baseHeaders, ...additionalHeaders }; config.headers = { ...this.baseHeaders, ...additionalHeaders };
return await this.execute(config); return this.execute(config);
} }
private async execute(config: AxiosRequestConfig): Promise<any> { private async execute(config: AxiosRequestConfig): Promise<any> {
if (this.username && this.authToken) { if (this.username && this.authToken) {
const token = "Token " + btoa(`${this.username}:${this.authToken}`); const token = "Token " + btoa(`${this.username}:${this.authToken}`);
config.headers["Authorization"] = token; config.headers["Authorization"] = token;
// console.log(token);
} }
try { try {

View File

@ -32,6 +32,11 @@
<input type="checkbox" name="useOriginalSource" v-model="useOriginalSource" /> <input type="checkbox" name="useOriginalSource" v-model="useOriginalSource" />
</label> </label>
<br /> <br />
<label>
Show how often a tag is used in the selected szurubooru instance
<input type="checkbox" name="loadTagCounts" v-model="loadTagCounts" />
</label>
<br />
<div class="settings-footer-buttons"> <div class="settings-footer-buttons">
<button @click="testConnection">Test connection</button> <button @click="testConnection">Test connection</button>
<button @click="saveSettings" class="primary">Save settings</button> <button @click="saveSettings" class="primary">Save settings</button>
@ -52,6 +57,7 @@ export default Vue.extend({
username: "", username: "",
authToken: "", authToken: "",
autoSearchSimilar: false, autoSearchSimilar: false,
loadTagCounts: false,
useOriginalSource: false, useOriginalSource: false,
}; };
}, },
@ -87,6 +93,7 @@ export default Vue.extend({
siteConfig.authToken = this.authToken; siteConfig.authToken = this.authToken;
config.sites = [siteConfig]; config.sites = [siteConfig];
config.autoSearchSimilar = this.autoSearchSimilar; config.autoSearchSimilar = this.autoSearchSimilar;
config.loadTagCounts = this.loadTagCounts;
config.useOriginalSource = this.useOriginalSource; config.useOriginalSource = this.useOriginalSource;
await config.save(); await config.save();
@ -105,6 +112,7 @@ export default Vue.extend({
} }
this.autoSearchSimilar = config.autoSearchSimilar; this.autoSearchSimilar = config.autoSearchSimilar;
this.loadTagCounts = config.loadTagCounts;
this.useOriginalSource = config.useOriginalSource; this.useOriginalSource = config.useOriginalSource;
}, },
}); });

View File

@ -2,7 +2,7 @@
<div class="popup-container"> <div class="popup-container">
<div class="popup-messages"> <div class="popup-messages">
<ul class="messages"> <ul class="messages">
<li v-for="msg in messages" :key="msg.name" :class="getMessageClasses(msg)" v-html="msg.content"></li> <li v-for="msg in messages" :key="msg.content" :class="getMessageClasses(msg)" v-html="msg.content"></li>
</ul> </ul>
</div> </div>
@ -56,7 +56,7 @@
<li v-for="tag in selectedPost.tags" :key="tag.name"> <li v-for="tag in selectedPost.tags" :key="tag.name">
<a class="remove-tag" @click="removeTag(tag)">x</a> <a class="remove-tag" @click="removeTag(tag)">x</a>
<span :class="getTagClasses(tag)">{{ tag.name }}</span> <span :class="getTagClasses(tag)">{{ tag.name }}</span>
<span class="tag-usages">0</span> <span class="tag-usages">{{ tag.usages ? tag.usages : "" }}</span>
</li> </li>
</ul> </ul>
</div> </div>
@ -74,7 +74,7 @@
<div class="popup-right"> <div class="popup-right">
<div class="popup-section"> <div class="popup-section">
<select v-model="selectedPost"> <select v-model="selectedPost">
<option v-for="post in posts" v-bind:key="post" v-bind:value="post">{{ post.name }}</option> <option v-for="post in posts" v-bind:key="post.name" v-bind:value="post">{{ post.name }}</option>
</select> </select>
</div> </div>
@ -96,14 +96,43 @@
<script lang="ts"> <script lang="ts">
import Vue from "vue"; import Vue from "vue";
import { browser, Runtime, WebRequest } from "webextension-polyfill-ts"; import { browser, Runtime, WebRequest } from "webextension-polyfill-ts";
import { ScrapedPost, ScrapedTag, ScrapeResults } from "neo-scraper"; import { ContentType, ScrapedPost, ScrapedTag, ScrapeResults, BooruTypes } from "neo-scraper";
import SzuruWrapper from "../SzuruWrapper"; import SzuruWrapper from "../SzuruWrapper";
import { Post, SzuruError } from "../SzuruTypes"; import { Post } from "../SzuruTypes";
import { Config, SzuruSiteConfig } from "../Config"; import { Config, SzuruSiteConfig } from "../Config";
import { BrowserCommand, Message, getUrl, isChrome } from "../Common"; import { BrowserCommand, Message, getUrl, isChrome, encodeTagName } from "../Common";
class ScrapedPostViewModel extends ScrapedPost { class TagViewModel {
name: string;
category?: BooruTypes.TagCategory;
usages: number | undefined;
constructor(tag: ScrapedTag) {
this.name = tag.name;
this.category = tag.category;
this.usages = 0;
}
}
class PostViewModel {
name: string = ""; name: string = "";
tags: TagViewModel[] = [];
contentUrl: string;
pageUrl: string;
contentType: ContentType;
rating: BooruTypes.SafetyRating;
source: string | undefined;
referrer: string | undefined;
constructor(post: ScrapedPost) {
this.contentUrl = post.contentUrl;
this.pageUrl = post.pageUrl;
this.contentType = post.contentType;
this.rating = post.rating;
this.source = post.source;
this.referrer = post.referrer;
this.tags = post.tags.map((x) => new TagViewModel(x));
}
} }
export default Vue.extend({ export default Vue.extend({
@ -112,7 +141,7 @@ export default Vue.extend({
config: null as Config | null, config: null as Config | null,
activeSite: null as SzuruSiteConfig | null, activeSite: null as SzuruSiteConfig | null,
szuru: null as SzuruWrapper | null, szuru: null as SzuruWrapper | null,
posts: [] as ScrapedPostViewModel[], posts: [] as PostViewModel[],
selectedPost: new ScrapedPost(), // Shouldn't be null because VueJS gets a mental breakdown when it sees a null. selectedPost: new ScrapedPost(), // Shouldn't be null because VueJS gets a mental breakdown when it sees a null.
messages: [] as Message[], messages: [] as Message[],
addTagInput: "", addTagInput: "",
@ -152,10 +181,10 @@ export default Vue.extend({
for (var result of res.results) { for (var result of res.results) {
for (var i in result.posts) { for (var i in result.posts) {
const vm = Object.assign(new ScrapedPostViewModel(), result.posts[i]); const vm = new PostViewModel(result.posts[i]);
vm.name = `[${result.engine}] Post ${parseInt(i) + 1}`; // parseInt() is required! vm.name = `[${result.engine}] Post ${parseInt(i) + 1}`; // parseInt() is required!
if (this.config?.useOriginalSource == false || vm.source == undefined) { if (!this.config?.useOriginalSource || vm.source == undefined) {
vm.source = vm.pageUrl; vm.source = vm.pageUrl;
} }
@ -165,6 +194,9 @@ export default Vue.extend({
if (this.posts.length > 0) { if (this.posts.length > 0) {
this.selectedPost = this.posts[0]; this.selectedPost = this.posts[0];
if (this.config?.loadTagCounts) {
this.loadTagCounts();
}
} else { } else {
this.pushInfo("No posts found."); this.pushInfo("No posts found.");
} }
@ -293,12 +325,40 @@ export default Vue.extend({
} }
}, },
// Returns ScrapedPost for a contentUrl, or undefined if the given contentUrl does not belong to a ScrapedPost. // Returns ScrapedPost for a contentUrl, or undefined if the given contentUrl does not belong to a ScrapedPost.
getPostForUrl(contentUrl: string): ScrapedPostViewModel | undefined { getPostForUrl(contentUrl: string): PostViewModel | undefined {
for (const post of this.posts) { for (const post of this.posts) {
if (post.contentUrl == contentUrl) return post; if (post.contentUrl == contentUrl) return post;
} }
return undefined; return undefined;
}, },
async loadTagCounts() {
const allTags = this.posts.flatMap((x) => x.tags);
console.log(allTags);
for (let i = 0; i < allTags.length; i += 100) {
const query = allTags
.slice(i, i + 101)
.map((x) => encodeTagName(x.name))
.join();
const resp = await this.szuru?.getTags(query);
console.dir(resp);
if (resp) {
for (let post of this.posts) {
for (let tag of resp.results) {
for (let postTag of post.tags) {
if (postTag.name == tag.names[0]) {
postTag.usages = tag.usages;
break;
}
}
}
}
}
console.dir(this.posts);
}
},
}, },
async mounted() { async mounted() {
this.config = await Config.load(); this.config = await Config.load();

View File

@ -1,9 +1,9 @@
import { ScrapedPost } from "neo-scraper"; import { ScrapedPost } from "neo-scraper";
import { browser, Runtime } from "webextension-polyfill-ts"; import { browser, Runtime } from "webextension-polyfill-ts";
import { BrowserCommand, Message } from "../Common"; import { BrowserCommand, Message, encodeTagName, getUrl } from "../Common";
import { Config } from "../Config"; import { Config } from "../Config";
import SzuruWrapper from "../SzuruWrapper"; import SzuruWrapper from "../SzuruWrapper";
import { SzuruError } from "../SzuruTypes"; import { PostAlreadyUploadedError, SzuruError } from "../SzuruTypes";
async function uploadPost(post: ScrapedPost) { async function uploadPost(post: ScrapedPost) {
console.dir(post); console.dir(post);
@ -38,7 +38,7 @@ async function uploadPost(post: ScrapedPost) {
new BrowserCommand("push_message", new Message(`${unsetCategoryTags.length} tags need a different category`)) new BrowserCommand("push_message", new Message(`${unsetCategoryTags.length} tags need a different category`))
); );
// unsetCategoryTags is of type MicroTag[] and we need a Tag resource to update it, so let's get those // unsetCategoryTags is of type MicroTag[] and we need a Tag resource to update it, so let's get those
const query = "?query=" + unsetCategoryTags.map(x => encodeTagName(x.names[0])); const query = unsetCategoryTags.map(x => encodeTagName(x.names[0])).join();
const tags = (await szuru.getTags(query)).results; const tags = (await szuru.getTags(query)).results;
const existingCategories = (await szuru.getTagCategories()).results; const existingCategories = (await szuru.getTagCategories()).results;
let categoriesChangedCount = 0; let categoriesChangedCount = 0;
@ -55,7 +55,9 @@ async function uploadPost(post: ScrapedPost) {
await szuru.updateTag(tags[i]); await szuru.updateTag(tags[i]);
categoriesChangedCount++; categoriesChangedCount++;
} else { } else {
console.log(`Not adding the '${wantedCategory}' category to the tag '${tags[i].names[0]}' because the szurubooru instance does not have this category.`); console.log(
`Not adding the '${wantedCategory}' category to the tag '${tags[i].names[0]}' because the szurubooru instance does not have this category.`
);
} }
} }
@ -68,10 +70,21 @@ async function uploadPost(post: ScrapedPost) {
} }
} }
} catch (ex) { } catch (ex) {
const error = ex as SzuruError; browser.runtime.sendMessage(new BrowserCommand("remove_messages", 1));
let error = ex as SzuruError;
if (error) { if (error) {
console.error(error); console.error(error);
browser.runtime.sendMessage(new BrowserCommand("push_message", new Message(ex, "error"))); switch (error.name) {
case "PostAlreadyUploadedError":
const otherPostId = (error as PostAlreadyUploadedError).otherPostId;
const url = getUrl(szuru.apiUrl.replace("api", ""), "post", otherPostId.toString());
const msg = `<a href='${url}' target='_blank'>Post already uploaded (${otherPostId})</a>`;
browser.runtime.sendMessage(new BrowserCommand("push_message", new Message(msg, "error")));
break;
default:
browser.runtime.sendMessage(new BrowserCommand("push_message", new Message(ex, "error")));
break;
}
} else { } else {
console.error(ex); console.error(ex);
browser.runtime.sendMessage(new BrowserCommand("push_message", new Message(ex, "error"))); browser.runtime.sendMessage(new BrowserCommand("push_message", new Message(ex, "error")));
@ -79,12 +92,6 @@ async function uploadPost(post: ScrapedPost) {
} }
} }
function encodeTagName(tagName: string) {
// Searching for posts with re:zero will show an error message about unknown named token.
// Searching for posts with re\:zero will show posts tagged with re:zero.
return encodeURIComponent(tagName.replace(/\:/g, "\\:"));
}
async function messageHandler(cmd: BrowserCommand, sender: Runtime.MessageSender): Promise<any> { async function messageHandler(cmd: BrowserCommand, sender: Runtime.MessageSender): Promise<any> {
switch (cmd.name) { switch (cmd.name) {
case "upload_post": case "upload_post":

View File

@ -2470,10 +2470,10 @@ neo-async@^2.5.0, neo-async@^2.6.1:
resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f" resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f"
integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw== integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==
neo-scraper@^0.4.3: neo-scraper@^0.4.4:
version "0.4.3" version "0.4.4"
resolved "https://registry.yarnpkg.com/neo-scraper/-/neo-scraper-0.4.3.tgz#6bd3dc36bc02dc4f6c892821a53d43689a631e46" resolved "https://registry.yarnpkg.com/neo-scraper/-/neo-scraper-0.4.4.tgz#cb6a31158f39d0b8f0e465a76ec95fb9c78adb06"
integrity sha512-o+4iPmz9RrTVeBr8QFRLSTF6UWNaWcyKk8FWQr8JmaI43JzCK8jCeh320/i8sB2Fg6aia8dODP1eoNlM93BJPA== integrity sha512-cNK3nJlqo4rryy5/bwKD6X96Jl2E81nsDhNMfWJcZTlbVfteVEOvpbF9x2AAGgI8bwqHp5LALnPng/O4LmcqXQ==
nice-try@^1.0.4: nice-try@^1.0.4:
version "1.0.5" version "1.0.5"