Show how often a tag is used in your szurubooru instance + cleanup

next
neobooru 2021-06-13 17:05:10 +02:00
parent 3f195327ea
commit 1f719dc0cb
9 changed files with 124 additions and 37 deletions

View File

@ -1,6 +1,6 @@
{
"name": "szuru-chrome",
"version": "0.6.4",
"version": "0.7.0",
"description": "Import media from various sites into szurubooru.",
"author": "neobooru",
"license": "MIT",
@ -16,7 +16,7 @@
},
"dependencies": {
"axios": "^0.21.1",
"neo-scraper": "^0.4.3",
"neo-scraper": "^0.4.4",
"vue": "^2.6.11",
"webextension-polyfill": "^0.3.1"
},

View File

@ -34,3 +34,9 @@ export function getUrl(root: string, ...parts: string[]): string {
export function isChrome() {
return /Chrome/.test(navigator.userAgent);
}
export function encodeTagName(tagName: string) {
// Searching for posts with re:zero will show an error message about unknown named token.
// Searching for posts with re\:zero will show posts tagged with re:zero.
return tagName.replace(/\:/g, "\\:");
}

View File

@ -12,6 +12,7 @@ export class Config {
importCommentary: boolean = false;
useOriginalSource: boolean = true;
autoSearchSimilar: boolean = false;
loadTagCounts: boolean = true;
sites: Array<SzuruSiteConfig> = [];
public async save() {

View File

@ -13,6 +13,11 @@ export interface SzuruError {
description: string;
}
export interface PostAlreadyUploadedError extends SzuruError {
otherPostId: number;
otherPostUrl: string;
}
export interface PagedSearchResult<T> {
query: string;
offset: number;

View File

@ -40,12 +40,13 @@ export default class SzuruWrapper {
return (await this.apiPut("tag/" + tag.names[0], tag)).data;
}
async getTags(query: string): Promise<TagsResult> {
let url = "tags";
async getTags(query: string, offset: number = 0): Promise<TagsResult> {
const params = new URLSearchParams();
params.append("offset", offset.toString());
if (query) {
url += query;
params.append("query", query);
}
return (await this.apiGet(url)).data;
return (await this.apiGet("tags?" + params.toString())).data;
}
async getTagCategories(): Promise<TagCategoriesResult> {
@ -82,7 +83,7 @@ export default class SzuruWrapper {
};
config.headers = { ...this.baseHeaders, ...additionalHeaders };
return await this.execute(config);
return this.execute(config);
}
private async apiPost(url: string, data: any, additionalHeaders: any = {}): Promise<any> {
@ -94,7 +95,7 @@ export default class SzuruWrapper {
};
config.headers = { ...this.baseHeaders, ...additionalHeaders };
return await this.execute(config);
return this.execute(config);
}
private async apiPut(url: string, data: any, additionalHeaders: any = {}): Promise<any> {
@ -106,14 +107,13 @@ export default class SzuruWrapper {
};
config.headers = { ...this.baseHeaders, ...additionalHeaders };
return await this.execute(config);
return this.execute(config);
}
private async execute(config: AxiosRequestConfig): Promise<any> {
if (this.username && this.authToken) {
const token = "Token " + btoa(`${this.username}:${this.authToken}`);
config.headers["Authorization"] = token;
// console.log(token);
}
try {

View File

@ -32,6 +32,11 @@
<input type="checkbox" name="useOriginalSource" v-model="useOriginalSource" />
</label>
<br />
<label>
Show how often a tag is used in the selected szurubooru instance
<input type="checkbox" name="loadTagCounts" v-model="loadTagCounts" />
</label>
<br />
<div class="settings-footer-buttons">
<button @click="testConnection">Test connection</button>
<button @click="saveSettings" class="primary">Save settings</button>
@ -52,6 +57,7 @@ export default Vue.extend({
username: "",
authToken: "",
autoSearchSimilar: false,
loadTagCounts: false,
useOriginalSource: false,
};
},
@ -87,6 +93,7 @@ export default Vue.extend({
siteConfig.authToken = this.authToken;
config.sites = [siteConfig];
config.autoSearchSimilar = this.autoSearchSimilar;
config.loadTagCounts = this.loadTagCounts;
config.useOriginalSource = this.useOriginalSource;
await config.save();
@ -105,6 +112,7 @@ export default Vue.extend({
}
this.autoSearchSimilar = config.autoSearchSimilar;
this.loadTagCounts = config.loadTagCounts;
this.useOriginalSource = config.useOriginalSource;
},
});

View File

@ -2,7 +2,7 @@
<div class="popup-container">
<div class="popup-messages">
<ul class="messages">
<li v-for="msg in messages" :key="msg.name" :class="getMessageClasses(msg)" v-html="msg.content"></li>
<li v-for="msg in messages" :key="msg.content" :class="getMessageClasses(msg)" v-html="msg.content"></li>
</ul>
</div>
@ -56,7 +56,7 @@
<li v-for="tag in selectedPost.tags" :key="tag.name">
<a class="remove-tag" @click="removeTag(tag)">x</a>
<span :class="getTagClasses(tag)">{{ tag.name }}</span>
<span class="tag-usages">0</span>
<span class="tag-usages">{{ tag.usages ? tag.usages : "" }}</span>
</li>
</ul>
</div>
@ -74,7 +74,7 @@
<div class="popup-right">
<div class="popup-section">
<select v-model="selectedPost">
<option v-for="post in posts" v-bind:key="post" v-bind:value="post">{{ post.name }}</option>
<option v-for="post in posts" v-bind:key="post.name" v-bind:value="post">{{ post.name }}</option>
</select>
</div>
@ -96,14 +96,43 @@
<script lang="ts">
import Vue from "vue";
import { browser, Runtime, WebRequest } from "webextension-polyfill-ts";
import { ScrapedPost, ScrapedTag, ScrapeResults } from "neo-scraper";
import { ContentType, ScrapedPost, ScrapedTag, ScrapeResults, BooruTypes } from "neo-scraper";
import SzuruWrapper from "../SzuruWrapper";
import { Post, SzuruError } from "../SzuruTypes";
import { Post } from "../SzuruTypes";
import { Config, SzuruSiteConfig } from "../Config";
import { BrowserCommand, Message, getUrl, isChrome } from "../Common";
import { BrowserCommand, Message, getUrl, isChrome, encodeTagName } from "../Common";
class ScrapedPostViewModel extends ScrapedPost {
class TagViewModel {
name: string;
category?: BooruTypes.TagCategory;
usages: number | undefined;
constructor(tag: ScrapedTag) {
this.name = tag.name;
this.category = tag.category;
this.usages = 0;
}
}
class PostViewModel {
name: string = "";
tags: TagViewModel[] = [];
contentUrl: string;
pageUrl: string;
contentType: ContentType;
rating: BooruTypes.SafetyRating;
source: string | undefined;
referrer: string | undefined;
constructor(post: ScrapedPost) {
this.contentUrl = post.contentUrl;
this.pageUrl = post.pageUrl;
this.contentType = post.contentType;
this.rating = post.rating;
this.source = post.source;
this.referrer = post.referrer;
this.tags = post.tags.map((x) => new TagViewModel(x));
}
}
export default Vue.extend({
@ -112,7 +141,7 @@ export default Vue.extend({
config: null as Config | null,
activeSite: null as SzuruSiteConfig | null,
szuru: null as SzuruWrapper | null,
posts: [] as ScrapedPostViewModel[],
posts: [] as PostViewModel[],
selectedPost: new ScrapedPost(), // Shouldn't be null because VueJS gets a mental breakdown when it sees a null.
messages: [] as Message[],
addTagInput: "",
@ -152,10 +181,10 @@ export default Vue.extend({
for (var result of res.results) {
for (var i in result.posts) {
const vm = Object.assign(new ScrapedPostViewModel(), result.posts[i]);
const vm = new PostViewModel(result.posts[i]);
vm.name = `[${result.engine}] Post ${parseInt(i) + 1}`; // parseInt() is required!
if (this.config?.useOriginalSource == false || vm.source == undefined) {
if (!this.config?.useOriginalSource || vm.source == undefined) {
vm.source = vm.pageUrl;
}
@ -165,6 +194,9 @@ export default Vue.extend({
if (this.posts.length > 0) {
this.selectedPost = this.posts[0];
if (this.config?.loadTagCounts) {
this.loadTagCounts();
}
} else {
this.pushInfo("No posts found.");
}
@ -293,12 +325,40 @@ export default Vue.extend({
}
},
// Returns ScrapedPost for a contentUrl, or undefined if the given contentUrl does not belong to a ScrapedPost.
getPostForUrl(contentUrl: string): ScrapedPostViewModel | undefined {
getPostForUrl(contentUrl: string): PostViewModel | undefined {
for (const post of this.posts) {
if (post.contentUrl == contentUrl) return post;
}
return undefined;
},
async loadTagCounts() {
const allTags = this.posts.flatMap((x) => x.tags);
console.log(allTags);
for (let i = 0; i < allTags.length; i += 100) {
const query = allTags
.slice(i, i + 101)
.map((x) => encodeTagName(x.name))
.join();
const resp = await this.szuru?.getTags(query);
console.dir(resp);
if (resp) {
for (let post of this.posts) {
for (let tag of resp.results) {
for (let postTag of post.tags) {
if (postTag.name == tag.names[0]) {
postTag.usages = tag.usages;
break;
}
}
}
}
}
console.dir(this.posts);
}
},
},
async mounted() {
this.config = await Config.load();

View File

@ -1,9 +1,9 @@
import { ScrapedPost } from "neo-scraper";
import { browser, Runtime } from "webextension-polyfill-ts";
import { BrowserCommand, Message } from "../Common";
import { BrowserCommand, Message, encodeTagName, getUrl } from "../Common";
import { Config } from "../Config";
import SzuruWrapper from "../SzuruWrapper";
import { SzuruError } from "../SzuruTypes";
import { PostAlreadyUploadedError, SzuruError } from "../SzuruTypes";
async function uploadPost(post: ScrapedPost) {
console.dir(post);
@ -38,7 +38,7 @@ async function uploadPost(post: ScrapedPost) {
new BrowserCommand("push_message", new Message(`${unsetCategoryTags.length} tags need a different category`))
);
// unsetCategoryTags is of type MicroTag[] and we need a Tag resource to update it, so let's get those
const query = "?query=" + unsetCategoryTags.map(x => encodeTagName(x.names[0]));
const query = unsetCategoryTags.map(x => encodeTagName(x.names[0])).join();
const tags = (await szuru.getTags(query)).results;
const existingCategories = (await szuru.getTagCategories()).results;
let categoriesChangedCount = 0;
@ -55,7 +55,9 @@ async function uploadPost(post: ScrapedPost) {
await szuru.updateTag(tags[i]);
categoriesChangedCount++;
} else {
console.log(`Not adding the '${wantedCategory}' category to the tag '${tags[i].names[0]}' because the szurubooru instance does not have this category.`);
console.log(
`Not adding the '${wantedCategory}' category to the tag '${tags[i].names[0]}' because the szurubooru instance does not have this category.`
);
}
}
@ -68,10 +70,21 @@ async function uploadPost(post: ScrapedPost) {
}
}
} catch (ex) {
const error = ex as SzuruError;
browser.runtime.sendMessage(new BrowserCommand("remove_messages", 1));
let error = ex as SzuruError;
if (error) {
console.error(error);
browser.runtime.sendMessage(new BrowserCommand("push_message", new Message(ex, "error")));
switch (error.name) {
case "PostAlreadyUploadedError":
const otherPostId = (error as PostAlreadyUploadedError).otherPostId;
const url = getUrl(szuru.apiUrl.replace("api", ""), "post", otherPostId.toString());
const msg = `<a href='${url}' target='_blank'>Post already uploaded (${otherPostId})</a>`;
browser.runtime.sendMessage(new BrowserCommand("push_message", new Message(msg, "error")));
break;
default:
browser.runtime.sendMessage(new BrowserCommand("push_message", new Message(ex, "error")));
break;
}
} else {
console.error(ex);
browser.runtime.sendMessage(new BrowserCommand("push_message", new Message(ex, "error")));
@ -79,12 +92,6 @@ async function uploadPost(post: ScrapedPost) {
}
}
function encodeTagName(tagName: string) {
// Searching for posts with re:zero will show an error message about unknown named token.
// Searching for posts with re\:zero will show posts tagged with re:zero.
return encodeURIComponent(tagName.replace(/\:/g, "\\:"));
}
async function messageHandler(cmd: BrowserCommand, sender: Runtime.MessageSender): Promise<any> {
switch (cmd.name) {
case "upload_post":

View File

@ -2470,10 +2470,10 @@ neo-async@^2.5.0, neo-async@^2.6.1:
resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f"
integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==
neo-scraper@^0.4.3:
version "0.4.3"
resolved "https://registry.yarnpkg.com/neo-scraper/-/neo-scraper-0.4.3.tgz#6bd3dc36bc02dc4f6c892821a53d43689a631e46"
integrity sha512-o+4iPmz9RrTVeBr8QFRLSTF6UWNaWcyKk8FWQr8JmaI43JzCK8jCeh320/i8sB2Fg6aia8dODP1eoNlM93BJPA==
neo-scraper@^0.4.4:
version "0.4.4"
resolved "https://registry.yarnpkg.com/neo-scraper/-/neo-scraper-0.4.4.tgz#cb6a31158f39d0b8f0e465a76ec95fb9c78adb06"
integrity sha512-cNK3nJlqo4rryy5/bwKD6X96Jl2E81nsDhNMfWJcZTlbVfteVEOvpbF9x2AAGgI8bwqHp5LALnPng/O4LmcqXQ==
nice-try@^1.0.4:
version "1.0.5"