parent
9843f0b641
commit
eae1b3949f
|
|
@ -70,7 +70,7 @@ jobs:
|
|||
runs-on: ubuntu-20.04
|
||||
# Doesn't really need the Chrome build, but we can reuse its cache.
|
||||
needs: build_chrome
|
||||
# Only run on master branch AND if tagging or manually dispatching.
|
||||
# Only run (on master branch && if tagged) or (manually dispatching).
|
||||
if: startsWith(github.ref, 'refs/tags/v') || github.event.inputs.publish_amo_listed == true
|
||||
steps:
|
||||
- name: Checkout
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@
|
|||
"deepmerge": "^4.3.1",
|
||||
"lodash": "^4.17.21",
|
||||
"markdown-it": "^14.0.0",
|
||||
"neo-scraper": "^0.8.0",
|
||||
"neo-scraper": "^0.9.0",
|
||||
"normalize.css": "^8.0.1",
|
||||
"pinia": "^2.1.7",
|
||||
"primeflex": "^3.3.1",
|
||||
|
|
|
|||
|
|
@ -24,8 +24,8 @@ importers:
|
|||
specifier: ^14.0.0
|
||||
version: 14.0.0
|
||||
neo-scraper:
|
||||
specifier: ^0.8.0
|
||||
version: 0.8.0
|
||||
specifier: ^0.9.0
|
||||
version: 0.9.0
|
||||
normalize.css:
|
||||
specifier: ^8.0.1
|
||||
version: 8.0.1
|
||||
|
|
@ -2248,8 +2248,8 @@ packages:
|
|||
resolution: {integrity: sha512-zIdGUrPRFTUELUvr3Gmc7KZ2Sw/h1PiVM0Af/oHB6zgnV1ikqSfRk+TOufi79aHYCW3NiOXmr1BP5nWbzojLaA==}
|
||||
hasBin: true
|
||||
|
||||
neo-scraper@0.8.0:
|
||||
resolution: {integrity: sha512-E01oBGuHx9MJOrerkBJpiV0cAHO0uZi28xly4SbELmajf8LQgg1HOH1pq30krBXj5J1px5oRr+3WSO1+1H+sHA==}
|
||||
neo-scraper@0.9.0:
|
||||
resolution: {integrity: sha512-KpDrkNSO3pYwYewFWLjrY/OZzLDWh+6Ge021JBExVxA0ZAr67MAKzaNkDXVptW+laz1eZ9Cnl5rvhQZcn+Y0Vw==}
|
||||
|
||||
nice-try@1.0.5:
|
||||
resolution: {integrity: sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==}
|
||||
|
|
@ -5725,7 +5725,7 @@ snapshots:
|
|||
ncp@2.0.0:
|
||||
optional: true
|
||||
|
||||
neo-scraper@0.8.0:
|
||||
neo-scraper@0.9.0:
|
||||
dependencies:
|
||||
js-cookie: 3.0.5
|
||||
turndown: 7.1.2
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ export class TagDetails {
|
|||
public names: string[],
|
||||
public category?: string,
|
||||
public usages?: number,
|
||||
) {}
|
||||
) { }
|
||||
|
||||
get name() {
|
||||
return this.names[0];
|
||||
|
|
@ -38,7 +38,7 @@ export class PoolDetails {
|
|||
public names: string[],
|
||||
public category?: string,
|
||||
public postCount?: number,
|
||||
) {}
|
||||
) { }
|
||||
|
||||
get name() {
|
||||
return this.names[0];
|
||||
|
|
@ -73,11 +73,11 @@ export class ScrapedPostDetails {
|
|||
contentType: ContentType;
|
||||
contentSubType: string | undefined;
|
||||
rating: BooruTypes.SafetyRating;
|
||||
source = "";
|
||||
source;
|
||||
uploadMode: UploadMode;
|
||||
referrer?: string;
|
||||
resolution?: [number, number];
|
||||
instanceSpecificData: MappedInstanceSpecificData = {};
|
||||
uploadMode: UploadMode = "url";
|
||||
|
||||
constructor(post: ScrapedPost) {
|
||||
this.contentUrl = post.contentUrl;
|
||||
|
|
@ -91,6 +91,7 @@ export class ScrapedPostDetails {
|
|||
this.tags = post.tags.map((x) => TagDetails.fromScapedTag(x));
|
||||
this.notes = post.notes;
|
||||
this.resolution = post.resolution;
|
||||
this.uploadMode = post.uploadMode;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -108,7 +109,7 @@ export class SimilarPostInfo {
|
|||
constructor(
|
||||
public readonly id: number,
|
||||
public readonly percentage: number,
|
||||
) {}
|
||||
) { }
|
||||
}
|
||||
|
||||
export type PostUploadState = "uploading" | "uploaded" | "error";
|
||||
|
|
@ -147,7 +148,7 @@ export class PostUploadCommandData {
|
|||
constructor(
|
||||
public readonly post: ScrapedPostDetails,
|
||||
public readonly selectedSite: SzuruSiteConfig,
|
||||
) {}
|
||||
) { }
|
||||
}
|
||||
|
||||
export class SetPostUploadInfoData {
|
||||
|
|
@ -155,7 +156,7 @@ export class SetPostUploadInfoData {
|
|||
public instanceId: string,
|
||||
public postId: string,
|
||||
public info: PostUploadInfo,
|
||||
) {}
|
||||
) { }
|
||||
}
|
||||
|
||||
export class SetExactPostId {
|
||||
|
|
@ -163,7 +164,7 @@ export class SetExactPostId {
|
|||
public readonly instanceId: string,
|
||||
public readonly postId: string,
|
||||
public readonly exactPostId: number,
|
||||
) {}
|
||||
) { }
|
||||
}
|
||||
|
||||
export class PostUpdateCommandData {
|
||||
|
|
@ -171,14 +172,14 @@ export class PostUpdateCommandData {
|
|||
public readonly postId: number,
|
||||
public readonly updateRequest: UpdatePostRequest,
|
||||
public readonly selectedSite: SzuruSiteConfig,
|
||||
) {}
|
||||
) { }
|
||||
}
|
||||
|
||||
export class FetchCommandData {
|
||||
constructor(
|
||||
public readonly url: string,
|
||||
public readonly options: RequestInit | undefined = undefined,
|
||||
) {}
|
||||
) { }
|
||||
}
|
||||
|
||||
export class SzuruSiteConfig {
|
||||
|
|
@ -192,7 +193,7 @@ export class TagCategoryColor {
|
|||
constructor(
|
||||
public name: string,
|
||||
public color: string,
|
||||
) {}
|
||||
) { }
|
||||
}
|
||||
|
||||
export const getDefaultTagCategories = () => [
|
||||
|
|
|
|||
|
|
@ -11,6 +11,8 @@ import Column from "primevue/column";
|
|||
import { SzuruSiteConfig, TagCategoryColor, getDefaultTagCategories } from "~/models";
|
||||
import SzurubooruApi from "~/api";
|
||||
|
||||
type StatusType = "success" | "error" | "quiet";
|
||||
|
||||
const statusText = ref("");
|
||||
const statusType = ref("status-quiet");
|
||||
const versionInfo = "Version: " + (import.meta.env.VITE_SZ_VERSION ?? browser.runtime.getManifest().version);
|
||||
|
|
@ -26,9 +28,7 @@ const selectedSite = computed(() => {
|
|||
}
|
||||
});
|
||||
|
||||
type StatusType = "success" | "error" | "quiet";
|
||||
|
||||
let mode = useColorMode({ emitAuto: true });
|
||||
const mode = useColorMode({ emitAuto: true });
|
||||
|
||||
async function testConnection() {
|
||||
if (
|
||||
|
|
@ -221,8 +221,13 @@ wnd.szc_set_config_version = (v = 0) => (cfg.value.version = v);
|
|||
<template v-if="selectedSite">
|
||||
<div class="field col-12">
|
||||
<label>URL</label>
|
||||
<input v-if="selectedSite" text="Szurubooru URL" type="text" name="domain"
|
||||
v-model="selectedSite.domain" />
|
||||
<input
|
||||
v-if="selectedSite"
|
||||
text="Szurubooru URL"
|
||||
type="text"
|
||||
name="domain"
|
||||
v-model="selectedSite.domain"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div class="field col-12 md:col-6">
|
||||
|
|
@ -251,10 +256,14 @@ wnd.szc_set_config_version = (v = 0) => (cfg.value.version = v);
|
|||
<template v-if="field == 'color'">
|
||||
<div class="color-preview">
|
||||
<input type="text" class="color" v-model="data[field]" />
|
||||
<div class="preview background-preview"
|
||||
:style="{ 'border-color': data[field], 'background-color': data[field] }"></div>
|
||||
<div class="preview text-preview" :style="{ 'border-color': data[field], color: data[field] }">
|
||||
</div>
|
||||
<div
|
||||
class="preview background-preview"
|
||||
:style="{ 'border-color': data[field], 'background-color': data[field] }"
|
||||
></div>
|
||||
<div
|
||||
class="preview text-preview"
|
||||
:style="{ 'border-color': data[field], color: data[field] }"
|
||||
></div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
|
|
@ -285,6 +294,16 @@ wnd.szc_set_config_version = (v = 0) => (cfg.value.version = v);
|
|||
<!-- TODO: Category ignore list -->
|
||||
</TabPanel>
|
||||
|
||||
<!-- <TabPanel header="About">
|
||||
<div class="grid">
|
||||
<div v-for="engine in scraper.engines" :key="engine.name" class="col-12 md:col-4 flex flex-column">
|
||||
<span class="font-bold">{{ engine.name }}</span>
|
||||
<span>Features: {{ engine.features.join(", ") }}</span>
|
||||
<span>Hosts: {{ engine.supportedHosts.join(", ") }}</span>
|
||||
</div>
|
||||
</div>
|
||||
</TabPanel> -->
|
||||
|
||||
<!--
|
||||
<TabPanel header="Developer">
|
||||
<div class="flex flex-column gap-2">
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
<script setup lang="ts">
|
||||
import { useDark } from "@vueuse/core";
|
||||
import { cloneDeep } from "lodash";
|
||||
import { NeoScraper, ScrapeResults } from "neo-scraper";
|
||||
import { ScrapeResults } from "neo-scraper";
|
||||
import { getUrl, encodeTagName, getErrorMessage, getPostInfoSummary } from "~/utils";
|
||||
import {
|
||||
BrowserCommand,
|
||||
|
|
@ -14,7 +14,6 @@ import {
|
|||
SzuruSiteConfig,
|
||||
PoolDetails,
|
||||
} from "~/models";
|
||||
import { ImageSearchResult } from "~/api/models";
|
||||
import { isMobile } from "~/env";
|
||||
import { DeepReadonly } from "vue";
|
||||
import { cfg, usePopupStore } from "~/stores";
|
||||
|
|
@ -99,14 +98,11 @@ async function grabPost() {
|
|||
// Clear current posts array
|
||||
pop.posts.splice(0);
|
||||
|
||||
const scraper = new NeoScraper();
|
||||
|
||||
for (const result of res.results) {
|
||||
const engine = scraper.engines.find((x) => x.name == result.engine);
|
||||
|
||||
for (const i in result.posts) {
|
||||
const vm = new ScrapedPostDetails(result.posts[i]);
|
||||
vm.name = `[${result.engine}] Post ${parseInt(i) + 1}`; // parseInt() is required!
|
||||
const name = result.posts[i].name ?? `Post ${parseInt(i) + 1}`; // parseInt() is required!
|
||||
vm.name = `[${result.engine}] ${name}`;
|
||||
|
||||
if (!cfg.value.addAllParsedTags) {
|
||||
vm.tags.splice(0);
|
||||
|
|
@ -125,10 +121,6 @@ async function grabPost() {
|
|||
vm.instanceSpecificData[site.id] = {};
|
||||
}
|
||||
|
||||
if (engine) {
|
||||
vm.uploadMode = engine.uploadMode;
|
||||
}
|
||||
|
||||
pop.posts.push(vm);
|
||||
}
|
||||
}
|
||||
|
|
@ -145,7 +137,7 @@ async function grabPost() {
|
|||
}
|
||||
|
||||
if (cfg.value.fetchPostInfo) {
|
||||
await fetchPostInfo();
|
||||
await fetchPostsInfo();
|
||||
}
|
||||
|
||||
// Delayed call to findSimilar, as fetchPostInfo might change the post's contentUrl.
|
||||
|
|
@ -168,7 +160,7 @@ async function upload() {
|
|||
}
|
||||
|
||||
try {
|
||||
const post: ScrapedPostDetails = cloneDeep(pop.selectedPost);
|
||||
const post: ScrapedPostDetails = cloneDeep(pop.selectedPost)!;
|
||||
|
||||
// uploadMode "content" requires a content token to work. So ensure it is set.
|
||||
if (post.uploadMode == "content") {
|
||||
|
|
@ -291,10 +283,10 @@ async function findSimilar(post: ScrapedPostDetails | undefined) {
|
|||
|
||||
isSearchingForSimilarPosts.value++;
|
||||
|
||||
await ensurePostHasContentToken(post);
|
||||
|
||||
try {
|
||||
const res = await selectedInstance.reverseSearchToken(instanceSpecificData.contentToken);
|
||||
await ensurePostHasContentToken(post);
|
||||
|
||||
const res = await selectedInstance.reverseSearchToken(instanceSpecificData.contentToken!);
|
||||
|
||||
instanceSpecificData.reverseSearchResult = {
|
||||
exactPostId: res.exactPost?.id,
|
||||
|
|
@ -331,33 +323,58 @@ async function loadTagCounts() {
|
|||
}
|
||||
}
|
||||
|
||||
async function fetchPostInfo() {
|
||||
async function updatePostWithRemoteInfo(post: ScrapedPostDetails, contentUrl: string) {
|
||||
// We are missing cookies/etc which means that the request might fail.
|
||||
// If you want access to the cookies/session/etc then you need to execute this code inside the content script.
|
||||
try {
|
||||
// This request follows redirects.
|
||||
const res = await fetch(contentUrl, { method: "HEAD" });
|
||||
const size = res.headers.get("Content-Length");
|
||||
const type = res.headers.get("Content-Type");
|
||||
|
||||
if (type) {
|
||||
if (type.indexOf("text/html") != -1) {
|
||||
// If we get a HTML page back it usually means that the request failed.
|
||||
// Not all sites (e.g. e-hentai) reply with a 403.
|
||||
throw new Error(
|
||||
"Received a text/html content type. This probably means that we don't have permission to access the resource.",
|
||||
);
|
||||
}
|
||||
|
||||
const [_main, sub] = type.split("/");
|
||||
if (sub) post.contentSubType = sub.toUpperCase();
|
||||
}
|
||||
|
||||
// This should be after the type check, because the type check also checks whether the response we get is valid/unsable.
|
||||
if (size) post.contentSize = parseInt(size);
|
||||
|
||||
// Update url if it changes. The url can change when:
|
||||
// a. the extraContentUrl was successfully loaded
|
||||
// b. because of redirects
|
||||
if (res.url != post.contentUrl) {
|
||||
// The developer needs to make sure that the remote server can load this URL if `post.uploadMode == 'url'`!
|
||||
console.log(`Updating post.contentUrl to '${res.url}'`);
|
||||
post.contentUrl = res.url;
|
||||
}
|
||||
|
||||
return true;
|
||||
} catch (ex) {
|
||||
console.error(ex);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchPostsInfo() {
|
||||
for (const post of pop.posts) {
|
||||
if (!post.contentSize || post.extraContentUrl) {
|
||||
// We are missing cookies/etc which means that the request might fail.
|
||||
// If you want access to the cookies/session/etc then you need to execute this code inside the content script.
|
||||
try {
|
||||
const contentUrl = post.extraContentUrl ?? post.contentUrl;
|
||||
|
||||
// TODO: Check whether we need to call this in the background page, or if it is fine to call it from the popup.
|
||||
const res = await fetch(contentUrl, { method: "HEAD" });
|
||||
const size = res.headers.get("Content-Length");
|
||||
const type = res.headers.get("Content-Type");
|
||||
|
||||
if (size) post.contentSize = parseInt(size);
|
||||
|
||||
if (type) {
|
||||
const [_main, sub] = type.split("/");
|
||||
if (sub) post.contentSubType = sub.toUpperCase();
|
||||
}
|
||||
|
||||
// Resolve extraContentUrl redirects.
|
||||
if (post.extraContentUrl && res.url != post.contentUrl) {
|
||||
post.contentUrl = res.url;
|
||||
}
|
||||
} catch (ex) {
|
||||
// TODO: Maybe display an error in the UI.
|
||||
console.error(ex);
|
||||
let ok = false;
|
||||
if (post.extraContentUrl) {
|
||||
// Prioritize info from extraContentUrl.
|
||||
ok = await updatePostWithRemoteInfo(post, post.extraContentUrl);
|
||||
}
|
||||
if (!ok) {
|
||||
// Fall back to normal contentUrl if we can't get the data from the extraContentUrl.
|
||||
await updatePostWithRemoteInfo(post, post.contentUrl);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -369,7 +386,8 @@ function getUpdatedTagsText(count: number) {
|
|||
}
|
||||
|
||||
function onResolutionLoaded(res: any) {
|
||||
if (pop.selectedPost && !pop.selectedPost.resolution) {
|
||||
// This no longer checks for `!pop.selectedPost.resolution` because the resolution can change if the contentUrl is updated.
|
||||
if (pop.selectedPost) {
|
||||
pop.selectedPost.resolution = res;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue