Update neo-scraper to 0.9.0 + post.uploadMode changes

pull/55/head v0.12.1
neobooru 2024-06-25 23:46:05 +02:00
parent 9843f0b641
commit eae1b3949f
6 changed files with 107 additions and 69 deletions

View File

@ -70,7 +70,7 @@ jobs:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
# Doesn't really need the Chrome build, but we can reuse its cache. # Doesn't really need the Chrome build, but we can reuse its cache.
needs: build_chrome needs: build_chrome
# Only run on master branch AND if tagging or manually dispatching. # Only run (on master branch && if tagged) or (manually dispatching).
if: startsWith(github.ref, 'refs/tags/v') || github.event.inputs.publish_amo_listed == true if: startsWith(github.ref, 'refs/tags/v') || github.event.inputs.publish_amo_listed == true
steps: steps:
- name: Checkout - name: Checkout

View File

@ -32,7 +32,7 @@
"deepmerge": "^4.3.1", "deepmerge": "^4.3.1",
"lodash": "^4.17.21", "lodash": "^4.17.21",
"markdown-it": "^14.0.0", "markdown-it": "^14.0.0",
"neo-scraper": "^0.8.0", "neo-scraper": "^0.9.0",
"normalize.css": "^8.0.1", "normalize.css": "^8.0.1",
"pinia": "^2.1.7", "pinia": "^2.1.7",
"primeflex": "^3.3.1", "primeflex": "^3.3.1",

View File

@ -24,8 +24,8 @@ importers:
specifier: ^14.0.0 specifier: ^14.0.0
version: 14.0.0 version: 14.0.0
neo-scraper: neo-scraper:
specifier: ^0.8.0 specifier: ^0.9.0
version: 0.8.0 version: 0.9.0
normalize.css: normalize.css:
specifier: ^8.0.1 specifier: ^8.0.1
version: 8.0.1 version: 8.0.1
@ -2248,8 +2248,8 @@ packages:
resolution: {integrity: sha512-zIdGUrPRFTUELUvr3Gmc7KZ2Sw/h1PiVM0Af/oHB6zgnV1ikqSfRk+TOufi79aHYCW3NiOXmr1BP5nWbzojLaA==} resolution: {integrity: sha512-zIdGUrPRFTUELUvr3Gmc7KZ2Sw/h1PiVM0Af/oHB6zgnV1ikqSfRk+TOufi79aHYCW3NiOXmr1BP5nWbzojLaA==}
hasBin: true hasBin: true
neo-scraper@0.8.0: neo-scraper@0.9.0:
resolution: {integrity: sha512-E01oBGuHx9MJOrerkBJpiV0cAHO0uZi28xly4SbELmajf8LQgg1HOH1pq30krBXj5J1px5oRr+3WSO1+1H+sHA==} resolution: {integrity: sha512-KpDrkNSO3pYwYewFWLjrY/OZzLDWh+6Ge021JBExVxA0ZAr67MAKzaNkDXVptW+laz1eZ9Cnl5rvhQZcn+Y0Vw==}
nice-try@1.0.5: nice-try@1.0.5:
resolution: {integrity: sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==} resolution: {integrity: sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==}
@ -5725,7 +5725,7 @@ snapshots:
ncp@2.0.0: ncp@2.0.0:
optional: true optional: true
neo-scraper@0.8.0: neo-scraper@0.9.0:
dependencies: dependencies:
js-cookie: 3.0.5 js-cookie: 3.0.5
turndown: 7.1.2 turndown: 7.1.2

View File

@ -8,7 +8,7 @@ export class TagDetails {
public names: string[], public names: string[],
public category?: string, public category?: string,
public usages?: number, public usages?: number,
) {} ) { }
get name() { get name() {
return this.names[0]; return this.names[0];
@ -38,7 +38,7 @@ export class PoolDetails {
public names: string[], public names: string[],
public category?: string, public category?: string,
public postCount?: number, public postCount?: number,
) {} ) { }
get name() { get name() {
return this.names[0]; return this.names[0];
@ -73,11 +73,11 @@ export class ScrapedPostDetails {
contentType: ContentType; contentType: ContentType;
contentSubType: string | undefined; contentSubType: string | undefined;
rating: BooruTypes.SafetyRating; rating: BooruTypes.SafetyRating;
source = ""; source;
uploadMode: UploadMode;
referrer?: string; referrer?: string;
resolution?: [number, number]; resolution?: [number, number];
instanceSpecificData: MappedInstanceSpecificData = {}; instanceSpecificData: MappedInstanceSpecificData = {};
uploadMode: UploadMode = "url";
constructor(post: ScrapedPost) { constructor(post: ScrapedPost) {
this.contentUrl = post.contentUrl; this.contentUrl = post.contentUrl;
@ -91,6 +91,7 @@ export class ScrapedPostDetails {
this.tags = post.tags.map((x) => TagDetails.fromScapedTag(x)); this.tags = post.tags.map((x) => TagDetails.fromScapedTag(x));
this.notes = post.notes; this.notes = post.notes;
this.resolution = post.resolution; this.resolution = post.resolution;
this.uploadMode = post.uploadMode;
} }
} }
@ -108,7 +109,7 @@ export class SimilarPostInfo {
constructor( constructor(
public readonly id: number, public readonly id: number,
public readonly percentage: number, public readonly percentage: number,
) {} ) { }
} }
export type PostUploadState = "uploading" | "uploaded" | "error"; export type PostUploadState = "uploading" | "uploaded" | "error";
@ -147,7 +148,7 @@ export class PostUploadCommandData {
constructor( constructor(
public readonly post: ScrapedPostDetails, public readonly post: ScrapedPostDetails,
public readonly selectedSite: SzuruSiteConfig, public readonly selectedSite: SzuruSiteConfig,
) {} ) { }
} }
export class SetPostUploadInfoData { export class SetPostUploadInfoData {
@ -155,7 +156,7 @@ export class SetPostUploadInfoData {
public instanceId: string, public instanceId: string,
public postId: string, public postId: string,
public info: PostUploadInfo, public info: PostUploadInfo,
) {} ) { }
} }
export class SetExactPostId { export class SetExactPostId {
@ -163,7 +164,7 @@ export class SetExactPostId {
public readonly instanceId: string, public readonly instanceId: string,
public readonly postId: string, public readonly postId: string,
public readonly exactPostId: number, public readonly exactPostId: number,
) {} ) { }
} }
export class PostUpdateCommandData { export class PostUpdateCommandData {
@ -171,14 +172,14 @@ export class PostUpdateCommandData {
public readonly postId: number, public readonly postId: number,
public readonly updateRequest: UpdatePostRequest, public readonly updateRequest: UpdatePostRequest,
public readonly selectedSite: SzuruSiteConfig, public readonly selectedSite: SzuruSiteConfig,
) {} ) { }
} }
export class FetchCommandData { export class FetchCommandData {
constructor( constructor(
public readonly url: string, public readonly url: string,
public readonly options: RequestInit | undefined = undefined, public readonly options: RequestInit | undefined = undefined,
) {} ) { }
} }
export class SzuruSiteConfig { export class SzuruSiteConfig {
@ -192,7 +193,7 @@ export class TagCategoryColor {
constructor( constructor(
public name: string, public name: string,
public color: string, public color: string,
) {} ) { }
} }
export const getDefaultTagCategories = () => [ export const getDefaultTagCategories = () => [

View File

@ -11,6 +11,8 @@ import Column from "primevue/column";
import { SzuruSiteConfig, TagCategoryColor, getDefaultTagCategories } from "~/models"; import { SzuruSiteConfig, TagCategoryColor, getDefaultTagCategories } from "~/models";
import SzurubooruApi from "~/api"; import SzurubooruApi from "~/api";
type StatusType = "success" | "error" | "quiet";
const statusText = ref(""); const statusText = ref("");
const statusType = ref("status-quiet"); const statusType = ref("status-quiet");
const versionInfo = "Version: " + (import.meta.env.VITE_SZ_VERSION ?? browser.runtime.getManifest().version); const versionInfo = "Version: " + (import.meta.env.VITE_SZ_VERSION ?? browser.runtime.getManifest().version);
@ -26,9 +28,7 @@ const selectedSite = computed(() => {
} }
}); });
type StatusType = "success" | "error" | "quiet"; const mode = useColorMode({ emitAuto: true });
let mode = useColorMode({ emitAuto: true });
async function testConnection() { async function testConnection() {
if ( if (
@ -221,8 +221,13 @@ wnd.szc_set_config_version = (v = 0) => (cfg.value.version = v);
<template v-if="selectedSite"> <template v-if="selectedSite">
<div class="field col-12"> <div class="field col-12">
<label>URL</label> <label>URL</label>
<input v-if="selectedSite" text="Szurubooru URL" type="text" name="domain" <input
v-model="selectedSite.domain" /> v-if="selectedSite"
text="Szurubooru URL"
type="text"
name="domain"
v-model="selectedSite.domain"
/>
</div> </div>
<div class="field col-12 md:col-6"> <div class="field col-12 md:col-6">
@ -251,10 +256,14 @@ wnd.szc_set_config_version = (v = 0) => (cfg.value.version = v);
<template v-if="field == 'color'"> <template v-if="field == 'color'">
<div class="color-preview"> <div class="color-preview">
<input type="text" class="color" v-model="data[field]" /> <input type="text" class="color" v-model="data[field]" />
<div class="preview background-preview" <div
:style="{ 'border-color': data[field], 'background-color': data[field] }"></div> class="preview background-preview"
<div class="preview text-preview" :style="{ 'border-color': data[field], color: data[field] }"> :style="{ 'border-color': data[field], 'background-color': data[field] }"
</div> ></div>
<div
class="preview text-preview"
:style="{ 'border-color': data[field], color: data[field] }"
></div>
</div> </div>
</template> </template>
@ -285,6 +294,16 @@ wnd.szc_set_config_version = (v = 0) => (cfg.value.version = v);
<!-- TODO: Category ignore list --> <!-- TODO: Category ignore list -->
</TabPanel> </TabPanel>
<!-- <TabPanel header="About">
<div class="grid">
<div v-for="engine in scraper.engines" :key="engine.name" class="col-12 md:col-4 flex flex-column">
<span class="font-bold">{{ engine.name }}</span>
<span>Features: {{ engine.features.join(", ") }}</span>
<span>Hosts: {{ engine.supportedHosts.join(", ") }}</span>
</div>
</div>
</TabPanel> -->
<!-- <!--
<TabPanel header="Developer"> <TabPanel header="Developer">
<div class="flex flex-column gap-2"> <div class="flex flex-column gap-2">

View File

@ -1,7 +1,7 @@
<script setup lang="ts"> <script setup lang="ts">
import { useDark } from "@vueuse/core"; import { useDark } from "@vueuse/core";
import { cloneDeep } from "lodash"; import { cloneDeep } from "lodash";
import { NeoScraper, ScrapeResults } from "neo-scraper"; import { ScrapeResults } from "neo-scraper";
import { getUrl, encodeTagName, getErrorMessage, getPostInfoSummary } from "~/utils"; import { getUrl, encodeTagName, getErrorMessage, getPostInfoSummary } from "~/utils";
import { import {
BrowserCommand, BrowserCommand,
@ -14,7 +14,6 @@ import {
SzuruSiteConfig, SzuruSiteConfig,
PoolDetails, PoolDetails,
} from "~/models"; } from "~/models";
import { ImageSearchResult } from "~/api/models";
import { isMobile } from "~/env"; import { isMobile } from "~/env";
import { DeepReadonly } from "vue"; import { DeepReadonly } from "vue";
import { cfg, usePopupStore } from "~/stores"; import { cfg, usePopupStore } from "~/stores";
@ -99,14 +98,11 @@ async function grabPost() {
// Clear current posts array // Clear current posts array
pop.posts.splice(0); pop.posts.splice(0);
const scraper = new NeoScraper();
for (const result of res.results) { for (const result of res.results) {
const engine = scraper.engines.find((x) => x.name == result.engine);
for (const i in result.posts) { for (const i in result.posts) {
const vm = new ScrapedPostDetails(result.posts[i]); const vm = new ScrapedPostDetails(result.posts[i]);
vm.name = `[${result.engine}] Post ${parseInt(i) + 1}`; // parseInt() is required! const name = result.posts[i].name ?? `Post ${parseInt(i) + 1}`; // parseInt() is required!
vm.name = `[${result.engine}] ${name}`;
if (!cfg.value.addAllParsedTags) { if (!cfg.value.addAllParsedTags) {
vm.tags.splice(0); vm.tags.splice(0);
@ -125,10 +121,6 @@ async function grabPost() {
vm.instanceSpecificData[site.id] = {}; vm.instanceSpecificData[site.id] = {};
} }
if (engine) {
vm.uploadMode = engine.uploadMode;
}
pop.posts.push(vm); pop.posts.push(vm);
} }
} }
@ -145,7 +137,7 @@ async function grabPost() {
} }
if (cfg.value.fetchPostInfo) { if (cfg.value.fetchPostInfo) {
await fetchPostInfo(); await fetchPostsInfo();
} }
// Delayed call to findSimilar, as fetchPostInfo might change the post's contentUrl. // Delayed call to findSimilar, as fetchPostInfo might change the post's contentUrl.
@ -168,7 +160,7 @@ async function upload() {
} }
try { try {
const post: ScrapedPostDetails = cloneDeep(pop.selectedPost); const post: ScrapedPostDetails = cloneDeep(pop.selectedPost)!;
// uploadMode "content" requires a content token to work. So ensure it is set. // uploadMode "content" requires a content token to work. So ensure it is set.
if (post.uploadMode == "content") { if (post.uploadMode == "content") {
@ -291,10 +283,10 @@ async function findSimilar(post: ScrapedPostDetails | undefined) {
isSearchingForSimilarPosts.value++; isSearchingForSimilarPosts.value++;
try {
await ensurePostHasContentToken(post); await ensurePostHasContentToken(post);
try { const res = await selectedInstance.reverseSearchToken(instanceSpecificData.contentToken!);
const res = await selectedInstance.reverseSearchToken(instanceSpecificData.contentToken);
instanceSpecificData.reverseSearchResult = { instanceSpecificData.reverseSearchResult = {
exactPostId: res.exactPost?.id, exactPostId: res.exactPost?.id,
@ -331,33 +323,58 @@ async function loadTagCounts() {
} }
} }
async function fetchPostInfo() { async function updatePostWithRemoteInfo(post: ScrapedPostDetails, contentUrl: string) {
for (const post of pop.posts) {
if (!post.contentSize || post.extraContentUrl) {
// We are missing cookies/etc which means that the request might fail. // We are missing cookies/etc which means that the request might fail.
// If you want access to the cookies/session/etc then you need to execute this code inside the content script. // If you want access to the cookies/session/etc then you need to execute this code inside the content script.
try { try {
const contentUrl = post.extraContentUrl ?? post.contentUrl; // This request follows redirects.
// TODO: Check whether we need to call this in the background page, or if it is fine to call it from the popup.
const res = await fetch(contentUrl, { method: "HEAD" }); const res = await fetch(contentUrl, { method: "HEAD" });
const size = res.headers.get("Content-Length"); const size = res.headers.get("Content-Length");
const type = res.headers.get("Content-Type"); const type = res.headers.get("Content-Type");
if (size) post.contentSize = parseInt(size);
if (type) { if (type) {
if (type.indexOf("text/html") != -1) {
// If we get a HTML page back it usually means that the request failed.
// Not all sites (e.g. e-hentai) reply with a 403.
throw new Error(
"Received a text/html content type. This probably means that we don't have permission to access the resource.",
);
}
const [_main, sub] = type.split("/"); const [_main, sub] = type.split("/");
if (sub) post.contentSubType = sub.toUpperCase(); if (sub) post.contentSubType = sub.toUpperCase();
} }
// Resolve extraContentUrl redirects. // This should be after the type check, because the type check also checks whether the response we get is valid/unsable.
if (post.extraContentUrl && res.url != post.contentUrl) { if (size) post.contentSize = parseInt(size);
// Update url if it changes. The url can change when:
// a. the extraContentUrl was successfully loaded
// b. because of redirects
if (res.url != post.contentUrl) {
// The developer needs to make sure that the remote server can load this URL if `post.uploadMode == 'url'`!
console.log(`Updating post.contentUrl to '${res.url}'`);
post.contentUrl = res.url; post.contentUrl = res.url;
} }
return true;
} catch (ex) { } catch (ex) {
// TODO: Maybe display an error in the UI.
console.error(ex); console.error(ex);
return false;
}
}
async function fetchPostsInfo() {
for (const post of pop.posts) {
if (!post.contentSize || post.extraContentUrl) {
let ok = false;
if (post.extraContentUrl) {
// Prioritize info from extraContentUrl.
ok = await updatePostWithRemoteInfo(post, post.extraContentUrl);
}
if (!ok) {
// Fall back to normal contentUrl if we can't get the data from the extraContentUrl.
await updatePostWithRemoteInfo(post, post.contentUrl);
} }
} }
} }
@ -369,7 +386,8 @@ function getUpdatedTagsText(count: number) {
} }
function onResolutionLoaded(res: any) { function onResolutionLoaded(res: any) {
if (pop.selectedPost && !pop.selectedPost.resolution) { // This no longer checks for `!pop.selectedPost.resolution` because the resolution can change if the contentUrl is updated.
if (pop.selectedPost) {
pop.selectedPost.resolution = res; pop.selectedPost.resolution = res;
} }
} }