Update neo-scraper to 0.9.0 + post.uploadMode changes

pull/55/head v0.12.1
neobooru 2024-06-25 23:46:05 +02:00
parent 9843f0b641
commit eae1b3949f
6 changed files with 107 additions and 69 deletions

View File

@ -70,7 +70,7 @@ jobs:
runs-on: ubuntu-20.04
# Doesn't really need the Chrome build, but we can reuse its cache.
needs: build_chrome
# Only run on master branch AND if tagging or manually dispatching.
# Only run (on master branch && if tagged) or (manually dispatching).
if: startsWith(github.ref, 'refs/tags/v') || github.event.inputs.publish_amo_listed == true
steps:
- name: Checkout

View File

@ -32,7 +32,7 @@
"deepmerge": "^4.3.1",
"lodash": "^4.17.21",
"markdown-it": "^14.0.0",
"neo-scraper": "^0.8.0",
"neo-scraper": "^0.9.0",
"normalize.css": "^8.0.1",
"pinia": "^2.1.7",
"primeflex": "^3.3.1",

View File

@ -24,8 +24,8 @@ importers:
specifier: ^14.0.0
version: 14.0.0
neo-scraper:
specifier: ^0.8.0
version: 0.8.0
specifier: ^0.9.0
version: 0.9.0
normalize.css:
specifier: ^8.0.1
version: 8.0.1
@ -2248,8 +2248,8 @@ packages:
resolution: {integrity: sha512-zIdGUrPRFTUELUvr3Gmc7KZ2Sw/h1PiVM0Af/oHB6zgnV1ikqSfRk+TOufi79aHYCW3NiOXmr1BP5nWbzojLaA==}
hasBin: true
neo-scraper@0.8.0:
resolution: {integrity: sha512-E01oBGuHx9MJOrerkBJpiV0cAHO0uZi28xly4SbELmajf8LQgg1HOH1pq30krBXj5J1px5oRr+3WSO1+1H+sHA==}
neo-scraper@0.9.0:
resolution: {integrity: sha512-KpDrkNSO3pYwYewFWLjrY/OZzLDWh+6Ge021JBExVxA0ZAr67MAKzaNkDXVptW+laz1eZ9Cnl5rvhQZcn+Y0Vw==}
nice-try@1.0.5:
resolution: {integrity: sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==}
@ -5725,7 +5725,7 @@ snapshots:
ncp@2.0.0:
optional: true
neo-scraper@0.8.0:
neo-scraper@0.9.0:
dependencies:
js-cookie: 3.0.5
turndown: 7.1.2

View File

@ -73,11 +73,11 @@ export class ScrapedPostDetails {
contentType: ContentType;
contentSubType: string | undefined;
rating: BooruTypes.SafetyRating;
source = "";
source;
uploadMode: UploadMode;
referrer?: string;
resolution?: [number, number];
instanceSpecificData: MappedInstanceSpecificData = {};
uploadMode: UploadMode = "url";
constructor(post: ScrapedPost) {
this.contentUrl = post.contentUrl;
@ -91,6 +91,7 @@ export class ScrapedPostDetails {
this.tags = post.tags.map((x) => TagDetails.fromScapedTag(x));
this.notes = post.notes;
this.resolution = post.resolution;
this.uploadMode = post.uploadMode;
}
}

View File

@ -11,6 +11,8 @@ import Column from "primevue/column";
import { SzuruSiteConfig, TagCategoryColor, getDefaultTagCategories } from "~/models";
import SzurubooruApi from "~/api";
type StatusType = "success" | "error" | "quiet";
const statusText = ref("");
const statusType = ref("status-quiet");
const versionInfo = "Version: " + (import.meta.env.VITE_SZ_VERSION ?? browser.runtime.getManifest().version);
@ -26,9 +28,7 @@ const selectedSite = computed(() => {
}
});
type StatusType = "success" | "error" | "quiet";
let mode = useColorMode({ emitAuto: true });
const mode = useColorMode({ emitAuto: true });
async function testConnection() {
if (
@ -221,8 +221,13 @@ wnd.szc_set_config_version = (v = 0) => (cfg.value.version = v);
<template v-if="selectedSite">
<div class="field col-12">
<label>URL</label>
<input v-if="selectedSite" text="Szurubooru URL" type="text" name="domain"
v-model="selectedSite.domain" />
<input
v-if="selectedSite"
text="Szurubooru URL"
type="text"
name="domain"
v-model="selectedSite.domain"
/>
</div>
<div class="field col-12 md:col-6">
@ -251,10 +256,14 @@ wnd.szc_set_config_version = (v = 0) => (cfg.value.version = v);
<template v-if="field == 'color'">
<div class="color-preview">
<input type="text" class="color" v-model="data[field]" />
<div class="preview background-preview"
:style="{ 'border-color': data[field], 'background-color': data[field] }"></div>
<div class="preview text-preview" :style="{ 'border-color': data[field], color: data[field] }">
</div>
<div
class="preview background-preview"
:style="{ 'border-color': data[field], 'background-color': data[field] }"
></div>
<div
class="preview text-preview"
:style="{ 'border-color': data[field], color: data[field] }"
></div>
</div>
</template>
@ -285,6 +294,16 @@ wnd.szc_set_config_version = (v = 0) => (cfg.value.version = v);
<!-- TODO: Category ignore list -->
</TabPanel>
<!-- <TabPanel header="About">
<div class="grid">
<div v-for="engine in scraper.engines" :key="engine.name" class="col-12 md:col-4 flex flex-column">
<span class="font-bold">{{ engine.name }}</span>
<span>Features: {{ engine.features.join(", ") }}</span>
<span>Hosts: {{ engine.supportedHosts.join(", ") }}</span>
</div>
</div>
</TabPanel> -->
<!--
<TabPanel header="Developer">
<div class="flex flex-column gap-2">

View File

@ -1,7 +1,7 @@
<script setup lang="ts">
import { useDark } from "@vueuse/core";
import { cloneDeep } from "lodash";
import { NeoScraper, ScrapeResults } from "neo-scraper";
import { ScrapeResults } from "neo-scraper";
import { getUrl, encodeTagName, getErrorMessage, getPostInfoSummary } from "~/utils";
import {
BrowserCommand,
@ -14,7 +14,6 @@ import {
SzuruSiteConfig,
PoolDetails,
} from "~/models";
import { ImageSearchResult } from "~/api/models";
import { isMobile } from "~/env";
import { DeepReadonly } from "vue";
import { cfg, usePopupStore } from "~/stores";
@ -99,14 +98,11 @@ async function grabPost() {
// Clear current posts array
pop.posts.splice(0);
const scraper = new NeoScraper();
for (const result of res.results) {
const engine = scraper.engines.find((x) => x.name == result.engine);
for (const i in result.posts) {
const vm = new ScrapedPostDetails(result.posts[i]);
vm.name = `[${result.engine}] Post ${parseInt(i) + 1}`; // parseInt() is required!
const name = result.posts[i].name ?? `Post ${parseInt(i) + 1}`; // parseInt() is required!
vm.name = `[${result.engine}] ${name}`;
if (!cfg.value.addAllParsedTags) {
vm.tags.splice(0);
@ -125,10 +121,6 @@ async function grabPost() {
vm.instanceSpecificData[site.id] = {};
}
if (engine) {
vm.uploadMode = engine.uploadMode;
}
pop.posts.push(vm);
}
}
@ -145,7 +137,7 @@ async function grabPost() {
}
if (cfg.value.fetchPostInfo) {
await fetchPostInfo();
await fetchPostsInfo();
}
// Delayed call to findSimilar, as fetchPostInfo might change the post's contentUrl.
@ -168,7 +160,7 @@ async function upload() {
}
try {
const post: ScrapedPostDetails = cloneDeep(pop.selectedPost);
const post: ScrapedPostDetails = cloneDeep(pop.selectedPost)!;
// uploadMode "content" requires a content token to work. So ensure it is set.
if (post.uploadMode == "content") {
@ -291,10 +283,10 @@ async function findSimilar(post: ScrapedPostDetails | undefined) {
isSearchingForSimilarPosts.value++;
try {
await ensurePostHasContentToken(post);
try {
const res = await selectedInstance.reverseSearchToken(instanceSpecificData.contentToken);
const res = await selectedInstance.reverseSearchToken(instanceSpecificData.contentToken!);
instanceSpecificData.reverseSearchResult = {
exactPostId: res.exactPost?.id,
@ -331,33 +323,58 @@ async function loadTagCounts() {
}
}
async function fetchPostInfo() {
for (const post of pop.posts) {
if (!post.contentSize || post.extraContentUrl) {
async function updatePostWithRemoteInfo(post: ScrapedPostDetails, contentUrl: string) {
// We are missing cookies/etc which means that the request might fail.
// If you want access to the cookies/session/etc then you need to execute this code inside the content script.
try {
const contentUrl = post.extraContentUrl ?? post.contentUrl;
// TODO: Check whether we need to call this in the background page, or if it is fine to call it from the popup.
// This request follows redirects.
const res = await fetch(contentUrl, { method: "HEAD" });
const size = res.headers.get("Content-Length");
const type = res.headers.get("Content-Type");
if (size) post.contentSize = parseInt(size);
if (type) {
if (type.indexOf("text/html") != -1) {
// If we get a HTML page back it usually means that the request failed.
// Not all sites (e.g. e-hentai) reply with a 403.
throw new Error(
"Received a text/html content type. This probably means that we don't have permission to access the resource.",
);
}
const [_main, sub] = type.split("/");
if (sub) post.contentSubType = sub.toUpperCase();
}
// Resolve extraContentUrl redirects.
if (post.extraContentUrl && res.url != post.contentUrl) {
// This should be after the type check, because the type check also checks whether the response we get is valid/unsable.
if (size) post.contentSize = parseInt(size);
// Update url if it changes. The url can change when:
// a. the extraContentUrl was successfully loaded
// b. because of redirects
if (res.url != post.contentUrl) {
// The developer needs to make sure that the remote server can load this URL if `post.uploadMode == 'url'`!
console.log(`Updating post.contentUrl to '${res.url}'`);
post.contentUrl = res.url;
}
return true;
} catch (ex) {
// TODO: Maybe display an error in the UI.
console.error(ex);
return false;
}
}
async function fetchPostsInfo() {
for (const post of pop.posts) {
if (!post.contentSize || post.extraContentUrl) {
let ok = false;
if (post.extraContentUrl) {
// Prioritize info from extraContentUrl.
ok = await updatePostWithRemoteInfo(post, post.extraContentUrl);
}
if (!ok) {
// Fall back to normal contentUrl if we can't get the data from the extraContentUrl.
await updatePostWithRemoteInfo(post, post.contentUrl);
}
}
}
@ -369,7 +386,8 @@ function getUpdatedTagsText(count: number) {
}
function onResolutionLoaded(res: any) {
if (pop.selectedPost && !pop.selectedPost.resolution) {
// This no longer checks for `!pop.selectedPost.resolution` because the resolution can change if the contentUrl is updated.
if (pop.selectedPost) {
pop.selectedPost.resolution = res;
}
}