neo-scraper/src/Utility.ts

import TurndownService from "turndown";
import { ContentType, ScrapedNote, ScrapedPost } from "./ScrapeEngine.js";

// https://github.com/sindresorhus/video-extensions/blob/main/video-extensions.json
export const videoExtensions = [
  "3g2",
  "3gp",
  "aaf",
  "asf",
  "avchd",
  "avi",
  "drc",
  "flv",
  "m2v",
  "m4p",
  "m4v",
  "mkv",
  "mng",
  "mov",
  "mp2",
  "mp4",
  "mpe",
  "mpeg",
  "mpg",
  "mpv",
  "mxf",
  "nsv",
  "ogg",
  "ogv",
  "qt",
  "rm",
  "rmvb",
  "roq",
  "svi",
  "vob",
  "webm",
  "wmv",
  "yuv",
];

export function getUrlExtension(url: string) {
  // https://stackoverflow.com/a/47767860
  return url.split(/[#?]/)[0].split(".").pop()!.trim();
}

/**
 * Returns "image" when the guessed type is NOT a video.
 * This means that it will also return "image" when the content type
 * could not be guessed!
 * @param url
 * @returns "image" | "video"
 */
export function guessContentType(url: string | undefined): ContentType {
  if (url != undefined) {
    const ext = getUrlExtension(url);
    if (videoExtensions.indexOf(ext) != -1) return "video";
  }
  return "image";
}

export function htmlNoteToMarkdownNote(text: string) {
  return new TurndownService().turndown(text);
}

export function validateNote(note: ScrapedNote) {
  if (note.text.length == 0) {
    console.log("[validateNote] No text.");
    return false;
  }

  for (const poly of note.polygon) {
    if (poly.some((x) => x > 1 || x < 0)) {
      console.log("[validateNote] Polygon out of range.");
      return false;
    }
  }

  return true;
}

export function xywhToNormalizedPolygon(x: number, y: number, w: number, h: number, resolution: [number, number]) {
  // Normalize our numbers from 0 to 1
  x = x / resolution[0];
  y = y / resolution[1];
  w = w / resolution[0];
  h = h / resolution[1];

  return [
    // Top left
    [x, y],
    // Top right
    [x + w, y],
    // Bottom right
    [x + w, y + h],
    // Bottom left
    [x, y + h],
  ];
}

export function createNoteFromDanbooruArticle(post: ScrapedPost, el: HTMLElement): ScrapedNote | undefined {
  function logFail(str: string) {
    console.log("[createNoteFromDanbooruArticle] Can't create note. " + str);
  }

  function getInt(map: DOMStringMap, key: string) {
    const value = map[key];
    if (!value) {
      logFail(`Key '${key}' not found in data.`);
      return undefined;
    }
    const num = parseInt(value);
    if (isNaN(num)) {
      logFail(`parseInt on '${value}' returned NaN.`);
    }
    return num;
  }

  if (!post.resolution) {
    logFail("Post resolution is undefined.");
    return undefined;
  }

  // eslint-disable-next-line @typescript-eslint/no-explicit-any
  const data = (<any>el).dataset as DOMStringMap;

  if (!data) {
    logFail("Data is undefined.");
    return undefined;
  }

  let text = data["body"];
  if (text) {
    text = htmlNoteToMarkdownNote(text);
  }

  const x = getInt(data, "x");
  const y = getInt(data, "y");
  const w = getInt(data, "width");
  const h = getInt(data, "height");

  if (!text || !x || !y || !w || !h) {
    logFail("A required data field could not be parsed.");
    return undefined;
  }

  const polygon = xywhToNormalizedPolygon(x, y, w, h, post.resolution);
  const note = new ScrapedNote(text, polygon);

  if (validateNote(note)) {
    return note;
  } else {
    return undefined;
  }
}

/**
 * Creates notes by reading ".note-box" and ".note-body" elements.
 * @param document
 * @param boxSize
 * @returns
 */
export function createNotesFromMoebooruBoxes(document: Document, boxSize: [number, number]) {
  const notes: ScrapedNote[] = [];
  const noteBoxes = Array.from(document.querySelectorAll(".note-box")).map((x) => x as HTMLLIElement);
  const noteBodies = Array.from(document.querySelectorAll(".note-body")).map((x) => x as HTMLLIElement);

  if (noteBoxes.length != noteBodies.length) {
    console.log("[createNotesFromMoebooruBoxes] noteBoxes.length != noteBodies.length");
  } else {
    for (let i = 0; i < noteBoxes.length; i++) {
      const noteBox = noteBoxes[i];
      const noteBody = noteBodies[i];

      const text = htmlNoteToMarkdownNote(noteBody.innerHTML);

      const w = parseInt(noteBox.style.width);
      const h = parseInt(noteBox.style.height);
      const y = parseInt(noteBox.style.top);
      const x = parseInt(noteBox.style.left);

      const polygon = xywhToNormalizedPolygon(x, y, w, h, boxSize);
      const note = new ScrapedNote(text, polygon);

      if (validateNote(note)) {
        notes.push(note);
      }
    }
  }
  return notes;
}

export function parseResolutionString(str: string | undefined | null): [number, number] | undefined {
  if (!str) return undefined;

  // Example string: 1600x2200
  const res = str
    .split("x") // Split on 'x' character
    .map((v) => v.replace(/\D/g, "")) // Remove all non-digits
    .map((v) => parseInt(v)) // Parse ints
    .filter(Number); // Filter NaN. Also removes the number 0, but that's fine because the resolution can't be 0.

  if (res.length == 2) {
    // Kinda hacky, but this is currently the correct way to convert an array to a tuple.
    return [res[0], res[1]];
  } else {
    console.log(`[parseResolutionString] Couldn't parse resolution from '${str}'.`);
  }
}