Nes
/
neo-scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix twitter engine for new site design

pull/9/head
Ben Klein 2021-10-03 02:12:44 -04:00
parent 352ba505e1
commit 4fe95f0480
1 changed files with 25 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
src/engines/Twitter.ts
View File

@ -1,47 +1,45 @@
import { ScrapeEngine, ScrapeResult, ScrapedPost } from "../ScrapeEngine";
import { guessContentType } from "../Utility";
export function getOriginalImageUrl(imageUrl: string) {
const x = imageUrl.replace(/:large|:med|:small|:orig$/, ""); // Remove `:large`, `:small` etc from url
return x + ":orig"; // Add :orig to url (to get original image)
export function getOriginalImageUrl(image: HTMLImageElement) {
// if the link includes a resolution, change it to orig:
// https://pbs.twimg.com/media/stuff?format=jpg&name=900x900
// -> https://pbs.twimg.com/media/stuff?format=jpg&name=orig
let img_url = new URL((image as HTMLImageElement).src);
if (img_url.searchParams.has('name') && img_url.searchParams.get('name') != 'orig') {
img_url.searchParams.set('name', 'orig');
}
return img_url;
}
export default class Twitter implements ScrapeEngine {
name = "twitter";
canImport(url: Location): boolean {
return url.host == "twitter.com";
return url.host == "twitter.com" || url.host == "mobile.twitter.com";
}
scrapeDocument(document: Document): ScrapeResult {
let result = new ScrapeResult(this.name);
// first look for a focused image on desktop:
const center_img = document.querySelector("div[role=dialog] div[role=dialog] div[aria-label=Image] img");
if (center_img) {
let post = new ScrapedPost();
post.pageUrl = document.location.href;
// Set image url
// When clicked on post (not fullscreen, with post text etc)
const overlayElement = document.querySelector("#permalink-overlay-dialog");
if (overlayElement) {
const cardImageElement = overlayElement.querySelector("div[data-element-context='platform_photo_card'] > img");
if (cardImageElement) {
post.contentUrl = getOriginalImageUrl((cardImageElement as HTMLImageElement).src);
}
}
// When clicked on image (fullscreen)
const mediaImageElements = document.querySelectorAll("div.Gallery-media > img.media-image");
if (mediaImageElements.length > 0) {
post.contentUrl = getOriginalImageUrl((mediaImageElements[0] as HTMLImageElement).src);
}
if (post.contentUrl == undefined) {
return result;
}
// Set content type
post.contentType = guessContentType(post.contentUrl);
post.contentUrl = getOriginalImageUrl(center_img as HTMLImageElement).href;
result.tryAddPost(post);
}
// look for images within a tweet:
const imgs = [...document.querySelectorAll("article a[role='link'] div[data-testid='tweetPhoto'] img")];
for (const img of imgs) {
let post = new ScrapedPost();
post.pageUrl = document.location.href;
post.contentUrl = getOriginalImageUrl(img as HTMLImageElement).href;
result.tryAddPost(post);
}
return result;
}