delete extractHtmlMeta (#5478)
This commit is contained in:
noriaki watanabe
@@ -1,47 +0,0 @@
|
||||
export const exampleComHtml = `<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Example Domain</title>
|
||||
<meta name="description" content="An example website">
|
||||
|
||||
<meta charset="utf-8" />
|
||||
<meta http-equiv="Content-type" content="text/html; charset=utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<style type="text/css">
|
||||
body {
|
||||
background-color: #f0f0f2;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
font-family: -apple-system, system-ui, BlinkMacSystemFont, "Segoe UI", "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
|
||||
|
||||
}
|
||||
div {
|
||||
width: 600px;
|
||||
margin: 5em auto;
|
||||
padding: 2em;
|
||||
background-color: #fdfdff;
|
||||
border-radius: 0.5em;
|
||||
box-shadow: 2px 3px 7px 2px rgba(0,0,0,0.02);
|
||||
}
|
||||
a:link, a:visited {
|
||||
color: #38488f;
|
||||
text-decoration: none;
|
||||
}
|
||||
@media (max-width: 700px) {
|
||||
div {
|
||||
margin: 0 auto;
|
||||
width: auto;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<div>
|
||||
<h1>Example Domain</h1>
|
||||
<p>This domain is for use in illustrative examples in documents. You may use this
|
||||
domain in literature without prior coordination or asking for permission.</p>
|
||||
<p><a href="https://www.iana.org/domains/example">More information...</a></p>
|
||||
</div>
|
||||
</body>
|
||||
</html>`
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -1,134 +0,0 @@
|
||||
import {extractHtmlMeta} from '../../src/lib/link-meta/html'
|
||||
import {exampleComHtml} from './__mocks__/exampleComHtml'
|
||||
import {youtubeHTML} from './__mocks__/youtubeHtml'
|
||||
import {tiktokHtml} from './__mocks__/tiktokHtml'
|
||||
import {youtubeChannelHtml} from './__mocks__/youtubeChannelHtml'
|
||||
|
||||
describe('extractHtmlMeta', () => {
|
||||
const cases = [
|
||||
['', {}],
|
||||
['nothing', {}],
|
||||
['<title>title</title>', {title: 'title'}],
|
||||
['<title> aSd!@#AC </title>', {title: 'aSd!@#AC'}],
|
||||
['<title>\n title\n </title>', {title: 'title'}],
|
||||
['<meta name="title" content="meta title">', {title: 'meta title'}],
|
||||
[
|
||||
'<meta name="description" content="meta description">',
|
||||
{description: 'meta description'},
|
||||
],
|
||||
['<meta property="og:title" content="og title">', {title: 'og title'}],
|
||||
[
|
||||
'<meta property="og:description" content="og description">',
|
||||
{description: 'og description'},
|
||||
],
|
||||
[
|
||||
'<meta property="og:image" content="https://ogimage.com/foo.png">',
|
||||
{image: 'https://ogimage.com/foo.png'},
|
||||
],
|
||||
[
|
||||
'<meta property="twitter:title" content="twitter title">',
|
||||
{title: 'twitter title'},
|
||||
],
|
||||
[
|
||||
'<meta property="twitter:description" content="twitter description">',
|
||||
{description: 'twitter description'},
|
||||
],
|
||||
[
|
||||
'<meta property="twitter:image" content="https://twitterimage.com/foo.png">',
|
||||
{image: 'https://twitterimage.com/foo.png'},
|
||||
],
|
||||
['<meta\n name="title"\n content="meta title"\n>', {title: 'meta title'}],
|
||||
]
|
||||
|
||||
it.each(cases)(
|
||||
'given the html tag %p, returns %p',
|
||||
// @ts-ignore not worth fixing -prf
|
||||
(input, expectedResult) => {
|
||||
const output = extractHtmlMeta({html: input as string, hostname: ''})
|
||||
expect(output).toEqual(expectedResult)
|
||||
},
|
||||
)
|
||||
|
||||
it('extracts title and description from a generic HTML page', () => {
|
||||
const input = exampleComHtml
|
||||
const expectedOutput = {
|
||||
title: 'Example Domain',
|
||||
description: 'An example website',
|
||||
}
|
||||
const output = extractHtmlMeta({html: input, hostname: 'example.com'})
|
||||
expect(output).toEqual(expectedOutput)
|
||||
})
|
||||
|
||||
it('extracts title and description from a Tiktok HTML page', () => {
|
||||
const input = tiktokHtml
|
||||
const expectedOutput = {
|
||||
title:
|
||||
'Coca-Cola and Mentos! Super Reaction! #cocacola #mentos #reaction #bal... | TikTok',
|
||||
description:
|
||||
'5.5M Likes, 20.8K Comments. TikTok video from Power Vision Tests (@_powervision_): "Coca-Cola and Mentos! Super Reaction! #cocacola #mentos #reaction #balloon #sciencemoment #scienceexperiment #experiment #test #amazing #pvexp". оригинальный звук - Power Vision Tests.',
|
||||
}
|
||||
const output = extractHtmlMeta({html: input, hostname: 'tiktok.com'})
|
||||
expect(output).toEqual(expectedOutput)
|
||||
})
|
||||
|
||||
it('extracts title and description from a generic youtube page', () => {
|
||||
const input = youtubeHTML
|
||||
const expectedOutput = {
|
||||
title: 'HD Video (1080p) with Relaxing Music of Native American Shamans',
|
||||
description:
|
||||
'Stunning HD Video ( 1080p ) of Patagonian Nature with Relaxing Native American Shamanic Music. HD footage used from ',
|
||||
image: 'https://i.ytimg.com/vi/x6UITRjhijI/sddefault.jpg',
|
||||
}
|
||||
const output = extractHtmlMeta({html: input, hostname: 'youtube.com'})
|
||||
expect(output).toEqual(expectedOutput)
|
||||
})
|
||||
|
||||
it('extracts avatar from a youtube channel', () => {
|
||||
const input = youtubeChannelHtml
|
||||
const expectedOutput = {
|
||||
title: 'penguinz0',
|
||||
description:
|
||||
'Clips channel: https://www.youtube.com/channel/UC4EQHfzIbkL_Skit_iKt1aA\n\nTwitter: https://twitter.com/MoistCr1TiKaL\n\nInstagram: https://www.instagram.com/bigmoistcr1tikal/?hl=en\n\nTwitch: https://www.twitch.tv/moistcr1tikal\n\nSnapchat: Hugecharles\n\nTik Tok: Hugecharles\n\nI don't have any other public accounts.',
|
||||
image:
|
||||
'https://yt3.googleusercontent.com/ytc/AL5GRJWOhJOuUC6C2b7gP-5D2q6ypXbcOOckyAE1En4RUQ=s176-c-k-c0x00ffffff-no-rj',
|
||||
}
|
||||
const output = extractHtmlMeta({html: input, hostname: 'youtube.com'})
|
||||
expect(output).toEqual(expectedOutput)
|
||||
})
|
||||
|
||||
it('extracts username from the url a twitter profile page', () => {
|
||||
const expectedOutput = {
|
||||
title: '@bluesky on Twitter',
|
||||
}
|
||||
const output = extractHtmlMeta({
|
||||
html: '',
|
||||
hostname: 'twitter.com',
|
||||
pathname: '/bluesky',
|
||||
})
|
||||
expect(output).toEqual(expectedOutput)
|
||||
})
|
||||
|
||||
it('extracts username from the url a tweet', () => {
|
||||
const expectedOutput = {
|
||||
title: 'Tweet by @bluesky',
|
||||
}
|
||||
const output = extractHtmlMeta({
|
||||
html: '',
|
||||
hostname: 'twitter.com',
|
||||
pathname: '/bluesky/status/1582437529969917953',
|
||||
})
|
||||
expect(output).toEqual(expectedOutput)
|
||||
})
|
||||
|
||||
it("does not extract username from the url when it's not a tweet or profile page", () => {
|
||||
const expectedOutput = {
|
||||
title: 'Twitter',
|
||||
}
|
||||
const output = extractHtmlMeta({
|
||||
html: '',
|
||||
hostname: 'twitter.com',
|
||||
pathname: '/i/articles/follows/-1675653703?time_window=24',
|
||||
})
|
||||
expect(output).toEqual(expectedOutput)
|
||||
})
|
||||
})
|
||||
@@ -1,71 +0,0 @@
|
||||
import {extractTwitterMeta} from './twitter'
|
||||
import {extractYoutubeMeta} from './youtube'
|
||||
|
||||
interface ExtractHtmlMetaInput {
|
||||
html: string
|
||||
hostname?: string
|
||||
pathname?: string
|
||||
}
|
||||
|
||||
export const extractHtmlMeta = ({
|
||||
html,
|
||||
hostname,
|
||||
pathname,
|
||||
}: ExtractHtmlMetaInput): Record<string, string> => {
|
||||
const htmlTitleRegex = /<title.*>([^<]+)<\/title>/i
|
||||
|
||||
let res: Record<string, string> = {}
|
||||
|
||||
const match = htmlTitleRegex.exec(html)
|
||||
|
||||
if (match) {
|
||||
res.title = match[1].trim()
|
||||
}
|
||||
|
||||
let metaMatch
|
||||
let propMatch
|
||||
const metaRe = /<meta[\s]([^>]+)>/gis
|
||||
while ((metaMatch = metaRe.exec(html))) {
|
||||
let propName
|
||||
let propValue
|
||||
const propRe = /(name|property|content)="([^"]+)"/gis
|
||||
while ((propMatch = propRe.exec(metaMatch[1]))) {
|
||||
if (propMatch[1] === 'content') {
|
||||
propValue = propMatch[2]
|
||||
} else {
|
||||
propName = propMatch[2]
|
||||
}
|
||||
}
|
||||
if (!propName || !propValue) {
|
||||
continue
|
||||
}
|
||||
switch (propName?.trim()) {
|
||||
case 'title':
|
||||
case 'og:title':
|
||||
case 'twitter:title':
|
||||
res.title = propValue?.trim()
|
||||
break
|
||||
case 'description':
|
||||
case 'og:description':
|
||||
case 'twitter:description':
|
||||
res.description = propValue?.trim()
|
||||
break
|
||||
case 'og:image':
|
||||
case 'twitter:image':
|
||||
res.image = propValue?.trim()
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
const isYoutubeUrl =
|
||||
hostname?.includes('youtube.') || hostname?.includes('youtu.be')
|
||||
const isTwitterUrl = hostname?.includes('twitter.')
|
||||
// Workaround for some websites not having a title or description in the meta tags in the initial serve
|
||||
if (isYoutubeUrl) {
|
||||
res = {...res, ...extractYoutubeMeta(html)}
|
||||
} else if (isTwitterUrl && pathname) {
|
||||
res = {...extractTwitterMeta({pathname})}
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
export const extractTwitterMeta = ({
|
||||
pathname,
|
||||
}: {
|
||||
pathname: string
|
||||
}): Record<string, string> => {
|
||||
const res = {title: 'Twitter'}
|
||||
const parsedPathname = pathname.split('/')
|
||||
if (parsedPathname.length <= 1 || parsedPathname[1].length <= 1) {
|
||||
// Excluding one letter usernames as they're reserved by twitter for things like cases like twitter.com/i/articles/follows/-1675653703
|
||||
return res
|
||||
}
|
||||
const username = parsedPathname?.[1]
|
||||
const isUserProfile = parsedPathname?.length === 2
|
||||
|
||||
res.title = isUserProfile
|
||||
? `@${username} on Twitter`
|
||||
: `Tweet by @${username}`
|
||||
|
||||
return res
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
export const extractYoutubeMeta = (html: string): Record<string, string> => {
|
||||
const res: Record<string, string> = {}
|
||||
const youtubeTitleRegex = /"videoDetails":.*"title":"([^"]*)"/i
|
||||
const youtubeDescriptionRegex =
|
||||
/"videoDetails":.*"shortDescription":"([^"]*)"/i
|
||||
const youtubeThumbnailRegex = /"videoDetails":.*"url":"(.*)(default\.jpg)/i
|
||||
const youtubeAvatarRegex =
|
||||
/"avatar":{"thumbnails":\[{.*?url.*?url.*?url":"([^"]*)"/i
|
||||
const youtubeTitleMatch = youtubeTitleRegex.exec(html)
|
||||
const youtubeDescriptionMatch = youtubeDescriptionRegex.exec(html)
|
||||
const youtubeThumbnailMatch = youtubeThumbnailRegex.exec(html)
|
||||
const youtubeAvatarMatch = youtubeAvatarRegex.exec(html)
|
||||
|
||||
if (youtubeTitleMatch && youtubeTitleMatch.length >= 1) {
|
||||
res.title = decodeURI(youtubeTitleMatch[1])
|
||||
}
|
||||
if (youtubeDescriptionMatch && youtubeDescriptionMatch.length >= 1) {
|
||||
res.description = decodeURI(youtubeDescriptionMatch[1]).replace(
|
||||
/\\n/g,
|
||||
'\n',
|
||||
)
|
||||
}
|
||||
if (youtubeThumbnailMatch && youtubeThumbnailMatch.length >= 2) {
|
||||
res.image = youtubeThumbnailMatch[1] + 'default.jpg'
|
||||
}
|
||||
if (!res.image && youtubeAvatarMatch && youtubeAvatarMatch.length >= 1) {
|
||||
res.image = youtubeAvatarMatch[1]
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
Reference in New Issue
Block a user