Use indexifembedded for embeds

packages/tests/src/client/embed-html.ts

@@ -1,9 +1,9 @@
 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
 
-import { expect } from 'chai'
-import { ServerConfig, VideoPlaylistCreateResult } from '@peertube/peertube-models'
+import { ServerConfig } from '@peertube/peertube-models'
 import { cleanupTests, makeHTMLRequest, PeerTubeServer } from '@peertube/peertube-server-commands'
 import { checkIndexTags, prepareClientTests } from '@tests/shared/client.js'
+import { expect } from 'chai'
 
 describe('Test embed HTML generation', function () {
   let servers: PeerTubeServer[]
@@ -18,7 +18,6 @@ describe('Test embed HTML generation', function () {
   let passwordProtectedVideoId: string
 
   let playlistIds: (string | number)[] = []
-  let playlist: VideoPlaylistCreateResult
   let privatePlaylistId: string
   let unlistedPlaylistId: string
   let playlistName: string
@@ -27,9 +26,8 @@ describe('Test embed HTML generation', function () {
   let instanceConfig: { name: string, shortDescription: string }
 
   before(async function () {
-    this.timeout(120000);
-
-    ({
+    this.timeout(120000)
+    ;({
       servers,
       videoIds,
       privateVideoId,
@@ -42,7 +40,6 @@ describe('Test embed HTML generation', function () {
       playlistIds,
       playlistName,
       playlistDescription,
-      playlist,
       unlistedPlaylistId,
       privatePlaylistId,
       instanceConfig
@@ -83,86 +80,82 @@ describe('Test embed HTML generation', function () {
   })
 
   describe('Canonical tags', function () {
-
-    it('Should use the original video URL for the canonical tag', async function () {
+    it('Should not use the original video URL for the canonical tag', async function () {
       for (const id of videoIds) {
         const res = await makeHTMLRequest(servers[0].url, '/videos/embed/' + id)
-        expect(res.text).to.contain(`<link rel="canonical" href="${servers[0].url}/w/${servers[0].store.video.shortUUID}" />`)
+        expect(res.text).to.not.contain(`<link rel="canonical" `)
       }
     })
 
-    it('Should use the original playlist URL for the canonical tag', async function () {
+    it('Should not use the original playlist URL for the canonical tag', async function () {
       for (const id of playlistIds) {
         const res = await makeHTMLRequest(servers[0].url, '/video-playlists/embed/' + id)
-        expect(res.text).to.contain(`<link rel="canonical" href="${servers[0].url}/w/p/${playlist.shortUUID}" />`)
+        expect(res.text).to.not.contain(`<link rel="canonical" `)
       }
     })
-
   })
 
   describe('Indexation tags', function () {
-
-    it('Should not index remote videos', async function () {
+    it('Should index remote videos', async function () {
       for (const id of videoIds) {
         {
           const res = await makeHTMLRequest(servers[1].url, '/videos/embed/' + id)
-          expect(res.text).to.contain('<meta name="robots" content="noindex" />')
+          expect(res.text).to.contain('<meta name="robots" content="noindex, indexifembedded" />')
         }
 
         {
           const res = await makeHTMLRequest(servers[0].url, '/videos/embed/' + id)
-          expect(res.text).to.not.contain('<meta name="robots" content="noindex" />')
+          expect(res.text).to.contain('<meta name="robots" content="noindex, indexifembedded" />')
         }
       }
     })
 
-    it('Should not index remote playlists', async function () {
+    it('Should index remote playlists', async function () {
       for (const id of playlistIds) {
         {
           const res = await makeHTMLRequest(servers[1].url, '/video-playlists/embed/' + id)
-          expect(res.text).to.contain('<meta name="robots" content="noindex" />')
+          expect(res.text).to.contain('<meta name="robots" content="noindex, indexifembedded" />')
         }
 
         {
           const res = await makeHTMLRequest(servers[0].url, '/video-playlists/embed/' + id)
-          expect(res.text).to.not.contain('<meta name="robots" content="noindex" />')
+          expect(res.text).to.contain('<meta name="robots" content="noindex, indexifembedded" />')
         }
       }
     })
 
-    it('Should add noindex meta tags for unlisted video', async function () {
+    it('Should not add noindex meta tags for unlisted video', async function () {
       {
         const res = await makeHTMLRequest(servers[0].url, '/videos/embed/' + videoIds[0])
 
-        expect(res.text).to.not.contain('<meta name="robots" content="noindex" />')
+        expect(res.text).to.contain('<meta name="robots" content="noindex, indexifembedded" />')
       }
 
       {
         const res = await makeHTMLRequest(servers[0].url, '/videos/embed/' + unlistedVideoId)
 
         expect(res.text).to.contain('unlisted')
-        expect(res.text).to.contain('<meta name="robots" content="noindex" />')
+        expect(res.text).to.contain('<meta name="robots" content="noindex, indexifembedded" />')
       }
     })
 
-    it('Should add noindex meta tags for unlisted playlist', async function () {
+    it('Should not add noindex meta tags for unlisted playlist', async function () {
       {
         const res = await makeHTMLRequest(servers[0].url, '/video-playlists/embed/' + playlistIds[0])
 
-        expect(res.text).to.not.contain('<meta name="robots" content="noindex" />')
+        expect(res.text).to.contain('<meta name="robots" content="noindex, indexifembedded" />')
       }
 
       {
         const res = await makeHTMLRequest(servers[0].url, '/video-playlists/embed/' + unlistedPlaylistId)
 
         expect(res.text).to.contain('unlisted')
-        expect(res.text).to.contain('<meta name="robots" content="noindex" />')
+        expect(res.text).to.contain('<meta name="robots" content="noindex, indexifembedded" />')
       }
     })
   })
 
   describe('Check leak of private objects', function () {
-
     it('Should not leak video information in embed', async function () {
       for (const id of [ privateVideoId, internalVideoId, passwordProtectedVideoId ]) {
         const res = await makeHTMLRequest(servers[0].url, '/videos/embed/' + id)

server/core/lib/html/shared/actor-html.ts

@@ -11,7 +11,6 @@ import { PageHtml } from './page-html.js'
 import { TagsHtml, TagsOptions } from './tags-html.js'
 
 export class ActorHtml {
-
   static async getAccountHTMLPage (nameWithHost: string, req: express.Request, res: express.Response) {
     const accountModelPromise = AccountModel.loadByNameWithHost(nameWithHost)
 
@@ -43,9 +42,10 @@ export class ActorHtml {
     return this.getAccountOrChannelHTMLPage({
       loader: () => Promise.resolve(account || channel),
 
-      getRSSFeeds: () => account
-        ? getDefaultRSSFeeds(WEBSERVER.URL, CONFIG.INSTANCE.NAME)
-        : getChannelRSSFeeds(WEBSERVER.URL, CONFIG.INSTANCE.NAME, channel),
+      getRSSFeeds: () =>
+        account
+          ? getDefaultRSSFeeds(WEBSERVER.URL, CONFIG.INSTANCE.NAME)
+          : getChannelRSSFeeds(WEBSERVER.URL, CONFIG.INSTANCE.NAME, channel),
 
       req,
       res
@@ -109,6 +109,7 @@ export class ActorHtml {
       },
 
       forbidIndexation: !entity.Actor.isOwned(),
+      embedIndexation: false,
 
       rssFeeds: getRSSFeeds(entity)
     }, {})

server/core/lib/html/shared/common.ts

@@ -12,5 +12,5 @@ export function buildEmptyEmbedHTML (options: {
   let htmlResult = TagsHtml.addTitleTag(html)
   htmlResult = TagsHtml.addDescriptionTag(htmlResult)
 
-  return TagsHtml.addTags(htmlResult, { forbidIndexation: true }, { playlist, video })
+  return TagsHtml.addTags(htmlResult, { forbidIndexation: true, embedIndexation: true }, { playlist, video })
 }

server/core/lib/html/shared/page-html.ts

@@ -21,7 +21,6 @@ import { ServerConfigManager } from '../../server-config-manager.js'
 import { TagsHtml } from './tags-html.js'
 
 export class PageHtml {
-
   private static htmlCache: { [path: string]: string } = {}
 
   static invalidateCache () {
@@ -60,6 +59,7 @@ export class PageHtml {
       ogType: 'website',
       twitterCard: 'summary_large_image',
       forbidIndexation: false,
+      embedIndexation: false,
       rssFeeds: getDefaultRSSFeeds(WEBSERVER.URL, CONFIG.INSTANCE.NAME)
     }, {})
 
@@ -132,7 +132,6 @@ export class PageHtml {
         sameSite: 'none',
         maxAge: 1000 * 3600 * 24 * 90 // 3 months
       })
-
     } else if (req.cookies.clientLanguage && is18nLocale(req.cookies.clientLanguage)) {
       lang = req.cookies.clientLanguage
     } else {
@@ -140,7 +139,8 @@ export class PageHtml {
     }
 
     logger.debug(
-      'Serving %s HTML language', buildFileLocale(lang),
+      'Serving %s HTML language',
+      buildFileLocale(lang),
       { cookie: req.cookies?.clientLanguage, paramLang, acceptLanguage: req.headers['accept-language'] }
     )
 

server/core/lib/html/shared/playlist-html.ts

@@ -13,7 +13,6 @@ import { PageHtml } from './page-html.js'
 import { TagsHtml } from './tags-html.js'
 
 export class PlaylistHtml {
-
   static async getWatchPlaylistHTML (videoPlaylistIdArg: string, req: express.Request, res: express.Response) {
     const videoPlaylistId = toCompleteUUID(videoPlaylistIdArg)
 
@@ -40,6 +39,7 @@ export class PlaylistHtml {
       addEmbedInfo: true,
       addOG: true,
       addTwitterCard: true,
+      isEmbed: false,
 
       currentQuery: req.query
     })
@@ -65,6 +65,7 @@ export class PlaylistHtml {
       addEmbedInfo: true,
       addOG: false,
       addTwitterCard: false,
+      isEmbed: true,
 
       // TODO: Implement it so we can send query params to oembed service
       currentQuery: {}
@@ -83,9 +84,11 @@ export class PlaylistHtml {
     addTwitterCard: boolean
     addEmbedInfo: boolean
 
+    isEmbed: boolean
+
     currentQuery: Record<string, string>
   }) {
-    const { html, playlist, addEmbedInfo, addOG, addTwitterCard, currentQuery = {} } = options
+    const { html, playlist, addEmbedInfo, addOG, addTwitterCard, isEmbed, currentQuery = {} } = options
     const escapedTruncatedDescription = TagsHtml.buildEscapedTruncatedDescription(playlist.description)
 
     let htmlResult = TagsHtml.addTitleTag(html, playlist.name)
@@ -113,7 +116,11 @@ export class PlaylistHtml {
       escapedTitle: escapeHTML(playlist.name),
       escapedTruncatedDescription,
 
-      forbidIndexation: !playlist.isOwned() || playlist.privacy !== VideoPlaylistPrivacy.PUBLIC,
+      forbidIndexation: isEmbed
+        ? playlist.privacy !== VideoPlaylistPrivacy.PUBLIC && playlist.privacy !== VideoPlaylistPrivacy.UNLISTED
+        : !playlist.isOwned() || playlist.privacy !== VideoPlaylistPrivacy.PUBLIC,
+
+      embedIndexation: isEmbed,
 
       image: playlist.hasThumbnail()
         ? { url: playlist.getThumbnailUrl(), width: playlist.Thumbnail.width, height: playlist.Thumbnail.height }

server/core/lib/html/shared/tags-html.ts

@@ -9,6 +9,7 @@ import { Hooks } from '../../plugins/hooks.js'
 
 export type TagsOptions = {
   forbidIndexation: boolean
+  embedIndexation: boolean
 
   url?: string
 
@@ -93,7 +94,7 @@ export class TagsHtml {
     }
     const schemaTags = await this.generateSchemaTagsOptions(tagsValues, context)
 
-    const { url, escapedTitle, oembedUrl, forbidIndexation, relMe, rssFeeds } = tagsValues
+    const { url, escapedTitle, oembedUrl, forbidIndexation, embedIndexation, relMe, rssFeeds } = tagsValues
 
     const oembedLinkTags: { type: string, href: string, escapedTitle: string }[] = []
 
@@ -133,13 +134,12 @@ export class TagsHtml {
       }
     }
 
-    // SEO, use origin URL
-    if (forbidIndexation !== true && url) {
-      tagsStr += `<link rel="canonical" href="${url}" />`
-    }
-
     if (forbidIndexation === true) {
       tagsStr += `<meta name="robots" content="noindex" />`
+    } else if (embedIndexation) {
+      tagsStr += `<meta name="robots" content="noindex, indexifembedded" />`
+    } else if (url) { // SEO, use origin URL
+      tagsStr += `<link rel="canonical" href="${url}" />`
     }
 
     for (const rssLink of (rssFeeds || [])) {

server/core/lib/html/shared/video-html.ts

@@ -15,7 +15,6 @@ import { PageHtml } from './page-html.js'
 import { TagsHtml } from './tags-html.js'
 
 export class VideoHtml {
-
   static async getWatchVideoHTML (videoIdArg: string, req: express.Request, res: express.Response) {
     const videoId = toCompleteUUID(videoIdArg)
 
@@ -42,7 +41,8 @@ export class VideoHtml {
       currentQuery: req.query,
       addEmbedInfo: true,
       addOG: true,
-      addTwitterCard: true
+      addTwitterCard: true,
+      isEmbed: false
     })
   }
 
@@ -66,6 +66,7 @@ export class VideoHtml {
       addEmbedInfo: true,
       addOG: false,
       addTwitterCard: false,
+      isEmbed: true,
 
       // TODO: Implement it so we can send query params to oembed service
       currentQuery: {}
@@ -84,9 +85,11 @@ export class VideoHtml {
     addTwitterCard: boolean
     addEmbedInfo: boolean
 
+    isEmbed: boolean
+
     currentQuery: Record<string, string>
   }) {
-    const { html, video, addEmbedInfo, addOG, addTwitterCard, currentQuery = {} } = options
+    const { html, video, addEmbedInfo, addOG, addTwitterCard, isEmbed, currentQuery = {} } = options
     const escapedTruncatedDescription = TagsHtml.buildEscapedTruncatedDescription(video.description)
 
     let customHTML = TagsHtml.addTitleTag(html, video.name)
@@ -120,7 +123,11 @@ export class VideoHtml {
       escapedTitle: escapeHTML(video.name),
       escapedTruncatedDescription,
 
-      forbidIndexation: video.remote || video.privacy !== VideoPrivacy.PUBLIC,
+      forbidIndexation: isEmbed
+        ? video.privacy !== VideoPrivacy.PUBLIC && video.privacy !== VideoPrivacy.UNLISTED
+        : video.remote || video.privacy !== VideoPrivacy.PUBLIC,
+
+      embedIndexation: isEmbed,
 
       image: preview
         ? { url: WEBSERVER.URL + video.getPreviewStaticPath(), width: preview.width, height: preview.height }