如何在所有浏览器的 canvas 上正确绘制视频缩略图？

Question

我有以下功能可以从视频生成缩略图：

async function getThumbnailForVideo(videoUrl) {
  const video = document.createElement("video");
  const canvas = document.createElement("canvas");
  video.style.display = "none";
  canvas.style.display = "none";

  // Trigger video load
  await new Promise((resolve, reject) => {
    video.addEventListener("loadedmetadata", () => {
      video.width = video.videoWidth;
      video.height = video.videoHeight;
      canvas.width = video.videoWidth;
      canvas.height = video.videoHeight;
      // Seek the video to 25%
      video.currentTime = video.duration * 0.25;
    });
    video.addEventListener("seeked", () => resolve());
    video.src = videoUrl;
  });

  // Draw the thumbnail
  canvas
    .getContext("2d")
    .drawImage(video, 0, 0, video.videoWidth, video.videoHeight);
  const imageUrl = canvas.toDataURL("image/png");
  return imageUrl;
}

与 URL.createObjectURL 配对，我能够从用户选择的视频文件生成缩略图。我在 StackBlitz for testing: App Editor App Preview

上创建了以下测试项目

虽然这似乎适用于 Chrome 和 Safari，但 Firefox 似乎不尊重视频的 EXIF 信息，因此无法正确绘制它。

MDN documentation for CanvasRenderingContext2D.drawImage 明确指出：

drawImage() will ignore all EXIF metadata in images, including the Orientation.. You should detect the Orientation yourself and use rotate() to make it right.

Modernizr hints at a solution via its exiforientation feature detection 我是否应该能够从文件中读取旋转数据，以便我只需要在 Firefox 上执行额外的转换。

我很好奇，是否有更幂等的解决方案来在所有浏览器上从 HTMLVideoElement 绘制图像？

Answer 1

所以事实证明，Modernizr exiforientation 测试仅检查 img 元素是否尊重图像的 EXIF 数据，但不检查绘制在 canvas 上的相同图像正确呈现。

我着手通过在 canvas 上绘制已知视频并对其进行测试来创建自己的测试。我使用 ffmpeg:

创建了视频

ffmpeg -filter_complex \
        "color=color=#ffffff:duration=1us:size=4x4[bg]; \
         color=color=#ff0000:duration=1us:size=2x2[r]; \
         color=color=#00ff00:duration=1us:size=2x2[g]; \
         color=color=#0000ff:duration=1us:size=2x2[b]; \
         [bg][r]overlay=x=2:y=0:format=rgb:alpha=premultiplied[bg+r]; \
         [bg+r][g]overlay=x=0:y=2:format=rgb:alpha=premultiplied[bg+r+g]; \
         [bg+r+g][b]overlay=x=2:y=2:format=rgb:alpha=premultiplied[bg+r+g+b]" \
       -map "[bg+r+g+b]" \
       -y wrgb-0.mp4

ffmpeg -i wrgb-0.mp4 -c copy -metadata:s:v:0 rotate=180 -y wrgb-180.mp4

使用 the same demo，我可以看到 Chrome 和 Firefox 在 canvas 上生成不同的视频预览。

Chrome:,蓝,绿,红,白
Firefox：、白、红、绿、蓝

接下来，我只需要一个函数，这样给定一个来自 canvas 的 RGBA 值数组，它会吐出 canvas 上的图案：

function getColourPattern(rgbaData) {
  let pattern = "";
  for (let i = 0; i < rgbaData.length; i += 4) {
    const r = rgbaData[i] / 255;
    const g = rgbaData[i + 1] / 255;
    const b = rgbaData[i + 2] / 255;
    const w = (r + g + b) / 3;

    if (w > 0.9) {
      pattern += "w";
      continue;
    }

    switch (Math.max(r, g, b)) {
      case r:
        pattern += "r";
        break;
      case g:
        pattern += "g";
        break;
      case b:
        pattern += "b";
        break;
    }
  }

  return pattern;
}

returns bbggbbggrrwwrrww 在 Chrome 和 Safari 上，wwrrwwrrggbbggbb 在 Firefox 上（canvas 指纹识别关闭）

然后我使用 basenc --base64 wrgb-180.mp4 -w 0 获取视频的 base64 表示，以便我可以将其嵌入到单个测试函数中：

export async function canvasUsesEXIF() {
  const videoUrl = `data:video/mp4;base64,AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1wNDEAAAAIZnJlZQAAAvxtZGF0AAACrgYF//+q3EXpvebZSLeWLNgg2SPu73gyNjQgLSBjb3JlIDE1OSByMjk5OSAyOTY0OTRhIC0gSC4yNjQvTVBFRy00IEFWQyBjb2RlYyAtIENvcHlsZWZ0IDIwMDMtMjAyMCAtIGh0dHA6Ly93d3cudmlkZW9sYW4ub3JnL3gyNjQuaHRtbCAtIG9wdGlvbnM6IGNhYmFjPTEgcmVmPTMgZGVibG9jaz0xOjA6MCBhbmFseXNlPTB4MzoweDExMyBtZT1oZXggc3VibWU9NyBwc3k9MSBwc3lfcmQ9MS4wMDowLjAwIG1peGVkX3JlZj0xIG1lX3JhbmdlPTE2IGNocm9tYV9tZT0xIHRyZWxsaXM9MSA4eDhkY3Q9MSBjcW09MCBkZWFkem9uZT0yMSwxMSBmYXN0X3Bza2lwPTEgY2hyb21hX3FwX29mZnNldD0tMiB0aHJlYWRzPTEgbG9va2FoZWFkX3RocmVhZHM9MSBzbGljZWRfdGhyZWFkcz0wIG5yPTAgZGVjaW1hdGU9MSBpbnRlcmxhY2VkPTAgYmx1cmF5X2NvbXBhdD0wIGNvbnN0cmFpbmVkX2ludHJhPTAgYmZyYW1lcz0zIGJfcHlyYW1pZD0yIGJfYWRhcHQ9MSBiX2JpYXM9MCBkaXJlY3Q9MSB3ZWlnaHRiPTEgb3Blbl9nb3A9MCB3ZWlnaHRwPTIga2V5aW50PTI1MCBrZXlpbnRfbWluPTI1IHNjZW5lY3V0PTQwIGludHJhX3JlZnJlc2g9MCByY19sb29rYWhlYWQ9NDAgcmM9Y3JmIG1idHJlZT0xIGNyZj0yMy4wIHFjb21wPTAuNjAgcXBtaW49MCBxcG1heD02OSBxcHN0ZXA9NCBpcF9yYXRpbz0xLjQwIGFxPTE6MS4wMACAAAAAPmWIhAAt/9pbuD7Z/gvI3kF2QzYeJnVbANgW8XnGVlnoDJNW7zJawMem6POfQ3cvmVl9l7mrZDdjuR26xB2/AAADAm1vb3YAAABsbXZoZAAAAAAAAAAAAAAAAAAAA+gAAAAoAAEAAAEAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAIsdHJhawAAAFx0a2hkAAAAAwAAAAAAAAAAAAAAAQAAAAAAAAAoAAAAAAAAAAAAAAAAAAAAAP//AAAAAAAAAAAAAAAAAAD//wAAAAAAAAAAAAAAAAAAQAAAAAAEAAAABAAAAAAAJGVkdHMAAAAcZWxzdAAAAAAAAAABAAAAKAAAAAAAAQAAAAABpG1kaWEAAAAgbWRoZAAAAAAAAAAAAAAAAAAAMgAAAAIAVcQAAAAAAC1oZGxyAAAAAAAAAAB2aWRlAAAAAAAAAAAAAAAAVmlkZW9IYW5kbGVyAAAAAU9taW5mAAAAFHZtaGQAAAABAAAAAAAAAAAAAAAkZGluZgAAABxkcmVmAAAAAAAAAAEAAAAMdXJsIAAAAAEAAAEPc3RibAAAAKtzdHNkAAAAAAAAAAEAAACbYXZjMQAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAEAAQASAAAAEgAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABj//wAAADVhdmNDAWQACv/hABhnZAAKrNlfnnwEQAAAAwBAAAAMg8SJZYABAAZo6+PLIsD9+PgAAAAAEHBhc3AAAAABAAAAAQAAABhzdHRzAAAAAAAAAAEAAAABAAACAAAAABxzdHNjAAAAAAAAAAEAAAABAAAAAQAAAAEAAAAUc3RzegAAAAAAAAL0AAAAAQAAABRzdGNvAAAAAAAAAAEAAAAwAAAAYnVkdGEAAABabWV0YQAAAAAAAAAhaGRscgAAAAAAAAAAbWRpcmFwcGwAAAAAAAAAAAAAAAAtaWxzdAAAACWpdG9vAAAAHWRhdGEAAAABAAAAAExhdmY1OC40NS4xMDA=`;
  const video = document.createElement("video");
  const canvas = document.createElement("canvas");
  video.style.display = "none";
  canvas.style.display = "none";

  await new Promise((resolve, reject) => {
    video.addEventListener("canplay", () => {
      video.width = video.videoWidth;
      video.height = video.videoHeight;
      canvas.width = video.videoWidth;
      canvas.height = video.videoHeight;
      video.currentTime = 0;
    });
    video.addEventListener("seeked", () => resolve());
    video.src = videoUrl;
  });

  const context = canvas.getContext("2d");
  context.drawImage(video, 0, 0, video.videoWidth, video.videoHeight);
  const { data } = context.getImageData(0, 0, 4, 4);

  return getColourPattern(data) === "bbggbbggrrwwrrww";
}

现在假设您有视频的旋转元数据，您应该能够测试是否需要在 canvas 上手动旋转它

编辑 1:

这应该可以修复 Firefox 在 Windows 上抛出 NS_ERROR_NOT_AVAILABLE 错误。

9c9
<     video.addEventListener("loadedmetadata", () => {
---
>     video.addEventListener("canplay", () => {

如何在所有浏览器的 canvas 上正确绘制视频缩略图？

How do I correctly draw a video thumbnail on a canvas on all browsers?

javascript

exif

canvas

video-thumbnails