canvas 30 fps at 1080p 的高质量媒体记录器

Question

我有一个 canvas 应用程序，它当前捕获 canvas 的图像并编译发送到 ffmpeg 的视频，然后输出他们选择的视频格式。问题是它超级慢！不是在视频转换上，而是在实际帧的编译上，你看我必须暂停视频和动画并截取 canvas 的屏幕截图。因此，我没有截取屏幕截图，而是考虑使用 MediaRecorder 和 canvas.captureStream。我能够获得视频输出，但质量非常低，视频不断丢帧。我需要帧速率至少为 30 fps 或更高，并且质量要高。这是我的记录功能

async [RECORD] ({state}) {
    state.videoOutputURL = null;
    state.outputVideo = document.createElement("video");
    const videoStream = state.canvas.captureStream(30);
    const mediaRecorder = new MediaRecorder(videoStream);
    mediaRecorder.ondataavailable = function(e) {
      state.captures.push(e.data);
    };
    
    mediaRecorder.onstop = function(e) {
      const blob = new Blob(state.captures);
      state.captures = [];
      const videoURL = URL.createObjectURL(blob);
      state.outputVideo.src = videoURL;
      state.outputVideo.width = 1280;
      state.outputVideo.height = 720;
      document.body.append(state.outputVideo);
    }; 
    mediaRecorder.start();
    
    state.anim.start();
    state.video.play();
    lottie.play();
    
    state.video.addEventListener("ended", async () => {
      mediaRecorder.stop();
    });
  }

Answer 1

我发现最好的方法是在 canvas 上实际暂停视频并使用 canvas.toDataURL 截取屏幕截图。我使用名为 Whammy 的库将屏幕截图编译成视频，然后将其发送到 FFmpeg 以翻录最终内容。下面的代码应该给出了一个很好的主意

async [TAKE_SCREENSHOT]({ state, dispatch }) {
    let seekResolve;
    if (!state.ended && state.video) {
      state.video.addEventListener("seeked", async () => {
        if (seekResolve) seekResolve();
      });
      await new Promise(async (resolve, reject) => {
        if (state.animations.length) {
          dispatch(PAUSE_LOTTIES);
        }
        dispatch(PAUSE_VIDEO);
        await new Promise(r => (seekResolve = r));
        if (state.layer) {
          state.layer.draw();
        }
        if (state.canvas) {
          state.captures.push(state.canvas.toDataURL("image/webp"));
        }
        resolve();
        dispatch(TAKE_SCREENSHOT);
      });
    }
  },
  async [PAUSE_VIDEO]({ state, dispatch, commit }) {
    state.video.pause();
    const oneFrame = 1 / 30;
    if (state.video.currentTime + oneFrame < state.video.duration) {
      state.video.currentTime += oneFrame;
      const percent = `${Math.round(
        (state.video.currentTime / state.video.duration) * 100
      )}%`;
      commit(SET_MODAL_STATUS, percent);
    } else {
      commit(SET_MODAL_STATUS, "Uploading your video");
      state.video.play();
      state.ended = true;
      await dispatch(GENERATE_VIDEO);
    }
  },
  async [PAUSE_LOTTIES]({ state }) {
    for (let i = 0; i < state.animations.length; i++) {
      let step = 0;
      let animation = state.animations[i].lottie;
      if (animation.currentFrame <= animation.totalFrames) {
        step = animation.currentFrame + 1;
      }
      await lottie.goToAndStop(step, true, animation.name);
    }
  },
  async [GENERATE_VIDEO]({ state, rootState, dispatch, commit }) {
    let status;
    state.editingZoom = null;
    const username =
      rootState.user.currentUser.username;
    const email = rootState.user.currentUser.email || rootState.user.guestEmail;
    const name = rootState.user.currentUser.firstName || "guest";
    const s3Id = rootState.templates.currentVideo.stock_s3_id || state.s3Id;
    const type = rootState.dataClay.fileFormat || state.type;
    const vid = new Whammy.fromImageArray(state.captures, 30);
    vid.lastModifiedDate = new Date();
    vid.name = "canvasVideo.webm";
    const data = new FormData();
    const id = `${username}_${new Date().getTime()}`;
    data.append("id", id);
    data.append("upload", vid);
    let projectId,
      fileName,
      matrix = null;
    if (!state.editorMode) {
      projectId = await dispatch(INSERT_PROJECT);
      fileName = `${rootState.dataClay.projectName}.${type}`;
      matrix = rootState.dataClay.matrix[0];
    } else {
      matrix = rootState.canvasSidebarMenu.selectedDisplay;
      projectId = id;
      fileName = `${id}.${type}`;
    }
    if (projectId || state.editorMode) {
      await dispatch(UPLOAD_TEMP_FILE, data);
      const key = await dispatch(CONVERT_FILE_TYPE, {
        id,
        username,
        type,
        projectId,
        matrix,
        name,
        email,
        editorMode: state.editorMode
      });
      const role = rootState.user.currentUser.role;
      state.file = `/api/files/${key}`;
      let message;
      let title = "Your video is ready";
      status = "rendered";
      if (!key) {
        status = "failed";
        message =
          "<p class='error'>Error processing video! If error continues please contact Creative Group. We are sorry for any inconvenience.</p>";
        title = "Error!";
      } else if (!rootState.user.currentUser.id) {
        message = `<p>Your video is ready. Signup for more great content!</p> <a href="${
          state.file
        }" download="${fileName}" class="btn btn-primary btn-block">Download</a>`;
      } else if (role != "banner") {
        message = `<p>Your video is ready.</p> <a href="${
          state.file
        }" download="${fileName}" class="btn btn-primary btn-block">Download</a>`;
      } else {
        message = `<p>Your video is ready. You may download your file from your banner account</p>`;
        await dispatch(EXPORT_TO_BANNER, {
          s3Id,
          fileUrl: key,
          extension: `.${type}`,
          resolution: matrix
        });
      }
      if (state.editorMode) {
        await dispatch(SAVE_CANVAS, { status, fileId: projectId });
      }
      state.video.loop = "loop";
      state.anim.stop();
      state.video.pause();
      lottie.unfreeze();
      await dispatch(DELETE_PROJECT_IN_PROGRESS);
      commit(RESET_PROJECT_IN_PROGRESS);
      commit(RESET_CANVAS);
      if (rootState.user.currentUser.id) {
        router.push("/account/projects");
      } else {
        router.push("/pricing");
      }
      dispatch(SHOW_MODAL, {
        name: "message",
        title,
        message
      });
    } else {
      await dispatch(FETCH_ALL_PUBLISHED_TEMPLATES);
      await dispatch(DELETE_PROJECT_IN_PROGRESS);
      commit(RESET_PROJECT_IN_PROGRESS);
      commit(RESET_CANVAS);
    }
  },

canvas 30 fps at 1080p 的高质量媒体记录器

high quality media recorder from canvas 30 fps at 1080p

javascript

html5-canvas

video-capture

web-mediarecorder