All Classes Namespaces Functions Variables Enumerations Properties Pages
movieexporter.cpp
1 /*
2 
3 Pencil2D - Traditional Animation Software
4 Copyright (C) 2012-2020 Matthew Chiawen Chang
5 
6 This program is free software; you can redistribute it and/or
7 modify it under the terms of the GNU General Public License
8 as published by the Free Software Foundation; version 2 of the License.
9 
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
14 
15 */
16 
17 #include "movieexporter.h"
18 
19 #include <ctime>
20 #include <vector>
21 #include <cstdint>
22 #include <QDir>
23 #include <QDebug>
24 #include <QProcess>
25 #include <QApplication>
26 #include <QStandardPaths>
27 #include <QThread>
28 #include <QtMath>
29 #include <QPainter>
30 
31 #include "object.h"
32 #include "layercamera.h"
33 #include "layersound.h"
34 #include "soundclip.h"
35 #include "util.h"
36 
37 MovieExporter::MovieExporter()
38 {
39 }
40 
41 MovieExporter::~MovieExporter()
42 {
43 }
44 
72  const ExportMovieDesc& desc,
73  std::function<void(float, float)> majorProgress,
74  std::function<void(float)> minorProgress,
75  std::function<void(QString)> progressMessage)
76 {
77  majorProgress(0.f, 0.03f);
78  minorProgress(0.f);
79  progressMessage(tr("Checking environment..."));
80 
81  clock_t t1 = clock();
82 
83  QString ffmpegPath = ffmpegLocation();
84  qDebug() << ffmpegPath;
85  if (!QFile::exists(ffmpegPath))
86  {
87 #ifdef _WIN32
88  qCritical() << "Please place ffmpeg.exe in " << ffmpegPath << " directory";
89 #else
90  qCritical() << "Please place ffmpeg in " << ffmpegPath << " directory";
91 #endif
92  return Status::ERROR_FFMPEG_NOT_FOUND;
93  }
94 
95  STATUS_CHECK(checkInputParameters(desc))
96  mDesc = desc;
97 
98  qDebug() << "OutFile: " << mDesc.strFileName;
99 
100  // Setup temporary folder
101  if (!mTempDir.isValid())
102  {
103  Q_ASSERT(false && "Cannot create temp folder.");
104  return Status::FAIL;
105  }
106 
107  mTempWorkDir = mTempDir.path();
108 
109  minorProgress(0.f);
110  if (desc.strFileName.endsWith("gif", Qt::CaseInsensitive))
111  {
112  majorProgress(0.03f, 1.f);
113  progressMessage(tr("Generating GIF..."));
114  minorProgress(0.f);
115  STATUS_CHECK(generateGif(obj, ffmpegPath, desc.strFileName, minorProgress))
116  }
117  else
118  {
119  majorProgress(0.03f, 0.25f);
120  progressMessage(tr("Assembling audio..."));
121  minorProgress(0.f);
122  STATUS_CHECK(assembleAudio(obj, ffmpegPath, minorProgress))
123  minorProgress(1.f);
124  majorProgress(0.25f, 1.f);
125  progressMessage(tr("Generating movie..."));
126  STATUS_CHECK(generateMovie(obj, ffmpegPath, desc.strFileName, minorProgress))
127  }
128  minorProgress(1.f);
129  majorProgress(1.f, 1.f);
130  progressMessage(tr("Done"));
131 
132  clock_t t2 = clock() - t1;
133  qDebug("MOVIE = %.1f sec", static_cast<double>(t2 / CLOCKS_PER_SEC));
134 
135  return Status::OK;
136 }
137 
138 QString MovieExporter::error()
139 {
140  return QString();
141 }
142 
156  QString ffmpegPath,
157  std::function<void(float)> progress)
158 {
159  // Quicktime assemble call
160  const int startFrame = mDesc.startFrame;
161  const int endFrame = mDesc.endFrame;
162  const int fps = mDesc.fps;
163 
164  Q_ASSERT(startFrame >= 0);
165  Q_ASSERT(endFrame >= startFrame);
166 
167  QDir dir(mTempWorkDir);
168  Q_ASSERT(dir.exists());
169 
170  QString tempAudioPath = QDir(mTempWorkDir).filePath("tmpaudio.wav");
171  qDebug() << "TempAudio=" << tempAudioPath;
172 
173  std::vector< SoundClip* > allSoundClips;
174 
175  std::vector< LayerSound* > allSoundLayers = obj->getLayersByType<LayerSound>();
176  for (LayerSound* layer : allSoundLayers)
177  {
178  layer->foreachKeyFrame([&allSoundClips](KeyFrame* key)
179  {
180  if (!key->fileName().isEmpty())
181  {
182  allSoundClips.push_back(static_cast<SoundClip*>(key));
183  }
184  });
185  }
186 
187  if (allSoundClips.empty()) return Status::SAFE;
188 
189  int clipCount = 0;
190 
191  QString filterComplex, amergeInput, panChannelLayout;
192  QStringList args;
193 
194  int wholeLen = qCeil((endFrame - startFrame) * 44100.0 / fps);
195  for (auto clip : allSoundClips)
196  {
197  if (mCanceled)
198  {
199  return Status::CANCELED;
200  }
201 
202  // Add sound file as input
203  args << "-i" << clip->fileName();
204 
205  // Offset the sound to its correct position
206  // See https://superuser.com/questions/716320/ffmpeg-placing-audio-at-specific-location
207  filterComplex += QString("[%1:a:0] aformat=sample_fmts=fltp:sample_rates=44100:channel_layouts=mono,volume=1,adelay=%2S|%2S,apad=whole_len=%3[ad%1];")
208  .arg(clipCount).arg(qRound(44100.0 * (clip->pos() - 1) / fps)).arg(wholeLen);
209  amergeInput += QString("[ad%1]").arg(clipCount);
210  panChannelLayout += QString("c%1+").arg(clipCount);
211 
212  clipCount++;
213  }
214  // Remove final '+'
215  panChannelLayout.chop(1);
216  // Output arguments
217  // Mix audio
218  args << "-filter_complex" << QString("%1%2 amerge=inputs=%3, pan=mono|c0=%4 [out]")
219  .arg(filterComplex).arg(amergeInput).arg(clipCount).arg(panChannelLayout);
220  // Convert audio file: 44100Hz sampling rate, stereo, signed 16 bit little endian
221  // Supported audio file types: wav, mp3, ogg... ( all file types supported by ffmpeg )
222  args << "-ar" << "44100" << "-acodec" << "pcm_s16le" << "-ac" << "2" << "-map" << "[out]" << "-y";
223  // Trim audio
224  args << "-ss" << QString::number((startFrame - 1) / static_cast<double>(fps));
225  args << "-to" << QString::number(endFrame / static_cast<double>(fps));
226  // Output path
227  args << tempAudioPath;
228 
229  STATUS_CHECK(MovieExporter::executeFFmpeg(ffmpegPath, args, [&progress, this] (int frame) { progress(frame / static_cast<float>(mDesc.endFrame - mDesc.startFrame)); return !mCanceled; }))
230  qDebug() << "audio file: " + tempAudioPath;
231 
232  return Status::OK;
233 }
234 
252  const Object* obj,
253  QString ffmpegPath,
254  QString strOutputFile,
255  std::function<void(float)> progress)
256 {
257  if (mCanceled)
258  {
259  return Status::CANCELED;
260  }
261 
262  // Frame generation setup
263 
264  int frameStart = mDesc.startFrame;
265  int frameEnd = mDesc.endFrame;
266  const QSize exportSize = mDesc.exportSize;
267  bool transparency = mDesc.alpha;
268  QString strCameraName = mDesc.strCameraName;
269  bool loop = mDesc.loop;
270 
271  auto cameraLayer = static_cast<LayerCamera*>(obj->findLayerByName(strCameraName, Layer::CAMERA));
272  if (cameraLayer == nullptr)
273  {
274  cameraLayer = obj->getLayersByType< LayerCamera >().front();
275  }
276  int currentFrame = frameStart;
277 
278  /* We create an image with the correct dimensions and background
279  * color here and then copy this and draw over top of it to
280  * generate each frame. This is faster than having to generate
281  * a new background image for each frame.
282  */
283  QImage imageToExportBase(exportSize, QImage::Format_ARGB32_Premultiplied);
284  QColor bgColor = Qt::white;
285  if (transparency)
286  {
287  bgColor.setAlpha(0);
288  }
289  imageToExportBase.fill(bgColor);
290 
291  QSize camSize = cameraLayer->getViewSize();
292  QTransform centralizeCamera;
293  centralizeCamera.translate(camSize.width() / 2, camSize.height() / 2);
294 
295  int failCounter = 0;
296  /* Movie export uses a "sliding window" to reduce memory usage
297  * while having a relatively small impact on speed. This basically
298  * means that there is a maximum number of frames that can be waiting
299  * to be encoded by ffmpeg at any one time. The limit is set by the
300  * frameWindow variable which is designed to take up a maximum of
301  * about 1GB of memory
302  */
303  int frameWindow = static_cast<int>(1e9 / (camSize.width() * camSize.height() * 4.0));
304 
305  // Build FFmpeg command
306 
307  //int exportFps = mDesc.videoFps;
308  const QString tempAudioPath = QDir(mTempWorkDir).filePath("tmpaudio.wav");
309 
310  QStringList args = {"-f", "rawvideo", "-pixel_format", "bgra"};
311  args << "-video_size" << QString("%1x%2").arg(exportSize.width()).arg(exportSize.height());
312  args << "-framerate" << QString::number(mDesc.fps);
313 
314  //args << "-r" << QString::number(exportFps);
315  args << "-i" << "-";
316  args << "-threads" << (QThread::idealThreadCount() == 1 ? "0" : QString::number(QThread::idealThreadCount()));
317 
318  if (QFile::exists(tempAudioPath))
319  {
320  args << "-i" << tempAudioPath;
321  }
322 
323  if (strOutputFile.endsWith(".apng", Qt::CaseInsensitive))
324  {
325  args << "-plays" << (loop ? "0" : "1");
326  }
327 
328  if (strOutputFile.endsWith("mp4", Qt::CaseInsensitive))
329  {
330  args << "-pix_fmt" << "yuv420p";
331  }
332 
333  if (strOutputFile.endsWith(".avi", Qt::CaseInsensitive))
334  {
335  args << "-q:v" << "5";
336  }
337 
338  args << "-y";
339  args << strOutputFile;
340 
341  // Run FFmpeg command
342 
343  STATUS_CHECK(executeFFMpegPipe(ffmpegPath, args, progress, [&](QProcess& ffmpeg, int framesProcessed)
344  {
345  if(framesProcessed < 0)
346  {
347  failCounter++;
348  }
349 
350  if(currentFrame > frameEnd)
351  {
352  ffmpeg.closeWriteChannel();
353  return false;
354  }
355 
356  if((currentFrame - frameStart <= framesProcessed + frameWindow || failCounter > 10) && currentFrame <= frameEnd)
357  {
358  QImage imageToExport = imageToExportBase.copy();
359  QPainter painter(&imageToExport);
360 
361  QTransform view = cameraLayer->getViewAtFrame(currentFrame);
362  painter.setWorldTransform(view * centralizeCamera);
363  painter.setWindow(QRect(0, 0, camSize.width(), camSize.height()));
364 
365  obj->paintImage(painter, currentFrame, false, true);
366  painter.end();
367 
368  // Should use sizeInBytes instead of byteCount to support large images,
369  // but this is only supported in QT 5.10+
370  int bytesWritten = ffmpeg.write(reinterpret_cast<const char*>(imageToExport.constBits()), imageToExport.byteCount());
371  Q_ASSERT(bytesWritten == imageToExport.byteCount());
372 
373  currentFrame++;
374  failCounter = 0;
375  return true;
376  }
377 
378  return false;
379  }));
380 
381  return Status::OK;
382 }
383 
397  const Object* obj,
398  QString ffmpegPath,
399  QString strOut,
400  std::function<void(float)> progress)
401 {
402 
403  if (mCanceled)
404  {
405  return Status::CANCELED;
406  }
407 
408  // Frame generation setup
409 
410  int frameStart = mDesc.startFrame;
411  int frameEnd = mDesc.endFrame;
412  const QSize exportSize = mDesc.exportSize;
413  bool transparency = false;
414  QString strCameraName = mDesc.strCameraName;
415  bool loop = mDesc.loop;
416  int bytesWritten;
417 
418  auto cameraLayer = static_cast<LayerCamera*>(obj->findLayerByName(strCameraName, Layer::CAMERA));
419  if (cameraLayer == nullptr)
420  {
421  cameraLayer = obj->getLayersByType< LayerCamera >().front();
422  }
423  int currentFrame = frameStart;
424 
425  /* We create an image with the correct dimensions and background
426  * color here and then copy this and draw over top of it to
427  * generate each frame. This is faster than having to generate
428  * a new background image for each frame.
429  */
430  QImage imageToExportBase(exportSize, QImage::Format_ARGB32_Premultiplied);
431  QColor bgColor = Qt::white;
432  if (transparency)
433  {
434  bgColor.setAlpha(0);
435  }
436  imageToExportBase.fill(bgColor);
437 
438  QSize camSize = cameraLayer->getViewSize();
439  QTransform centralizeCamera;
440  centralizeCamera.translate(camSize.width() / 2, camSize.height() / 2);
441 
442  // Build FFmpeg command
443 
444  QStringList args = {"-f", "rawvideo", "-pixel_format", "bgra"};
445  args << "-video_size" << QString("%1x%2").arg(exportSize.width()).arg(exportSize.height());
446  args << "-framerate" << QString::number(mDesc.fps);
447 
448  args << "-i" << "-";
449 
450  args << "-y";
451 
452  args << "-filter_complex" << "[0:v]palettegen [p]; [0:v][p] paletteuse";
453 
454  args << "-loop" << (loop ? "0" : "-1");
455  args << strOut;
456 
457  // Run FFmpeg command
458 
459  STATUS_CHECK(executeFFMpegPipe(ffmpegPath, args, progress, [&](QProcess& ffmpeg, int framesProcessed)
460  {
461  /* The GIF FFmpeg command requires the entires stream to be
462  * written before FFmpeg can encode the GIF. This is because
463  * the generated pallete is based off of the colors in all
464  * frames. The only way to avoid this would be to generate
465  * all the frames twice and run two separate commands, which
466  * would likely have unacceptable speed costs.
467  */
468 
469  Q_UNUSED(framesProcessed);
470  if(currentFrame > frameEnd)
471  {
472  ffmpeg.closeWriteChannel();
473  return false;
474  }
475 
476  QImage imageToExport = imageToExportBase.copy();
477  QPainter painter(&imageToExport);
478 
479  QTransform view = cameraLayer->getViewAtFrame(currentFrame);
480  painter.setWorldTransform(view * centralizeCamera);
481  painter.setWindow(QRect(0, 0, camSize.width(), camSize.height()));
482 
483  obj->paintImage(painter, currentFrame, false, true);
484 
485  bytesWritten = ffmpeg.write(reinterpret_cast<const char*>(imageToExport.constBits()), imageToExport.byteCount());
486  Q_ASSERT(bytesWritten == imageToExport.byteCount());
487 
488  currentFrame++;
489 
490  return true;
491  }));
492 
493  return Status::OK;
494 }
495 
512 Status MovieExporter::executeFFmpeg(const QString& cmd, const QStringList& args, std::function<bool(int)> progress)
513 {
514  qDebug() << cmd;
515 
516  QProcess ffmpeg;
518  // FFmpeg writes to stderr only for some reason, so we just read both channels together
520  ffmpeg.start(cmd, args);
521 
522  Status status = Status::OK;
523  DebugDetails dd;
524  dd << QStringLiteral("Command: %1 %2").arg(cmd).arg(args.join(' '));
525  if (ffmpeg.waitForStarted())
526  {
527  while(ffmpeg.state() == QProcess::Running)
528  {
529  if(!ffmpeg.waitForReadyRead()) break;
530 
531  QString output(ffmpeg.readAll());
532  QStringList sList = output.split(QRegExp("[\r\n]"), QString::SkipEmptyParts);
533  for (const QString& s : sList)
534  {
535  qDebug() << "[ffmpeg]" << s;
536  dd << s;
537  }
538 
539  if(output.startsWith("frame="))
540  {
541  QString frame = output.mid(6, output.indexOf(' '));
542 
543  bool shouldContinue = progress(frame.toInt());
544  if (!shouldContinue)
545  {
546  ffmpeg.terminate();
547  ffmpeg.waitForFinished(3000);
548  if (ffmpeg.state() == QProcess::Running) ffmpeg.kill();
549  ffmpeg.waitForFinished();
550  return Status::CANCELED;
551  }
552  }
553  }
554 
555  QString output(ffmpeg.readAll());
556  QStringList sList = output.split(QRegExp("[\r\n]"), QString::SkipEmptyParts);
557  for (const QString& s : sList)
558  {
559  qDebug() << "[ffmpeg]" << s;
560  dd << s;
561  }
562 
563  if(ffmpeg.exitStatus() != QProcess::NormalExit || ffmpeg.exitCode() != 0)
564  {
565  status = Status::FAIL;
566  status.setTitle(tr("Something went wrong"));
567  status.setDescription(tr("Looks like our video backend did not exit normally. Your movie may not have exported correctly. Please try again and report this if it persists."));
568  dd << QString("Exit status: ").append(QProcess::NormalExit ? "NormalExit": "CrashExit")
569  << QString("Exit code: %1").arg(ffmpeg.exitCode());
570  status.setDetails(dd);
571  return status;
572  }
573  }
574  else
575  {
576  qDebug() << "ERROR: Could not execute FFmpeg.";
577  status = Status::FAIL;
578  status.setTitle(tr("Something went wrong"));
579  status.setDescription(tr("Couldn't start the video backend, please try again."));
580  status.setDetails(dd);
581  }
582  return status;
583 }
584 
628 Status MovieExporter::executeFFMpegPipe(const QString& cmd, const QStringList& args, std::function<void(float)> progress, std::function<bool(QProcess&, int)> writeFrame)
629 {
630  qDebug() << cmd;
631 
632  QProcess ffmpeg;
634  // FFmpeg writes to stderr only for some reason, so we just read both channels together
636  ffmpeg.start(cmd, args);
637 
638  Status status = Status::OK;
639  DebugDetails dd;
640  dd << QStringLiteral("Command: %1 %2").arg(cmd).arg(args.join(' '));
641  if (ffmpeg.waitForStarted())
642  {
643  int framesGenerated = 0;
644  int lastFrameProcessed = 0;
645  const int frameStart = mDesc.startFrame;
646  const int frameEnd = mDesc.endFrame;
647  while(ffmpeg.state() == QProcess::Running)
648  {
649  if (mCanceled)
650  {
651  ffmpeg.terminate();
652  if (ffmpeg.state() == QProcess::Running) ffmpeg.kill();
653  return Status::CANCELED;
654  }
655 
656  // Check FFmpeg progress
657 
658  int framesProcessed = -1;
659  if(ffmpeg.waitForReadyRead(10))
660  {
661  QString output(ffmpeg.readAll());
662  QStringList sList = output.split(QRegExp("[\r\n]"), QString::SkipEmptyParts);
663  for (const QString& s : sList)
664  {
665  qDebug() << "[ffmpeg]" << s;
666  dd << s;
667  }
668  if(output.startsWith("frame="))
669  {
670  lastFrameProcessed = framesProcessed = output.mid(6, output.indexOf(' ')).toInt();
671  }
672  }
673 
674  if(!ffmpeg.isWritable())
675  {
676  continue;
677  }
678 
679  while(writeFrame(ffmpeg, framesProcessed))
680  {
681  framesGenerated++;
682 
683  const float percentGenerated = framesGenerated / static_cast<float>(frameEnd - frameStart);
684  const float percentConverted = lastFrameProcessed / static_cast<float>(frameEnd - frameStart);
685  progress((percentGenerated + percentConverted) / 2);
686  }
687  const float percentGenerated = framesGenerated / static_cast<float>(frameEnd - frameStart);
688  const float percentConverted = lastFrameProcessed / static_cast<float>(frameEnd - frameStart);
689  progress((percentGenerated + percentConverted) / 2);
690  }
691 
692  QString output(ffmpeg.readAll());
693  QStringList sList = output.split(QRegExp("[\r\n]"), QString::SkipEmptyParts);
694  for (const QString& s : sList)
695  {
696  qDebug() << "[ffmpeg]" << s;
697  dd << s;
698  }
699 
700  if(ffmpeg.exitStatus() != QProcess::NormalExit || ffmpeg.exitCode() != 0)
701  {
702  status = Status::FAIL;
703  status.setTitle(tr("Something went wrong"));
704  status.setDescription(tr("Looks like our video backend did not exit normally. Your movie may not have exported correctly. Please try again and report this if it persists."));
705  dd << QString("Exit status: ").append(QProcess::NormalExit ? "NormalExit": "CrashExit")
706  << QString("Exit code: %1").arg(ffmpeg.exitCode());
707  status.setDetails(dd);
708  return status;
709  }
710  }
711  else
712  {
713  qDebug() << "ERROR: Could not execute FFmpeg.";
714  status = Status::FAIL;
715  status.setTitle(tr("Something went wrong"));
716  status.setDescription(tr("Couldn't start the video backend, please try again."));
717  status.setDetails(dd);
718  }
719 
720  return status;
721 }
722 
723 Status MovieExporter::checkInputParameters(const ExportMovieDesc& desc)
724 {
725  bool b = true;
726  b &= (!desc.strFileName.isEmpty());
727  b &= (desc.startFrame > 0);
728  b &= (desc.endFrame >= desc.startFrame);
729  b &= (desc.fps > 0);
730  b &= (!desc.strCameraName.isEmpty());
731 
732  return b ? Status::OK : Status::INVALID_ARGUMENT;
733 }
QString & append(QChar ch)
void kill()
bool isWritable() const const
int width() const const
bool end()
Format_ARGB32_Premultiplied
virtual bool waitForReadyRead(int msecs) override
QString filePath(const QString &fileName) const const
void setAlpha(int alpha)
QString join(const QString &separator) const const
bool isValid() const const
bool exists() const const
int byteCount() const const
QImage copy(const QRect &rectangle) const const
void chop(int n)
void terminate()
Status assembleAudio(const Object *obj, QString ffmpegPath, std::function< void(float)> progress)
Combines all audio tracks in obj into a single file.
void setWindow(const QRect &rectangle)
QTransform & translate(qreal dx, qreal dy)
QString number(int n, int base)
bool exists() const const
void fill(uint pixelValue)
void setWorldTransform(const QTransform &matrix, bool combine)
CaseInsensitive
int toInt(bool *ok, int base) const const
bool isEmpty() const const
QByteArray readAll()
const uchar * constBits() const const
bool endsWith(const QString &s, Qt::CaseSensitivity cs) const const
Status run(const Object *obj, const ExportMovieDesc &desc, std::function< void(float, float)> majorProgress, std::function< void(float)> minorProgress, std::function< void(QString)> progressMessage)
Begin exporting the movie described by exportDesc.
QString path() const const
bool waitForStarted(int msecs)
static Status executeFFmpeg(const QString &cmd, const QStringList &args, std::function< bool(int)> progress)
Runs the specified command (should be ffmpeg) and allows for progress feedback.
void setProcessChannelMode(QProcess::ProcessChannelMode mode)
int idealThreadCount()
QString mid(int position, int n) const const
QString arg(qlonglong a, int fieldWidth, int base, QChar fillChar) const const
int height() const const
qint64 write(const char *data, qint64 maxSize)
Definition: object.h:41
Status generateGif(const Object *obj, QString ffmpeg, QString strOut, std::function< void(float)> progress)
Exports obj to a gif image at strOut using FFmpeg.
void setReadChannel(QProcess::ProcessChannel channel)
void closeWriteChannel()
QProcess::ExitStatus exitStatus() const const
int exitCode() const const
void start(const QString &program, const QStringList &arguments, QIODevice::OpenMode mode)
QProcess::ProcessState state() const const
Status generateMovie(const Object *obj, QString ffmpegPath, QString strOutputFile, std::function< void(float)> progress)
Exports obj to a movie image at strOut using FFmpeg.
Status executeFFMpegPipe(const QString &cmd, const QStringList &args, std::function< void(float)> progress, std::function< bool(QProcess &, int)> writeFrame)
Runs the specified command (should be ffmpeg), and lets writeFrame pipe data into it 1 frame at a tim...
bool waitForFinished(int msecs)