feat(copyWithProgress): report copy progress every second instead of every 1%

FlowHelpers/1.0.0/cliUtils.js

@@ -1,43 +1,92 @@
 'use strict';
 /**
- * Local mock of Tdarr FlowHelpers/1.0.0/cliUtils.js
+ * Local fallback/mock of Tdarr FlowHelpers/1.0.0/cliUtils.js
- * Used only for local testing — on real Tdarr this file comes from the server.
+ * Used for local testing and as safe fallback.
  *
  * Matches the real Tdarr CLI constructor signature:
- *   new CLI({ cli, spawnArgs, WorkDir, jobLog, inputFileObj, updateWorker })
+ *   new CLI({ cli, spawnArgs, spawnOpts, WorkDir, jobLog, inputFileObj, updateWorker, logFullCliOutput })
- * - inputFileObj : used by real Tdarr for ffprobe-based progress tracking
- * - updateWorker : callback used by real Tdarr to push progress to the UI
- * Both are accepted but unused here (local mock just logs to stdout).
  */
 const { spawn } = require('child_process');
 
 class CLI {
   constructor(config) {
-    this.config = config;
+    this.config = config || {};
   }
 
   runCli() {
     return new Promise((resolve) => {
-      const { cli, spawnArgs, WorkDir, jobLog } = this.config;
+      const {
+        cli,
+        spawnArgs = [],
+        spawnOpts = {},
+        WorkDir,
+        jobLog,
+        updateWorker,
+        logFullCliOutput = false,
+      } = this.config;
 
-      if (jobLog) jobLog(`[CLI] Running: ${cli} ${spawnArgs.join(' ')}\n`);
+      const opts = {
-
+        cwd: spawnOpts.cwd || WorkDir || process.cwd(),
-      const proc = spawn(cli, spawnArgs, {
-        cwd: WorkDir || process.cwd(),
         stdio: ['ignore', 'pipe', 'pipe'],
         shell: false,
-      });
+        ...spawnOpts,
+      };
 
-      proc.stdout.on('data', (d) => { if (jobLog) jobLog(d.toString()); });
+      if (jobLog) jobLog(`Running ${cli} ${spawnArgs.join(' ')}\n`);
-      proc.stderr.on('data', (d) => { if (jobLog) jobLog(d.toString()); });
+
+      const proc = spawn(cli, spawnArgs, opts);
+      const errorLogFull = [];
+
+      const handleData = (data) => {
+        const str = data.toString();
+        errorLogFull.push(str);
+
+        if (logFullCliOutput && jobLog) {
+          jobLog(str);
+        }
+
+        if (typeof updateWorker === 'function') {
+          // ffmpeg progress matching: time=HH:MM:SS
+          const fpsMatch = str.match(/fps=\s*(\d+(?:\.\d+)?)/);
+          if (fpsMatch) {
+            updateWorker({ fps: parseFloat(fpsMatch[1]) });
+          }
+
+          const timeMatch = str.match(/time=(\d{2}):(\d{2}):(\d{2}(?:\.\d+)?)/);
+          if (timeMatch) {
+            const curSec = parseInt(timeMatch[1], 10) * 3600
+              + parseInt(timeMatch[2], 10) * 60
+              + parseFloat(timeMatch[3]);
+            const duration = this.config.inputFileObj?.meta?.Duration
+              || parseFloat(this.config.inputFileObj?.ffProbeData?.format?.duration)
+              || 0;
+            if (duration > 0) {
+              const perc = Math.min(100, Math.max(0, +((curSec / duration) * 100).toFixed(1)));
+              updateWorker({ percentage: perc });
+            }
+          } else if (str.includes('%')) {
+            // mkvextract progress matching (e.g. "Progress: 45%" or "Progresso: 45%")
+            const pMatch = str.match(/(?:Progress|Progresso):\s*(\d+)%/i);
+            if (pMatch) {
+              updateWorker({ percentage: parseInt(pMatch[1], 10) });
+            }
+          }
+        }
+      };
+
+      proc.stdout.on('data', handleData);
+      proc.stderr.on('data', handleData);
 
       proc.on('close', (code) => {
-        resolve({ cliExitCode: code || 0 });
+        if (!logFullCliOutput && jobLog) {
+          jobLog(errorLogFull.slice(-1000).join(''));
+        }
+        resolve({ cliExitCode: code || 0, errorLogFull });
       });
 
       proc.on('error', (err) => {
-        if (jobLog) jobLog(`[CLI] Error: ${err.message}\n`);
+        if (jobLog) jobLog(`Error executing binary: ${cli}: ${err.message}\n`);
-        resolve({ cliExitCode: 1 });
+        resolve({ cliExitCode: 1, errorLogFull });
       });
     });
   }
@@ -47,3 +96,4 @@ const getFFmpegVar = () => {};
 
 module.exports = { CLI, getFFmpegVar };
 
+

FlowHelpers/1.0.0/copyWithProgress.js

@@ -5,7 +5,7 @@ const path = require('path');
 
 const MAX_RETRIES = 2;
 const RETRY_DELAY_MS = 1000;
-const LOG_INTERVAL_MS = 3000;
+const PROGRESS_INTERVAL_MS = 1000;
 
 const sleep = (milliseconds) => new Promise((resolve) => {
   setTimeout(resolve, milliseconds);
@@ -85,9 +85,7 @@ const copyWithProgress = async ({
 
       const startedAt = Date.now();
       let copiedBytes = 0;
-      let lastReportedPercent = -1;
+      let lastProgressTime = 0;
-      let lastLoggedPercent = -10;
-      let lastLoggedTime = startedAt;
       let intervalBytes = 0;
       let intervalTime = startedAt;
 
@@ -114,15 +112,15 @@ const copyWithProgress = async ({
           const now = Date.now();
           const percent = Math.min(Math.floor((copiedBytes / sourceSize) * 100), 99);
 
-          // Calculate ETA and speed
+          // Update UI and jobLog every second (PROGRESS_INTERVAL_MS) or on initial data chunk
+          const timeSinceLastProgress = now - lastProgressTime;
+          if (lastProgressTime === 0 || timeSinceLastProgress >= PROGRESS_INTERVAL_MS) {
             const totalElapsedSec = Math.max((now - startedAt) / 1000, 0.001);
             const avgSpeedBps = copiedBytes / totalElapsedSec;
             const remainingBytes = Math.max(0, sourceSize - copiedBytes);
             const remainingSec = avgSpeedBps > 0 ? remainingBytes / avgSpeedBps : 0;
             const eta = formatEta(remainingSec);
 
-          // Update Tdarr UI on every percentage increase
-          if (percent > lastReportedPercent) {
             if (typeof updateWorker === 'function') {
               updateWorker({
                 percentage: percent,
@@ -132,12 +130,7 @@ const copyWithProgress = async ({
             if (typeof updateProgress === 'function') {
               updateProgress({ progress: percent });
             }
-            lastReportedPercent = percent;
-          }
 
-          // Throttle jobLog output: every 10% or every LOG_INTERVAL_MS
-          const timeSinceLastLog = now - lastLoggedTime;
-          if (percent >= lastLoggedPercent + 10 || (timeSinceLastLog >= LOG_INTERVAL_MS && percent !== lastLoggedPercent)) {
             const intSec = Math.max((now - intervalTime) / 1000, 0.001);
             const currentSpeedMb = (intervalBytes / 1024 / 1024 / intSec).toFixed(1);
             const copiedMb = (copiedBytes / 1024 / 1024).toFixed(1);
@@ -150,8 +143,7 @@ const copyWithProgress = async ({
               );
             }
 
-            lastLoggedPercent = percent;
+            lastProgressTime = now;
-            lastLoggedTime = now;
             intervalBytes = 0;
             intervalTime = now;
           }

LocalFlowPlugins/audio/backupSourceAudio/1.0.0/index.js

@@ -74,10 +74,13 @@ const plugin = async (args) => {
       '-c:a', 'copy',
       outFile,
     ],
+    spawnOpts: { cwd: workDir },
     WorkDir: workDir,
     jobLog,
+    outputFilePath: outFile,
     inputFileObj,
     updateWorker,
+    logFullCliOutput: args.logFullCliOutput || false,
     args,
   });
 

LocalFlowPlugins/subtitles/extractSubtitlesAndOcr/1.0.0/index.js

@@ -99,10 +99,12 @@ const runCLI = async (cliBin, spawnArgs, args) => {
   const cli = new CLI({
     cli:              cliBin,
     spawnArgs,
+    spawnOpts:        { cwd: args.workDir },
     WorkDir:          args.workDir,
     jobLog:           args.jobLog,
     inputFileObj:     args.inputFileObj,
     updateWorker:     args.updateWorker,
+    logFullCliOutput: args.logFullCliOutput || false,
     args,
   });
   return cli.runCli();

LocalFlowPlugins/utility/extractChaptersXml/1.0.0/index.js

@@ -50,10 +50,12 @@ const plugin = async (args) => {
   const cli = new CLI({
     cli: mkvExtract,
     spawnArgs: [inputFile, 'chapters', xmlPath],
+    spawnOpts: { cwd: workDir },
     WorkDir: workDir,
     jobLog,
     inputFileObj,
     updateWorker,
+    logFullCliOutput: args.logFullCliOutput || false,
     args,
   });
 

LocalFlowPlugins/video/transcode2PassX265/1.0.0/index.js

@@ -156,6 +156,17 @@ const shouldCopyVideo = (videoStream, copyHevcBelowKbps, streams, format) => {
     && bitrateKbps > 0 && bitrateKbps < copyHevcBelowKbps;
 };
 
+const scalePassPercentage = (pass, percentage) => {
+  const p = Math.min(100, Math.max(0, Number(percentage) || 0));
+  if (pass === 1) {
+    return +(p * 0.5).toFixed(1);
+  }
+  if (pass === 2) {
+    return +(50 + p * 0.5).toFixed(1);
+  }
+  return p;
+};
+
 // ─── Plugin ───────────────────────────────────────────────────────────────────
 
 const plugin = async (args) => {
@@ -266,10 +277,13 @@ const plugin = async (args) => {
         '-sn',
         outFile,
       ],
+      spawnOpts: { cwd: workDir },
       WorkDir: workDir,
       jobLog,
+      outputFilePath: outFile,
       inputFileObj,
       updateWorker,
+      logFullCliOutput: args.logFullCliOutput || false,
       args,
     });
 
@@ -294,9 +308,62 @@ const plugin = async (args) => {
   }
 
   const { CLI } = require('../../../../FlowHelpers/1.0.0/cliUtils');
+  const { updateProgress } = args;
+
+  // Scale 2-pass progress so Pass 1 counts for 0-50% and Pass 2 counts for 50-100%
+  const pass1UpdateWorker = (progressData) => {
+    if (typeof updateWorker !== 'function') return;
+    if (progressData && typeof progressData.percentage === 'number') {
+      updateWorker({
+        ...progressData,
+        percentage: scalePassPercentage(1, progressData.percentage),
+      });
+    } else {
+      updateWorker(progressData);
+    }
+  };
+
+  const pass1UpdateProgress = (progressData) => {
+    if (typeof updateProgress !== 'function') return;
+    if (progressData && typeof progressData.progress === 'number') {
+      updateProgress({
+        ...progressData,
+        progress: scalePassPercentage(1, progressData.progress),
+      });
+    } else {
+      updateProgress(progressData);
+    }
+  };
+
+  const pass2UpdateWorker = (progressData) => {
+    if (typeof updateWorker !== 'function') return;
+    if (progressData && typeof progressData.percentage === 'number') {
+      updateWorker({
+        ...progressData,
+        percentage: scalePassPercentage(2, progressData.percentage),
+      });
+    } else {
+      updateWorker(progressData);
+    }
+  };
+
+  const pass2UpdateProgress = (progressData) => {
+    if (typeof updateProgress !== 'function') return;
+    if (progressData && typeof progressData.progress === 'number') {
+      updateProgress({
+        ...progressData,
+        progress: scalePassPercentage(2, progressData.progress),
+      });
+    } else {
+      updateProgress(progressData);
+    }
+  };
 
   // ══ PASS 1 ════════════════════════════════════════════════════════════════
   jobLog('[transcode2PassX265] Starting Pass 1...\n');
+  pass1UpdateWorker({ percentage: 0 });
+  pass1UpdateProgress({ progress: 0 });
+
   const pass1Cli = new CLI({
     cli: args.ffmpegPath,
     spawnArgs: [
@@ -310,18 +377,22 @@ const plugin = async (args) => {
       '-f', 'null',
       nullDev,
     ],
+    spawnOpts: { cwd: workDir },
     WorkDir: workDir,
     jobLog,
     inputFileObj,
-    updateWorker,
+    updateWorker: pass1UpdateWorker,
-    args,
+    logFullCliOutput: args.logFullCliOutput || false,
+    args: { ...args, updateWorker: pass1UpdateWorker, updateProgress: pass1UpdateProgress },
   });
 
   const { cliExitCode: pass1Exit } = await pass1Cli.runCli();
   if (pass1Exit !== 0) {
     throw new Error(`[transcode2PassX265] Pass 1 failed with exit code ${pass1Exit}`);
   }
-  jobLog('[transcode2PassX265] Pass 1 complete.\n');
+  pass1UpdateWorker({ percentage: 100 });
+  pass1UpdateProgress({ progress: 100 });
+  jobLog('[transcode2PassX265] Pass 1 complete (50% total progress).\n');
 
   // ══ PASS 2 ════════════════════════════════════════════════════════════════
   const audioArgs = buildAudioArgs(variables);
@@ -330,6 +401,9 @@ const plugin = async (args) => {
   }
 
   jobLog('[transcode2PassX265] Starting Pass 2...\n');
+  pass2UpdateWorker({ percentage: 0 });
+  pass2UpdateProgress({ progress: 0 });
+
   const pass2Cli = new CLI({
     cli: args.ffmpegPath,
     spawnArgs: [
@@ -345,18 +419,23 @@ const plugin = async (args) => {
       '-sn',
       outFile,
     ],
+    spawnOpts: { cwd: workDir },
     WorkDir: workDir,
     jobLog,
+    outputFilePath: outFile,
     inputFileObj,
-    updateWorker,
+    updateWorker: pass2UpdateWorker,
-    args,
+    logFullCliOutput: args.logFullCliOutput || false,
+    args: { ...args, updateWorker: pass2UpdateWorker, updateProgress: pass2UpdateProgress },
   });
 
   const { cliExitCode: pass2Exit } = await pass2Cli.runCli();
   if (pass2Exit !== 0) {
     throw new Error(`[transcode2PassX265] Pass 2 failed with exit code ${pass2Exit}`);
   }
-  jobLog('[transcode2PassX265] Pass 2 complete.\n');
+  pass2UpdateWorker({ percentage: 100 });
+  pass2UpdateProgress({ progress: 100 });
+  jobLog('[transcode2PassX265] Pass 2 complete (100% total progress).\n');
 
   // ── Cleanup 2-pass log files and temporary input copy ────────────────────
   [statsFile, `${statsFile}.mbtree`, `${statsFile}.cutree`].forEach((f) => {
@@ -381,3 +460,4 @@ const plugin = async (args) => {
 
 module.exports.details = details;
 module.exports.plugin = plugin;
+module.exports.scalePassPercentage = scalePassPercentage;

install.sh

@@ -105,6 +105,17 @@ download_flow_helpers() {
     curl -fsSL "$official_fu" -o "${dest}/fileUtils.js" || true
   fi
 
+  # Restore official Tdarr cliUtils.js if it was previously overwritten by a mock
+  local official_cu="https://raw.githubusercontent.com/HaveAGitGat/Tdarr_Plugins/master/FlowPlugins/FlowHelpers/1.0.0/cliUtils.js"
+  if [[ -f "${dest}/cliUtils.js" ]]; then
+    if grep -q "Local mock of Tdarr" "${dest}/cliUtils.js" || ! grep -q "ffmpegParser" "${dest}/cliUtils.js"; then
+      echo "  🔄 Restoring official Tdarr cliUtils.js..."
+      curl -fsSL "$official_cu" -o "${dest}/cliUtils.js" || true
+    fi
+  else
+    curl -fsSL "$official_cu" -o "${dest}/cliUtils.js" || true
+  fi
+
   # Download our custom copyWithProgress helper
   local url="${RAW}/FlowHelpers/1.0.0/copyWithProgress.js"
   if curl -fsSL "$url" -o "${dest}/copyWithProgress.js"; then

tests/quick-logic-test.js

@@ -236,8 +236,16 @@ const AUDIO_INPUTS = {
   ok(transcodeCode.includes("normalizedKey === 'bps' || normalizedKey === 'bps-eng'"), 'HEVC copy check falls back to Matroska BPS metadata');
   ok(transcodeCode.includes('containerBitrate - otherStreamBitrate'), 'HEVC copy check estimates video bitrate from container and companion streams');
   ok(!transcodeCode.includes("'-map_metadata -1'"), 'pass 1 separates -map_metadata from its value');
-  ok(!transcodeCode.includes("'-map_chapters -1'"), 'pass 1 separates -map_chapters from its value');
   ok(transcodeCode.includes('const statsForX265 = path.basename(statsFile)'), 'x265 stats file is relative to workDir on Windows');
+  const { scalePassPercentage } = require(path.join(ROOT, 'LocalFlowPlugins/video/transcode2PassX265/1.0.0/index.js'));
+  eq(scalePassPercentage(1, 0), 0, 'pass 1 at 0% maps to 0%');
+  eq(scalePassPercentage(1, 50), 25, 'pass 1 at 50% maps to 25%');
+  eq(scalePassPercentage(1, 100), 50, 'pass 1 at 100% maps to 50%');
+  eq(scalePassPercentage(2, 0), 50, 'pass 2 at 0% maps to 50%');
+  eq(scalePassPercentage(2, 50), 75, 'pass 2 at 50% maps to 75%');
+  eq(scalePassPercentage(2, 100), 100, 'pass 2 at 100% maps to 100%');
+  ok(transcodeCode.includes('scalePassPercentage(1,'), 'pass 1 wraps updateWorker with 50% scale');
+  ok(transcodeCode.includes('scalePassPercentage(2,'), 'pass 2 wraps updateWorker with 50% scale');
 
   // ── copyToWorkDir & replaceOriginalFile v1.0.0 checks ─────────────────────
   const copyPluginCode = require('fs').readFileSync(path.join(ROOT, 'LocalFlowPlugins/utility/copyToWorkDir/1.0.0/index.js'), 'utf8');
@@ -258,6 +266,7 @@ const AUDIO_INPUTS = {
   ok(copyHelperCode.includes('fs.unlinkSync(destination)'), 'copy helper removes partial copies before retrying');
   ok(copyHelperCode.includes('formatEta'), 'copy helper formats ETA');
   ok(copyHelperCode.includes('updateWorker'), 'copy helper updates Tdarr worker progress');
+  ok(copyHelperCode.includes('PROGRESS_INTERVAL_MS = 1000'), 'copy helper reports progress every 1 second (1000ms)');
 
   // ── Subtitle default naming checks ─────────────────────────────────────────
   console.log('\n── Subtitle default naming rules ────────────────────');