From 5d43ba6a267bceb1048ea6241694de5a4b8b8a48 Mon Sep 17 00:00:00 2001 From: PickleRick Date: Mon, 21 Sep 2026 12:34:15 +0200 Subject: [PATCH] Save uncommitted changes --- FlowHelpers/1.0.0/copyWithProgress.js | 71 +++++++++++ .../utility/copyToWorkDir/2.0.0/index.js | 70 +++++++++++ .../replaceOriginalFile/1.0.0/index.js | 114 ++++++++++++++++++ .../replaceOriginalFile/2.0.0/index.js | 99 +++++++++++++++ README.md | 35 +++++- docs/integration-guide.md | 25 ++-- install.sh | 28 +++-- tests/quick-logic-test.js | 8 ++ 8 files changed, 430 insertions(+), 20 deletions(-) create mode 100644 FlowHelpers/1.0.0/copyWithProgress.js create mode 100644 LocalFlowPlugins/utility/copyToWorkDir/2.0.0/index.js create mode 100644 LocalFlowPlugins/utility/replaceOriginalFile/1.0.0/index.js create mode 100644 LocalFlowPlugins/utility/replaceOriginalFile/2.0.0/index.js diff --git a/FlowHelpers/1.0.0/copyWithProgress.js b/FlowHelpers/1.0.0/copyWithProgress.js new file mode 100644 index 0000000..7aab71d --- /dev/null +++ b/FlowHelpers/1.0.0/copyWithProgress.js @@ -0,0 +1,71 @@ +'use strict'; + +const fs = require('fs'); + +const copyWithProgress = async ({ source, destination, jobLog, updateProgress, label }) => { + const sourceSize = fs.statSync(source).size; + const sourceSizeMb = (sourceSize / 1024 / 1024).toFixed(1); + const startMs = Date.now(); + let copiedBytes = 0; + let nextProgressPercent = 0; + + const report = (percent, message) => { + const boundedPercent = Math.min(Math.max(percent, 0), 100); + if (typeof updateProgress === 'function') { + updateProgress({ + progress: boundedPercent, + jobLog: message, + }); + } + if (typeof jobLog === 'function') jobLog(`${message}\n`); + }; + + await new Promise((resolve, reject) => { + const rd = fs.createReadStream(source); + const wr = fs.createWriteStream(destination); + let settled = false; + const fail = (error) => { + if (settled) return; + settled = true; + rd.destroy(); + wr.destroy(); + reject(error); + }; + + rd.on('error', fail); + wr.on('error', fail); + wr.on('finish', () => { + if (settled) return; + settled = true; + resolve(); + }); + rd.on('data', (chunk) => { + copiedBytes += chunk.length; + const percent = sourceSize === 0 ? 100 : Math.floor((copiedBytes / sourceSize) * 100); + if (percent < nextProgressPercent) return; + + const elapsedSeconds = Math.max((Date.now() - startMs) / 1000, 0.1); + const copiedMb = (copiedBytes / 1024 / 1024).toFixed(1); + const speedMbPerSecond = (copiedBytes / 1024 / 1024 / elapsedSeconds).toFixed(1); + report( + percent, + `[${label}] Progress: ${Math.min(percent, 100)}% ` + + `(${copiedMb}/${sourceSizeMb} MB, ${speedMbPerSecond} MB/s)`, + ); + nextProgressPercent = (Math.floor(percent / 10) + 1) * 10; + }); + rd.pipe(wr); + }); + + const destinationSize = fs.statSync(destination).size; + if (destinationSize !== sourceSize) { + throw new Error( + `[${label}] Copy size mismatch: expected ${sourceSize} bytes, got ${destinationSize}`, + ); + } + + const elapsed = ((Date.now() - startMs) / 1000).toFixed(1); + report(100, `[${label}] Done: ${sourceSizeMb} MB copied in ${elapsed}s`); +}; + +module.exports = { copyWithProgress }; diff --git a/LocalFlowPlugins/utility/copyToWorkDir/2.0.0/index.js b/LocalFlowPlugins/utility/copyToWorkDir/2.0.0/index.js new file mode 100644 index 0000000..3c90a43 --- /dev/null +++ b/LocalFlowPlugins/utility/copyToWorkDir/2.0.0/index.js @@ -0,0 +1,70 @@ +'use strict'; + +const path = require('path'); +const fs = require('fs'); +const { copyWithProgress } = require('../../../../FlowHelpers/1.0.0/copyWithProgress'); + +const details = () => ({ + name: 'Copy Source to Work Dir (Progress)', + description: + 'Copies the source file into the worker local workDir using a streamed, verified copy ' + + 'with progress updates. Use before transcode plugins for files on network shares.', + style: { borderColor: '#607D8B' }, + tags: 'utility,copy,progress', + isStartPlugin: false, + pType: '', + requiresVersion: '2.11.01', + sidebarPosition: 0, + icon: 'faCopy', + inputs: [ + { + label: 'Skip if already local', + name: 'skipIfLocal', + type: 'boolean', + defaultValue: 'true', + inputUI: { type: 'dropdown', options: ['true', 'false'] }, + tooltip: 'Skip when the source is already inside workDir.', + }, + ], + outputs: [{ number: 1, tooltip: 'File is ready in workDir — continue to next plugin' }], +}); + +const plugin = async (args) => { + const { jobLog, inputFileObj, variables, workDir } = args; + const inputFile = inputFileObj._id; + const ext = path.extname(inputFile); + const baseName = path.basename(inputFile, ext); + const localCopy = path.join(workDir, `${baseName}_old${ext}`); + const skipIfLocal = (args.inputs.skipIfLocal || 'true') === 'true'; + + const relativeInput = path.relative(path.resolve(workDir), path.resolve(inputFile)); + if (skipIfLocal && relativeInput !== '..' && !relativeInput.startsWith(`..${path.sep}`) + && !path.isAbsolute(relativeInput)) { + jobLog('[copyToWorkDir] Source file is already inside workDir — skipping copy.\n'); + return { outputNumber: 1, outputFileObj: inputFileObj, variables }; + } + if (!fs.existsSync(inputFile)) throw new Error(`[copyToWorkDir] Source file not found: ${inputFile}`); + + variables.originalLibraryFile = variables.originalLibraryFile || inputFile; + variables.originalLibraryDir = variables.originalLibraryDir || path.dirname(inputFile); + variables.workDirSourceBaseName = baseName; + variables.workDirCopyApplied = 'true'; + + jobLog(`[copyToWorkDir] Copying ${inputFile} to ${localCopy}\n`); + await copyWithProgress({ + source: inputFile, + destination: localCopy, + jobLog, + updateProgress: args.updateProgress, + label: 'copyToWorkDir', + }); + + return { + outputNumber: 1, + outputFileObj: { ...inputFileObj, _id: localCopy }, + variables, + }; +}; + +module.exports.details = details; +module.exports.plugin = plugin; diff --git a/LocalFlowPlugins/utility/replaceOriginalFile/1.0.0/index.js b/LocalFlowPlugins/utility/replaceOriginalFile/1.0.0/index.js new file mode 100644 index 0000000..801da4a --- /dev/null +++ b/LocalFlowPlugins/utility/replaceOriginalFile/1.0.0/index.js @@ -0,0 +1,114 @@ +'use strict'; + +const path = require('path'); +const fs = require('fs'); + +const details = () => ({ + name: 'Replace Original File', + description: + 'Copies the processed file to the original library path, verifies the copy, ' + + 'and atomically replaces the original with rollback if replacement fails.', + style: { borderColor: '#FF9800' }, + tags: 'utility,replace', + isStartPlugin: false, + pType: '', + requiresVersion: '2.11.01', + sidebarPosition: 1, + icon: 'faExchangeAlt', + inputs: [ + { + label: 'Delete processed work file', + name: 'deleteSource', + type: 'boolean', + defaultValue: 'true', + inputUI: { type: 'dropdown', options: ['true', 'false'] }, + tooltip: 'Delete the processed file after it has replaced the original.', + }, + ], + outputs: [{ number: 1, tooltip: 'Original file replaced — continue to next plugin' }], +}); + +const copyAndVerify = (source, destination) => { + fs.copyFileSync(source, destination); + const sourceSize = fs.statSync(source).size; + const destinationSize = fs.statSync(destination).size; + if (sourceSize !== destinationSize) { + throw new Error( + `Copy size mismatch: expected ${sourceSize} bytes, got ${destinationSize}`, + ); + } +}; + +const plugin = async (args) => { + const { inputFileObj, variables, jobLog } = args; + const source = inputFileObj._id; + const configuredDestination = variables.originalLibraryFile || args.originalLibraryFile; + const destination = typeof configuredDestination === 'string' + ? configuredDestination + : configuredDestination && configuredDestination._id; + + if (!destination || typeof destination !== 'string') { + throw new Error('[replaceOriginalFile] No original library path is available'); + } + if (!fs.existsSync(source)) { + throw new Error(`[replaceOriginalFile] Processed file not found: ${source}`); + } + if (path.resolve(source) === path.resolve(destination)) { + jobLog('[replaceOriginalFile] Processed file is already at the original path — skipping.\n'); + return { outputNumber: 1, outputFileObj: inputFileObj, variables }; + } + + const destinationDir = path.dirname(destination); + if (!fs.existsSync(destinationDir)) { + throw new Error(`[replaceOriginalFile] Destination directory does not exist: ${destinationDir}`); + } + + const temporary = `${destination}.tdarr.tmp`; + const backup = `${destination}.partial.old`; + let backupCreated = false; + let installed = false; + + try { + if (fs.existsSync(temporary)) fs.unlinkSync(temporary); + copyAndVerify(source, temporary); + + if (fs.existsSync(destination)) { + if (fs.existsSync(backup)) fs.unlinkSync(backup); + fs.renameSync(destination, backup); + backupCreated = true; + } + fs.renameSync(temporary, destination); + installed = true; + + if (backupCreated) { + try { + fs.unlinkSync(backup); + } catch (error) { + jobLog(`[replaceOriginalFile] WARNING: Could not remove backup ${backup}: ${error.message}\n`); + } + } + if ((args.inputs.deleteSource || 'true') === 'true') { + try { + fs.unlinkSync(source); + } catch (error) { + jobLog(`[replaceOriginalFile] WARNING: Could not delete processed file ${source}: ${error.message}\n`); + } + } + + jobLog(`[replaceOriginalFile] Replaced original file: ${destination}\n`); + return { + outputNumber: 1, + outputFileObj: { ...inputFileObj, _id: destination }, + variables, + }; + } catch (error) { + if (fs.existsSync(temporary)) fs.unlinkSync(temporary); + if (backupCreated && !installed && !fs.existsSync(destination)) { + fs.renameSync(backup, destination); + } + throw new Error(`[replaceOriginalFile] Replacement failed: ${error.message}`); + } +}; + +module.exports.details = details; +module.exports.plugin = plugin; diff --git a/LocalFlowPlugins/utility/replaceOriginalFile/2.0.0/index.js b/LocalFlowPlugins/utility/replaceOriginalFile/2.0.0/index.js new file mode 100644 index 0000000..305a75b --- /dev/null +++ b/LocalFlowPlugins/utility/replaceOriginalFile/2.0.0/index.js @@ -0,0 +1,99 @@ +'use strict'; + +const path = require('path'); +const fs = require('fs'); +const { copyWithProgress } = require('../../../../FlowHelpers/1.0.0/copyWithProgress'); + +const details = () => ({ + name: 'Replace Original File (Progress)', + description: + 'Copies the processed file to the original library path with streamed progress, ' + + 'size verification, an atomic rename, and rollback if replacement fails.', + style: { borderColor: '#FF9800' }, + tags: 'utility,replace,progress', + isStartPlugin: false, + pType: '', + requiresVersion: '2.11.01', + sidebarPosition: 1, + icon: 'faExchangeAlt', + inputs: [ + { + label: 'Delete processed work file', + name: 'deleteSource', + type: 'boolean', + defaultValue: 'true', + inputUI: { type: 'dropdown', options: ['true', 'false'] }, + tooltip: 'Delete the processed file after it has replaced the original.', + }, + ], + outputs: [{ number: 1, tooltip: 'Original file replaced — continue to next plugin' }], +}); + +const plugin = async (args) => { + const { inputFileObj, variables, jobLog } = args; + const source = inputFileObj._id; + const configuredDestination = variables.originalLibraryFile || args.originalLibraryFile; + const destination = typeof configuredDestination === 'string' + ? configuredDestination + : configuredDestination && configuredDestination._id; + if (!destination || typeof destination !== 'string') { + throw new Error('[replaceOriginalFile] No original library path is available'); + } + if (!fs.existsSync(source)) throw new Error(`[replaceOriginalFile] Processed file not found: ${source}`); + if (path.resolve(source) === path.resolve(destination)) { + jobLog('[replaceOriginalFile] Processed file is already at the original path — skipping.\n'); + return { outputNumber: 1, outputFileObj: inputFileObj, variables }; + } + + const destinationDir = path.dirname(destination); + if (!fs.existsSync(destinationDir)) { + throw new Error(`[replaceOriginalFile] Destination directory does not exist: ${destinationDir}`); + } + + const temporary = `${destination}.tdarr.tmp`; + const backup = `${destination}.partial.old`; + let backupCreated = false; + let installed = false; + try { + if (fs.existsSync(temporary)) fs.unlinkSync(temporary); + await copyWithProgress({ + source, + destination: temporary, + jobLog, + updateProgress: args.updateProgress, + label: 'replaceOriginalFile', + }); + + if (fs.existsSync(destination)) { + if (fs.existsSync(backup)) fs.unlinkSync(backup); + fs.renameSync(destination, backup); + backupCreated = true; + } + fs.renameSync(temporary, destination); + installed = true; + + if (backupCreated) { + try { + fs.unlinkSync(backup); + } catch (error) { + jobLog(`[replaceOriginalFile] WARNING: Could not remove backup ${backup}: ${error.message}\n`); + } + } + if ((args.inputs.deleteSource || 'true') === 'true') { + try { + fs.unlinkSync(source); + } catch (error) { + jobLog(`[replaceOriginalFile] WARNING: Could not delete processed file ${source}: ${error.message}\n`); + } + } + jobLog(`[replaceOriginalFile] Replaced original file: ${destination}\n`); + return { outputNumber: 1, outputFileObj: { ...inputFileObj, _id: destination }, variables }; + } catch (error) { + if (fs.existsSync(temporary)) fs.unlinkSync(temporary); + if (backupCreated && !installed && !fs.existsSync(destination)) fs.renameSync(backup, destination); + throw new Error(`[replaceOriginalFile] Replacement failed: ${error.message}`); + } +}; + +module.exports.details = details; +module.exports.plugin = plugin; diff --git a/README.md b/README.md index 913a2e6..b4f0010 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,8 @@ curl -fsSL "${BASE}/LocalFlowPlugins/${CATEGORY}/${PLUGIN}/1.0.0/index.js" -o "$ | `setAnimationFlag` | `utility` | | `extractChaptersXml` | `utility` | | `finalizeMedia` | `utility` | +| `copyToWorkDir` | `utility` | +| `replaceOriginalFile` | `utility` | | `audioSelectorClamping` | `audio` | | `backupSourceAudio` | `audio` | | `extractSubtitlesAndOcr` | `subtitles` | @@ -97,7 +99,10 @@ LocalFlowPlugins/ │ ├── mediaAnalyzerGOP/1.0.0/index.js │ ├── setAnimationFlag/1.0.0/index.js │ ├── extractChaptersXml/1.0.0/index.js -│ └── finalizeMedia/1.0.0/index.js +│ ├── finalizeMedia/1.0.0/index.js +│ ├── copyToWorkDir/2.0.0/index.js +│ ├── replaceOriginalFile/1.0.0/index.js +│ └── replaceOriginalFile/2.0.0/index.js ├── audio/ │ ├── audioSelectorClamping/1.0.0/index.js │ └── backupSourceAudio/1.0.0/index.js @@ -245,7 +250,33 @@ Performs 2-pass libx265 encoding using VBV parameters from `args.variables`. **Reads from variables**: `gopKeyint`, `gopMinKeyint`, `gopSceneCut`, `vBitrate`, `vMaxRate`, `vBufSize`, `x265Tune`, `audioHasIta`, `audioItaStreamIndex`, `audioItaOpusBitrate`, `audioHasEng`, `audioEngStreamIndex`, `audioEngOpusBitrate` -### 8. `finalizeMedia` — Finalize Media & Sidecars +### 8. `copyToWorkDir` — Copy Source to Work Dir (Progress) + +**Category**: `utility` · **Step**: before transcode + +Streams the source file to the worker's local `workDir`, verifies the byte count, +and reports throttled copy progress. It stores the original library path in +`variables.originalLibraryFile` for `replaceOriginalFile`. + +### 9. `replaceOriginalFile` — Replace Original File + +**Category**: `utility` · **Step**: after transcode + +The `1.0.0` version performs the same verified, atomic replacement as the +progress version but does not emit transfer progress callbacks or progress logs. +It is useful when the standard Tdarr replacement behavior is desired without +additional progress reporting. + +### 10. `replaceOriginalFile` — Replace Original File (Progress) + +**Category**: `utility` · **Step**: after transcode + +Streams the processed file to a temporary file beside the original, verifies its +size, then atomically replaces the original. An existing original is moved to +`.partial.old` during the operation and restored if replacement fails. The +processed work file is deleted after a successful replacement. + +### 11. `finalizeMedia` — Finalize Media & Sidecars Run after `Move To Directory`. It normalizes subtitle language codes, adds `.default` to `ita.forced`, removes duplicate `_chapters` sidecars, and removes diff --git a/docs/integration-guide.md b/docs/integration-guide.md index 755ffac..c6a62dd 100644 --- a/docs/integration-guide.md +++ b/docs/integration-guide.md @@ -15,7 +15,10 @@ Questa guida descrive come assemblare i 7 plugin nel canvas Flow di Tdarr, confi │ ├── mediaAnalyzerGOP/1.0.0/index.js │ ├── setAnimationFlag/1.0.0/index.js │ ├── extractChaptersXml/1.0.0/index.js - │ └── finalizeMedia/1.0.0/index.js + │ ├── finalizeMedia/1.0.0/index.js + │ ├── copyToWorkDir/2.0.0/index.js + │ ├── replaceOriginalFile/1.0.0/index.js + │ └── replaceOriginalFile/2.0.0/index.js ├── audio/ │ ├── audioSelectorClamping/1.0.0/index.js │ └── backupSourceAudio/1.0.0/index.js @@ -68,8 +71,9 @@ Nella Web UI: **Settings → Nodes → [tuo worker Windows]** > Il plugin `extractChaptersXml` deriva automaticamente il path di `mkvextract.exe` dalla stessa cartella di `mkvpropedit.exe`. -`copyToWorkDir` crea la copia temporanea con suffisso `_old`; il transcoder -produce così il nome originale senza `_x265`. +`copyToWorkDir` crea la copia temporanea con suffisso `_old` e conserva il +percorso originale in `variables.originalLibraryFile`. La versione 2.0.0 +trasmette il progresso della copia e verifica la dimensione finale. ### 3.3 Configurazione PATH Windows @@ -85,6 +89,8 @@ Aggiungi al PATH di sistema: ### 4.1 Ordine dei nodi ``` +[copyToWorkDir 2.0.0] (recommended for network libraries) + ▼ [mediaAnalyzerGOP] │ │ (Facoltativo — solo per anime non rilevati automaticamente) @@ -107,13 +113,14 @@ Aggiungi al PATH di sistema: [backupSourceAudio] ▼ [transcode2PassX265] - ▼ - [Copy/Move Folder Content] - ▼ - [Move To Directory] - ▼ - [finalizeMedia] + ▼ + [replaceOriginalFile 2.0.0] + ▼ + [finalizeMedia] ``` +`copyToWorkDir` 2.0.0 is optional but recommended when the source library is on +a network share. Place it before the first analysis/transcode node; it records +the library path for `replaceOriginalFile` 2.0.0. ### 4.2 Connessioni nel Canvas diff --git a/install.sh b/install.sh index 24c1028..d51efd5 100644 --- a/install.sh +++ b/install.sh @@ -29,6 +29,7 @@ declare -A PLUGIN_CATEGORY=( [extractChaptersXml]=utility [finalizeMedia]=utility [copyToWorkDir]=utility + [replaceOriginalFile]=utility [audioSelectorClamping]=audio [backupSourceAudio]=audio [extractSubtitlesAndOcr]=subtitles @@ -64,16 +65,25 @@ download_plugin() { return 1 fi - local dest="${PLUGIN_DIR}/LocalFlowPlugins/${category}/${name}/1.0.0" - local url="${RAW}/LocalFlowPlugins/${category}/${name}/1.0.0/index.js" - - mkdir -p "$dest" - if curl -fsSL "$url" -o "${dest}/index.js"; then - echo " ✓ ${name} → ${dest}/index.js" - else - echo " ✗ Failed to download ${name} from ${url}" - return 1 + local versions="1.0.0" + if [[ "$name" == "copyToWorkDir" ]]; then + versions="2.0.0" + elif [[ "$name" == "replaceOriginalFile" ]]; then + versions="1.0.0 2.0.0" fi + + for version in $versions; do + local dest="${PLUGIN_DIR}/LocalFlowPlugins/${category}/${name}/${version}" + local url="${RAW}/LocalFlowPlugins/${category}/${name}/${version}/index.js" + + mkdir -p "$dest" + if curl -fsSL "$url" -o "${dest}/index.js"; then + echo " ✓ ${name} ${version} → ${dest}/index.js" + else + echo " ✗ Failed to download ${name} ${version} from ${url}" + return 1 + fi + done } # ── Run ─────────────────────────────────────────────────────────────────────── diff --git a/tests/quick-logic-test.js b/tests/quick-logic-test.js index 7aeaa29..fde311c 100644 --- a/tests/quick-logic-test.js +++ b/tests/quick-logic-test.js @@ -210,6 +210,14 @@ const AUDIO_INPUTS = { const copyPluginCode = require('fs').readFileSync(path.join(ROOT, 'LocalFlowPlugins/utility/copyToWorkDir/1.0.0/index.js'), 'utf8'); ok(copyPluginCode.includes('[copyToWorkDir] Progress:'), 'copyToWorkDir reports copy progress in the job log'); + const copyPluginV2Code = require('fs').readFileSync(path.join(ROOT, 'LocalFlowPlugins/utility/copyToWorkDir/2.0.0/index.js'), 'utf8'); + ok(copyPluginV2Code.includes('updateProgress: args.updateProgress'), 'copyToWorkDir v2 forwards UI progress'); + const replacePluginCode = require('fs').readFileSync(path.join(ROOT, 'LocalFlowPlugins/utility/replaceOriginalFile/2.0.0/index.js'), 'utf8'); + ok(replacePluginCode.includes('.partial.old'), 'replaceOriginalFile v2 uses a rollback backup'); + ok(replacePluginCode.includes('fs.renameSync(temporary, destination)'), 'replaceOriginalFile v2 atomically installs the verified copy'); + const replacePluginV1Code = require('fs').readFileSync(path.join(ROOT, 'LocalFlowPlugins/utility/replaceOriginalFile/1.0.0/index.js'), 'utf8'); + ok(replacePluginV1Code.includes('copyAndVerify'), 'replaceOriginalFile v1 verifies the copied file'); + ok(!replacePluginV1Code.includes('updateProgress'), 'replaceOriginalFile v1 has no progress callback'); // ── Summary ─────────────────────────────────────────────────────────────── console.log(`\n${'='.repeat(55)}`);