From 1eb5fe2ff00048f86f462ed03f7cd6ef68fca774 Mon Sep 17 00:00:00 2001 From: walcutt Date: Wed, 25 Jun 2025 19:54:02 -0400 Subject: [PATCH] Improved logging for copy steps --- src/fs-helpers/copy-with-tracking.js | 23 ++++++++++++++ src/fs-helpers/total-filesize.js | 21 +++++++++++++ src/logging/human-readable-sizes.js | 29 ++++++++++++++++++ src/logging/spacer.js | 7 +++++ src/logging/status.js | 30 +++++++++++++++++++ src/pipeline.js | 45 ++++++++++++---------------- 6 files changed, 129 insertions(+), 26 deletions(-) create mode 100644 src/fs-helpers/copy-with-tracking.js create mode 100644 src/fs-helpers/total-filesize.js create mode 100644 src/logging/human-readable-sizes.js create mode 100644 src/logging/spacer.js create mode 100644 src/logging/status.js diff --git a/src/fs-helpers/copy-with-tracking.js b/src/fs-helpers/copy-with-tracking.js new file mode 100644 index 0000000..3add1db --- /dev/null +++ b/src/fs-helpers/copy-with-tracking.js @@ -0,0 +1,23 @@ +import { getStatusString } from "../logging/status.js"; +import fs from 'fs'; +import { getTotalFileSize } from "./total-filesize.js"; +import { humanReadableSize } from "../logging/human-readable-sizes.js"; + +export function copyWithTracking(src, dest, customStepName) { + const fullSize = getTotalFileSize(src); + let sizeMoved = 0; + const stepName = customStepName ?? `Copying from ${src} to ${dest}`; + + fs.cpSync(src, dest, { + recursive: true, + filter: (filterSource, filterDest) => { + const toLog = getStatusString(stepName, sizeMoved, fullSize, humanReadableSize(sizeMoved), 10); + process.stdout.write(`${toLog}\r`); + sizeMoved += fs.statSync(filterSource).size; + return true; + }, + }); + + const finalLog = getStatusString(stepName, fullSize, fullSize, humanReadableSize(fullSize), 10); + process.stdout.write(`${finalLog}\n`); +} \ No newline at end of file diff --git a/src/fs-helpers/total-filesize.js b/src/fs-helpers/total-filesize.js new file mode 100644 index 0000000..06c721a --- /dev/null +++ b/src/fs-helpers/total-filesize.js @@ -0,0 +1,21 @@ +import fs from 'fs'; + +export function getTotalFileSize(folder) { + const fileEnts = fs.readdirSync(folder, { encoding: 'utf-8', withFileTypes: true }); + + const files = fileEnts.filter( + (ent) => ent.isFile() + ); + + const fileSizes = files.map( + (fileEnt) => { + const stats = fs.statSync(`${fileEnt.parentPath}/${fileEnt.name}`); + return stats.size; + } + ); + + return fileSizes.reduce( + (a, b) => a + b, + 0 + ); +} \ No newline at end of file diff --git a/src/logging/human-readable-sizes.js b/src/logging/human-readable-sizes.js new file mode 100644 index 0000000..b6ff749 --- /dev/null +++ b/src/logging/human-readable-sizes.js @@ -0,0 +1,29 @@ +export function humanReadableSize(bytes) { + const scales = [ + { + unit: 'TiB', + scale: 1024 * 1024 * 1024 * 1024, + }, + { + unit: 'GiB', + scale: 1024 * 1024 * 1024, + }, + { + unit: 'MiB', + scale: 1024 * 1024, + }, + { + unit: 'KiB', + scale: 1024, + }, + ]; + + for(let i = 0; i < scales.length; i++) { + if(bytes >= scales[i].scale) { + const amount = (bytes / scales[i].scale).toFixed(2); + return `${amount}${scales[i].unit}`; + } + } + + return `${bytes} B`; +} \ No newline at end of file diff --git a/src/logging/spacer.js b/src/logging/spacer.js new file mode 100644 index 0000000..5bb959a --- /dev/null +++ b/src/logging/spacer.js @@ -0,0 +1,7 @@ +export function getSpacerString(length) { + let spacer = ''; + for(let i = 0; i < length; i++) { + spacer += ' '; + } + return spacer; +} \ No newline at end of file diff --git a/src/logging/status.js b/src/logging/status.js new file mode 100644 index 0000000..5750cd1 --- /dev/null +++ b/src/logging/status.js @@ -0,0 +1,30 @@ +import { getSpacerString } from "./spacer.js"; + +export function getStatusString(processName, currentProgress, maxProgress, currentStage, stageBufferSize) { + const BAR_SIZE = 20; + let bar = '['; + let ratio = currentProgress / maxProgress; + const percent = Math.floor(ratio * 100); + + for(let i = 0; i < BAR_SIZE; i++) { + let charRatio = (i / BAR_SIZE); + if(charRatio < ratio) { + bar += '='; + } else { + bar += ' '; + } + } + + bar += ']'; + + let percentSpacer = ''; + if(percent < 10) { + percentSpacer = ' '; + } else if(percent < 100) { + percentSpacer = ' '; + } + + const endSpacer = getSpacerString(stageBufferSize); + + return `${bar} ${percent}%${percentSpacer} | ${processName} | ${currentStage}${endSpacer}`; +} \ No newline at end of file diff --git a/src/pipeline.js b/src/pipeline.js index 87b4225..0813f82 100644 --- a/src/pipeline.js +++ b/src/pipeline.js @@ -1,5 +1,7 @@ import { help } from "./commands/help.js"; import { Folders } from "./constants/folders.js"; +import { copyWithTracking } from "./fs-helpers/copy-with-tracking.js"; +import { getStatusString } from "./logging/status.js"; import { getAllCommands } from "./parsing/parse-commands.js"; import fs from 'fs'; @@ -12,6 +14,7 @@ export function CreatePipeline(inputParams) { help(); return; } + // Create /temp/Step-0 if(fs.existsSync(Folders.TEMP)) { fs.rmSync(Folders.TEMP, { recursive: true, force: true}); @@ -19,7 +22,7 @@ export function CreatePipeline(inputParams) { fs.mkdirSync(Folders.TEMP); // Copy /input/* -> /temp/Step-0 - fs.cpSync(Folders.INPUT, getTempFolderName(0), { recursive: true }); + copyWithTracking(Folders.INPUT, getTempFolderName(0), 'Copying inputs to workspace...'); for(let i = 0; i < this.commands.length; i++) { const inputDir = getTempFolderName(i); @@ -35,33 +38,43 @@ export function CreatePipeline(inputParams) { (ent) => ent.name ); + const maxFilenameLength = files.reduce( + (prevLength, nextFileName) => { + if(nextFileName.length > prevLength) { + return nextFileName.length; + } + + return prevLength; + }, 0 + ); + const func = this.commands[i].func; const params = this.commands[i].parameters; for(let k = 0; k < files.length; k++) { - process.stdout.write(`${getStatus(this.commands[i].command, k, files.length)}\r`); + process.stdout.write(`${getStatusString(this.commands[i].command, k, files.length, files[k], maxFilenameLength)}\r`); const result = await func(inputDir, files[k], outputDir, params); if(this.commands[i].command === 'Help') { break; } } - process.stdout.write(`${getStatus(this.commands[i].command, files.length, files.length)}\n`); + process.stdout.write(`${getStatusString(this.commands[i].command, files.length, files.length, '(Complete)', maxFilenameLength)}\n`); // Reduce space usage by deleting unneeded step files fs.rmSync(inputDir, { force: true, recursive: true }); } // Copy /temp/Step-L/* -> /output - console.log(`Copying to ${Folders.OUTPUT} ...`); if(fs.existsSync(Folders.OUTPUT)) { fs.rmSync(Folders.OUTPUT, { recursive: true, force: true }); } - fs.cpSync(getTempFolderName(this.commands.length), Folders.OUTPUT, { recursive: true }); + copyWithTracking(getTempFolderName(this.commands.length), Folders.OUTPUT, 'Copying final files to outputs...'); // remove /temp/* - console.log('Cleaning up temporary files'); + console.log(''); + console.log('Cleaning up temporary files...'); fs.rmSync(Folders.TEMP, { recursive: true, force: true }); }, }; @@ -69,24 +82,4 @@ export function CreatePipeline(inputParams) { function getTempFolderName(i) { return `${Folders.TEMP}step-${i}/`; -} - -function getStatus(stepName, currentIndex, size) { - const BAR_SIZE = 15; - let bar = `[`; - let ratio = currentIndex / size; - const percent = Math.floor(ratio * 100); - - for(let i = 0; i < BAR_SIZE; i++) { - let charRatio = (i / BAR_SIZE); - if(charRatio < ratio) { - bar += '='; - } else { - bar += ' '; - } - } - - bar += ']'; - - return `${stepName}: ${bar} ${percent}%`; } \ No newline at end of file