Improved logging for copy steps

src/fs-helpers/copy-with-tracking.js

@@ -0,0 +1,23 @@
+import { getStatusString } from "../logging/status.js";
+import fs from 'fs';
+import { getTotalFileSize } from "./total-filesize.js";
+import { humanReadableSize } from "../logging/human-readable-sizes.js";
+
+export function copyWithTracking(src, dest, customStepName) {
+    const fullSize = getTotalFileSize(src);
+    let sizeMoved = 0;
+    const stepName = customStepName ?? `Copying from ${src} to ${dest}`;
+
+    fs.cpSync(src, dest, {
+        recursive: true,
+        filter: (filterSource, filterDest) => {
+            const toLog = getStatusString(stepName, sizeMoved, fullSize, humanReadableSize(sizeMoved), 10);
+            process.stdout.write(`${toLog}\r`);
+            sizeMoved += fs.statSync(filterSource).size;
+            return true;
+        },
+    });
+
+    const finalLog = getStatusString(stepName, fullSize, fullSize, humanReadableSize(fullSize), 10);
+    process.stdout.write(`${finalLog}\n`);
+}

src/fs-helpers/total-filesize.js

@@ -0,0 +1,21 @@
+import fs from 'fs';
+
+export function getTotalFileSize(folder) {
+    const fileEnts = fs.readdirSync(folder, { encoding: 'utf-8', withFileTypes: true });
+
+    const files = fileEnts.filter(
+        (ent) => ent.isFile()
+    );
+
+    const fileSizes = files.map(
+        (fileEnt) => {
+            const stats = fs.statSync(`${fileEnt.parentPath}/${fileEnt.name}`);
+            return stats.size;
+        }
+    );
+
+    return fileSizes.reduce(
+        (a, b) => a + b,
+        0
+    );
+}

src/logging/human-readable-sizes.js

@@ -0,0 +1,29 @@
+export function humanReadableSize(bytes) {
+    const scales = [
+        {
+            unit: 'TiB',
+            scale: 1024 * 1024 * 1024 * 1024,
+        },
+        {
+            unit: 'GiB',
+            scale: 1024 * 1024 * 1024,
+        },
+        {
+            unit: 'MiB',
+            scale: 1024 * 1024,
+        },
+        {
+            unit: 'KiB',
+            scale: 1024,
+        },
+    ];
+
+    for(let i = 0; i < scales.length; i++) {
+        if(bytes >= scales[i].scale) {
+            const amount = (bytes / scales[i].scale).toFixed(2);
+            return `${amount}${scales[i].unit}`;
+        }
+    }
+
+    return `${bytes} B`;
+}

src/logging/spacer.js

@@ -0,0 +1,7 @@
+export function getSpacerString(length) {
+    let spacer = '';
+    for(let i = 0; i < length; i++) {
+        spacer += ' ';
+    }
+    return spacer;
+}

src/logging/status.js

@@ -0,0 +1,30 @@
+import { getSpacerString } from "./spacer.js";
+
+export function getStatusString(processName, currentProgress, maxProgress, currentStage, stageBufferSize) {
+    const BAR_SIZE = 20;
+    let bar = '[';
+    let ratio = currentProgress / maxProgress;
+    const percent = Math.floor(ratio * 100);
+
+    for(let i = 0; i < BAR_SIZE; i++) {
+        let charRatio = (i / BAR_SIZE);
+        if(charRatio < ratio) {
+            bar += '=';
+        } else { 
+            bar += ' ';
+        }
+    }
+
+    bar += ']';
+
+    let percentSpacer = '';
+    if(percent < 10) {
+        percentSpacer = '  ';
+    } else if(percent < 100) {
+        percentSpacer = ' ';
+    }
+
+    const endSpacer = getSpacerString(stageBufferSize);
+
+    return `${bar} ${percent}%${percentSpacer} | ${processName} | ${currentStage}${endSpacer}`;
+}

src/pipeline.js

@@ -1,5 +1,7 @@
 import { help } from "./commands/help.js";
 import { Folders } from "./constants/folders.js";
+import { copyWithTracking } from "./fs-helpers/copy-with-tracking.js";
+import { getStatusString } from "./logging/status.js";
 import { getAllCommands } from "./parsing/parse-commands.js";
 import fs from 'fs';
 
@@ -12,6 +14,7 @@ export function CreatePipeline(inputParams) {
                 help();
                 return;
             }
+
             // Create /temp/Step-0
             if(fs.existsSync(Folders.TEMP)) {
                 fs.rmSync(Folders.TEMP, { recursive: true, force: true});
@@ -19,7 +22,7 @@ export function CreatePipeline(inputParams) {
             fs.mkdirSync(Folders.TEMP);
 
             // Copy /input/* -> /temp/Step-0
-            fs.cpSync(Folders.INPUT, getTempFolderName(0), { recursive: true });
+            copyWithTracking(Folders.INPUT, getTempFolderName(0), 'Copying inputs to workspace...');
 
             for(let i = 0; i < this.commands.length; i++) {
                 const inputDir = getTempFolderName(i);
@@ -35,33 +38,43 @@ export function CreatePipeline(inputParams) {
                     (ent) => ent.name
                 );
 
+                const maxFilenameLength = files.reduce(
+                    (prevLength, nextFileName) => {
+                        if(nextFileName.length > prevLength) {
+                            return nextFileName.length;
+                        }
+
+                        return prevLength;
+                    }, 0
+                );
+
                 const func = this.commands[i].func;
                 const params = this.commands[i].parameters;
 
                 for(let k = 0; k < files.length; k++) {
-                    process.stdout.write(`${getStatus(this.commands[i].command, k, files.length)}\r`);
+                    process.stdout.write(`${getStatusString(this.commands[i].command, k, files.length, files[k], maxFilenameLength)}\r`);
                     const result = await func(inputDir, files[k], outputDir, params);
                     if(this.commands[i].command === 'Help') {
                         break;
                     }
                 }
-                process.stdout.write(`${getStatus(this.commands[i].command, files.length, files.length)}\n`);
+                process.stdout.write(`${getStatusString(this.commands[i].command, files.length, files.length, '(Complete)', maxFilenameLength)}\n`);
                 
                 // Reduce space usage by deleting unneeded step files
                 fs.rmSync(inputDir, { force: true, recursive: true });
             }
 
             // Copy /temp/Step-L/* -> /output
-            console.log(`Copying to ${Folders.OUTPUT} ...`);
 
             if(fs.existsSync(Folders.OUTPUT)) {
                 fs.rmSync(Folders.OUTPUT, { recursive: true, force: true });
             }
 
-            fs.cpSync(getTempFolderName(this.commands.length), Folders.OUTPUT, { recursive: true });
+            copyWithTracking(getTempFolderName(this.commands.length), Folders.OUTPUT, 'Copying final files to outputs...');
 
             // remove /temp/*
-            console.log('Cleaning up temporary files');
+            console.log('');
+            console.log('Cleaning up temporary files...');
             fs.rmSync(Folders.TEMP, { recursive: true, force: true });
         },
     };
@@ -70,23 +83,3 @@ export function CreatePipeline(inputParams) {
 function getTempFolderName(i) {
     return `${Folders.TEMP}step-${i}/`;
 }
-
-function getStatus(stepName, currentIndex, size) {
-    const BAR_SIZE = 15;
-    let bar = `[`;
-    let ratio = currentIndex / size;
-    const percent = Math.floor(ratio * 100);
-
-    for(let i = 0; i < BAR_SIZE; i++) {
-        let charRatio = (i / BAR_SIZE);
-        if(charRatio < ratio) {
-            bar += '=';
-        } else {
-            bar += ' ';
-        }
-    }
-
-    bar += ']';
-
-    return `${stepName}: ${bar} ${percent}%`;
-}