feat: change gzip compression from tar package to zlib

tvapp2/index.js

@@ -12,7 +12,6 @@ import zlib from 'zlib';
 import chalk from 'chalk';
 import ejs from 'ejs';
 import moment from 'moment';
-import * as tar from 'tar';
 
 /*
     Old CJS variables converted to ESM
@@ -67,8 +66,8 @@ let FILE_TAR_MODIFIED = 0;
 const envUrlRepo = process.env.URL_REPO || `https://git.binaryninja.net/binaryninja`;
 const envStreamQuality = process.env.STREAM_QUALITY || `hd`;
 const envFileM3U = process.env.FILE_PLAYLIST || `playlist.m3u8`;
-const envFileXML = process.env.FILE_EPG || `xmltv.xml`;
-const envFileTAR = process.env.FILE_TAR || `xmltv.tar.gz`;
+const envFileXML = process.env.FILE_EPG || 'xmltv.xml';
+const envFileTAR = process.env.FILE_TAR || 'xmltv.xml.gz';
 const LOG_LEVEL = process.env.LOG_LEVEL || 10;
 
 /*
@@ -244,7 +243,7 @@ const semaphore = new Semaphore( 5 );
 
 async function downloadFile( url, filePath )
 {
-    Log.info( `Fetching ${ url }` );
+    Log.info( `Fetching`, chalk.white( `→` ), chalk.grey( `${ url }` ) );
 
     return new Promise( ( resolve, reject ) =>
     {
@@ -262,7 +261,7 @@ async function downloadFile( url, filePath )
                 response.pipe( file );
                 file.on( 'finish', () =>
                 {
-                    Log.ok( `Successfully fetched ${ filePath }` );
+                    Log.ok( `Received`, chalk.white( `→` ), `${ filePath }` );
                     file.close( () => resolve( true ) );
                 });
             })
@@ -299,9 +298,13 @@ function getFileModified( filename )
 
 */
 
-function getFilesizeHuman( filename, si = true, decimal = 1 )
+function getFileSizeHuman( filename, si = true, decimal = 1 )
 {
-    const stats = fs.statSync( filename );
+    let stats = [];
+    stats.size = 0;
+    if ( fs.existsSync( filename ) )
+        stats = fs.statSync( filename );
+
     let bytes = stats.size;
     const thresh = si ? 1000 : 1024;
 
@@ -341,13 +344,13 @@ function getFilesizeHuman( filename, si = true, decimal = 1 )
     @ret        none
 */
 
-async function ensureFileExists( url, filePath )
+async function getFile( url, filePath )
 {
     try
     {
         await downloadFile( url, filePath );
     }
-    catch ( error )
+    catch ( err )
     {
         if ( fs.existsSync( filePath ) )
         {
@@ -356,7 +359,90 @@ async function ensureFileExists( url, filePath )
         else
         {
             Log.error( `Failed to download file, and no local file exists; aborting`, chalk.white( `→` ), chalk.grey( `${ url }` ) );
-            throw error;
+            throw err;
+        }
+    }
+}
+
+/*
+    Func > Package GZip
+
+    locates the xmltv.xml and packages it into a xmltv.gz archive
+*/
+
+async function createGzip( )
+{
+    Log.debug( `Preparing to gzip`, chalk.white( `→` ), chalk.grey( `${ envFileXML }` ) );
+
+    return new Promise( ( resolve, reject ) =>
+    {
+        Log.debug( `createGzip[promise]`, chalk.white( `→` ), chalk.grey( `${ envFileXML }` ) );
+
+        fs.readFile( FILE_XML, ( err, buf ) =>
+        {
+            Log.debug( `createGzip[fs.readFile]`, chalk.white( `→` ), chalk.grey( `${ envFileXML }` ) );
+
+            if ( err )
+            {
+                Log.error( `Could not read file ${ envFileXML }. Error: `, chalk.white( `→` ), chalk.grey( `${ err }` ) );
+                return reject( new Error( `Could not read file ${ envFileXML }. Error: ${ err }` ) );
+            }
+
+            zlib.gzip( buf, ( err, buf ) =>
+            {
+                Log.debug( `createGzip[zlib.gzip]`, chalk.white( `→` ), chalk.grey( `${ envFileXML }` ), chalk.white( `→` ), chalk.grey( `${ envFileTAR }` ) );
+                if ( err )
+                {
+                    Log.error( `Could not write to archive. Error: `, chalk.white( `→` ), chalk.grey( `${ err }` ) );
+                    return reject( new Error( `Could not create ${ envFileTAR }. Error: ${ err }` ) );
+                }
+
+                Log.info( `Compressing`, chalk.white( `→` ), `${ envFileXML }`, chalk.white( `→` ), `${ FILE_TAR }` );
+                fs.writeFile( `${ FILE_TAR }`, buf, ( err ) =>
+                {
+                    if ( err )
+                    {
+                        Log.error( `Could not write XML file to archive. Error: `, chalk.white( `→` ), chalk.grey( `${ err }` ) );
+                        return reject( new Error( `Could not write XML file ${ envFileXML } to ${ envFileTAR }. Error: ${ err }` ) );
+                    }
+
+                    Log.ok( `Compressed`, chalk.white( `→` ), `${ envFileXML }`, chalk.white( `→` ), `${ FILE_TAR }` );
+                    resolve( true );
+                });
+            });
+        });
+    });
+}
+
+/*
+    Func > Ensure File Exists
+
+    if file exists; start download from external website utilizing url and file path arguments; or
+    throw error to user that file does not exist via the URL.
+
+    If file cannot be obtained from external url; use local copy if available
+
+    @arg        str url                         https://git.binaryninja.net/binaryninja/tvapp2-externals/raw/branch/main/urls.txt
+    @arg        str filePath                    H:\Repos\github\BinaryNinja\tvapp2\tvapp2\urls.txt
+    @ret        none
+*/
+
+async function prepareGzip( )
+{
+    try
+    {
+        await createGzip( );
+    }
+    catch ( err )
+    {
+        if ( fs.existsSync( FILE_XML ) )
+        {
+            Log.warn( `XML file found, but gzip failed to compress XML`, chalk.white( `→` ), chalk.grey( `${ FILE_XML }` ) );
+        }
+        else
+        {
+            Log.error( `XML file not found`, chalk.white( `→` ), chalk.grey( `${ FILE_XML }` ) );
+            throw err;
         }
     }
 }
@@ -652,9 +738,9 @@ async function serveM3UPlaylist( req, res )
         res.end( rewrittenPlaylist );
         Log.ok( `Served playlist` );
     }
-    catch ( error )
+    catch ( err )
     {
-        Log.error( `Error processing request:`, chalk.white( ` → ` ), chalk.grey( `${ error.message }` ) );
+        Log.error( `Error processing request:`, chalk.white( `→` ), chalk.grey( `${ err.message }` ) );
 
         if ( !res.headersSent )
         {
@@ -729,15 +815,15 @@ async function serveM3U( response, req )
 
         response.end( updatedContent );
     }
-    catch ( error )
+    catch ( err )
     {
-        Log.error( `Error in serveM3U:`, chalk.white( ` → ` ), chalk.grey( `${ error.message }` ) );
+        Log.error( `Error in serveM3U:`, chalk.white( `→` ), chalk.grey( `${ err.message }` ) );
 
         response.writeHead( 500, {
             'Content-Type': 'text/plain'
         });
 
-        response.end( `Error serving playlist: ${ error.message }` );
+        response.end( `Error serving playlist: ${ err.message }` );
     }
 }
 
@@ -761,15 +847,15 @@ async function serveXML( response, req )
 
         response.end( formattedContent );
     }
-    catch ( error )
+    catch ( err )
     {
-        Log.error( `Error in serveM3U:`, chalk.white( ` → ` ), chalk.grey( `${ error.message }` ) );
+        Log.error( `Error in serveM3U:`, chalk.white( `→` ), chalk.grey( `${ err.message }` ) );
 
         response.writeHead( 500, {
             'Content-Type': 'text/plain'
         });
 
-        response.end( `Error serving playlist: ${ error.message }` );
+        response.end( `Error serving playlist: ${ err.message }` );
     }
 };
 
@@ -793,15 +879,15 @@ async function serveTAR( response, req )
 
         response.end( formattedContent );
     }
-    catch ( error )
+    catch ( err )
     {
-        Log.error( `Error in serveTAR:`, chalk.white( ` → ` ), chalk.grey( `${ error.message }` ) );
+        Log.error( `Error in serveTAR:`, chalk.white( `→` ), chalk.grey( `${ err.message }` ) );
 
         response.writeHead( 500, {
             'Content-Type': 'text/plain'
         });
 
-        response.end( `Error serving tar.gz: ${ error.message }` );
+        response.end( `Error serving tar.gz: ${ err.message }` );
     }
 };
 
@@ -833,46 +919,23 @@ function getCache( key )
     }
 }
 
+/*
+    Initialization
+
+    this is the starting method to prepare tvapp2
+*/
+
 async function initialize()
 {
     try
     {
-        Log.info( `Initializing server...` );
+        Log.info( `Initialization Started` );
 
-        await ensureFileExists( extURL, FILE_URL );
-        await ensureFileExists( extXML, FILE_XML );
-        await ensureFileExists( extFormatted, FILE_M3U );
+        await getFile( extURL, FILE_URL );
+        await getFile( extXML, FILE_XML );
+        await getFile( extFormatted, FILE_M3U );
+        await prepareGzip();
 
-        /*
-            Create tar.gz of xml data
-            Must specify `cwd` - current working directory; without this, the absolute path to the .tar.gz file
-            will be added to the archive.
-
-            set cwd to the folder where the xmltv.1.xml file is located
-        */
-
-        const tarFiles = [];
-        await tar.c(
-        {
-            gzip: true,
-            cwd: path.resolve( __dirname ),
-            noPax: true,
-            onWriteEntry( entry )
-            {
-                Log.debug( `tar.gz: adding`, chalk.yellow( ` ${ entry.path } ` ), chalk.white( ` → ` ), chalk.grey( `${ entry.stat.mode.toString( 8 ) }` ) );
-                entry.path = entry.path.toLowerCase();
-                tarFiles.push([ entry.path, entry.stat.mode.toString( 8 ) ]);
-            },
-            onwarn: ( code, message, data ) =>
-            {
-                Log.warn( `tar.gz warning:`, chalk.white( ` → ` ), chalk.grey( `[${ code }] ${ message }` ) );
-            }
-        },
-        [`${ envFileXML }`]
-        )
-        .pipe( fs.createWriteStream( `${ envFileTAR }` ) )
-        .on( 'finish', () =>
-        {
         urls = fs.readFileSync( FILE_URL, 'utf-8' ).split( '\n' ).filter( Boolean );
         if ( urls.length === 0 )
             throw new Error( `No valid URLs found in ${ FILE_URL }` );
@@ -881,24 +944,19 @@ async function initialize()
             Calculate Sizes
         */
 
-            FILE_M3U_SIZE = getFilesizeHuman( FILE_M3U );
-            FILE_XML_SIZE = getFilesizeHuman( FILE_XML );
-            FILE_TAR_SIZE = getFilesizeHuman( FILE_TAR );
+        FILE_M3U_SIZE = getFileSizeHuman( FILE_M3U );
+        FILE_XML_SIZE = getFileSizeHuman( FILE_XML );
+        FILE_TAR_SIZE = getFileSizeHuman( FILE_TAR );
 
         FILE_M3U_MODIFIED = getFileModified( FILE_M3U );
         FILE_XML_MODIFIED = getFileModified( FILE_XML );
         FILE_TAR_MODIFIED = getFileModified( FILE_TAR );
 
-            Log.info( `Initializing Complete` );
-        })
-        .on( 'error', ( err ) =>
-        {
-            Log.error( `Could not create tar:`, chalk.white( ` → ` ), chalk.grey( `${ err }` ) );
-        });
+        Log.info( `Initialization Complete` );
     }
-    catch ( error )
+    catch ( err )
     {
-        Log.error( `Initialization error:`, chalk.white( ` → ` ), chalk.grey( `${ error.message }` ) );
+        Log.error( `Initialization error:`, chalk.white( `→` ), chalk.grey( `${ err.message }` ) );
     }
 }
 
@@ -1050,9 +1108,9 @@ const server = http.createServer( ( request, response ) =>
             }
         });
     };
-    handleRequest().catch( ( error ) =>
+    handleRequest().catch( ( err ) =>
     {
-        Log.error( `Error handling request:`, chalk.white( ` → ` ), chalk.grey( `${ error }` ) );
+        Log.error( `Error handling request:`, chalk.white( `→` ), chalk.grey( `${ err }` ) );
 
         response.writeHead( 500, {
             'Content-Type': 'text/plain'