mysql - Node js が SQL サーバーから 100k レコードを読み取るのに時間がかかる

Question

SQL サーバー データベースから 100,000 レコードをフェッチし、それらを mysql データベースに挿入するノード js アプリケーションを作成しています。コードを作成しましたが、アプリケーションで期待どおりのパフォーマンスが得られません。

100k レコードの選択には 9 ～ 10 秒かかり、一括挿入プロセスには 3 ～ 4 秒かかります。ノードの選択プロセスを高速化する必要があります (SQL サーバーからレコードを選択するのに 3 ～ 4 秒かかります)。2 つの方法を使用してみましたが、まだ予定の時間を達成できません。

/*First Way*/

var sql       = require('mssql');
var mysql     = require("mysql");
var sizeof    = require('sizeof');
var moment    = require('moment');

var config = {

        user: '**', 
        password: '*****',
        server: '******', 
        database: '*******'
}

var connection1 = mysql.createConnection({
			  host     : 'localhost',
			  user     : 'root',
			  password : 'root',
			  database : 'test'
		});

 connection1.connect();


  sql.connect(config, function(err) {
       
       if(err)
       	console.log(err);

      console.log('Connecting to Sql Server');
      console.time('Overall Time-Taken');
   
      var request = new sql.Request();

      console.time('Time-Taken to fetch');

      request.query('select TOP 100000 * from ShipmentAuditLog WITH (NOLOCK)', function(err, recordset) {
          
          if(err)
            console.log(err);
            else{
                 //console.log(sizeof(recordset));
                 console.timeEnd('Time-Taken to fetch');
                 syncing(recordset);
             } 
      });

  });


  function mapping(row) {
  
      var formatedString = '';

      var CreatedDate = new moment(row.CreatedDate).format('YYYY-MM-DD HH:mm:ss');

      row.Process = row.Process.replace(/'/g,"\\'");

      if(row.Comment != null){
           formatedString = row.Comment;
           formatedString = formatedString.replace(/'/g,"\\'");
           row.Comment    = formatedString;    
      }
       return "('" + row.ShippingID + "','" + row.BagNo + "','" + row.ProcessLocation + "','" + row.Process + "','" + row.Comment + "','" + CreatedDate + "','" + row.CreatedBy + "','" + row.LastModifiedDate + "','" + row.LastModifiedBy + "','" + row.DestinationLocation + "','" + row.VenderLostShipmentsDebitId + "', NOW())";
       //return "('" + row.ShippingID + "','" + row.BagNo + "','" + row.ProcessLocation + "','" + row.Process + "','" + row.Comment + "','" + CreatedDate + "','" + row.CreatedBy + "','" + row.LastModifiedDate + "','" + row.LastModifiedBy + "','" + row.DestinationLocation + "','" + row.VenderLostShipmentsDebitId + "')";
  }

    var syncing  = function(recordset){

      //var columns = [];
      var values  = [];

      /*for(col in recordset[0]){
          columns.push(col);
      }*/

      /*for(var i=0,len = recordset.length;i<len;i+=1){
         values.push(mapping(recordset[i]));
      }*/

      recordset.forEach(function(record){
          values.push(mapping(record));
      })
      //console.log(values);

      console.time('Time-Taken to Insert');

      var QRY = connection1.query('INSERT INTO shipmentauditlog VALUES'+ values.join(','), function(err, result) {
            if (err) {
                console.log(err);
            }
            else {
    
                console.timeEnd('Time-Taken to Insert');
                console.timeEnd('Overall Time-Taken');
               
            }
        });
        //console.log(QRY.sql); 
  }

    
    /*Second Way*/
    
    
    var sql       = require('mssql');
var mysql     = require("mysql");
var moment    = require('moment');

var config = {

        user: '**', 
        password: '***',
        server: '****', 
        database: '****',
        stream: true 
}

var connection1 = mysql.createConnection({
			  host     : 'localhost',
			  user     : 'root',
			  password : 'root',
			  database : 'test'
		});

connection1.connect();

 var values = [];

 function mapResult(row){

    var formatedString = '';

    var CreatedDate = new moment(row.CreatedDate).format('YYYY-MM-DD HH:mm:ss');

    row.Process = row.Process.replace(/'/g,"\\'");

      if(row.Comment != null){
           formatedString = row.Comment;
           formatedString = formatedString.replace(/'/g,"\\'");
           row.Comment    = formatedString;    
      }

     return "('" + row.ShippingID + "','" + row.BagNo + "','" + row.ProcessLocation + "','" + row.Process + "','" + row.Comment + "','" + CreatedDate + "','" + row.CreatedBy + "','" + row.LastModifiedDate + "','" + row.LastModifiedBy + "','" + row.DestinationLocation + "','" + row.VenderLostShipmentsDebitId + "', NOW())";
 }

   var connection = new sql.Connection(config, function(err) {

    if(err)
        console.log(err);

    console.log('Connecting to Sql Server');
    console.time('Overall Time-Taken');
 
    var request = new sql.Request(connection);

    //request.stream = true;
    
    request.query('select TOP 100000 * FROM ShipmentAuditLog WITH (NOLOCK)'); // or request.execute(procedure); 
       
    request.on('recordset',function(col){
         console.time('Time-Taken to fetch');
     });
    
    // Emitted for each row 
    request.on('row', function(row) {
        //Build array with resultset for bulk insert
         values.push(mapResult(row));              
    });
 
    request.on('error', function(err) {
        console.log(err);
    });
 
    // Emitted for the last one
    request.on('done', function(returnValue) {
        console.timeEnd('Time-Taken to fetch');
        // Function which perform bulk insert into mysql
        syncing(values);
        connection.close();
    });

});


 var syncing = function(values){

         /*var startTime = new Date();
         console.log("Start Time :"+ startTime);*/

         console.time('Time-Taken to Insert');

      var qry =  connection1.query('INSERT INTO shipmentauditlog VALUES'+values.join(','),function(err,res){

            if(err)
                console.log(err);
            else{
                /*var endTime = new Date();
                console.log('End Time:'+ endTime);*/
                console.timeEnd('Time-Taken to Insert');
                console.timeEnd('Overall Time-Taken');
            }

        });
 }