I wrote a script for reading a CSV, parsing it and saving data to MongoDB. I would love some input on how I'm handling errors and dealing with callbacks. I wrote it with nested without using async and the nesting wasn't too bad, but I tried this way thinking it might be better practice. It's a one-off script.
var fs = require('fs');
var parse = require('csv').parse;
var db = require('../models');
var async = require('async');
// take file path argument from command line (absolute or relative path)
// or default to the file path I used
var airportCSVPath = process.argv[2] || '../public/files/airports.csv';
// Read in airport data from csv file and write it to database.
async.waterfall([
async.apply(readCSV, airportCSVPath),
parseCSV,
saveAirports
], function(err){
if(err){
console.log(err);
return;
}
console.log('airports saved to db');
process.exit(0);
});
// read in airports csv
function readCSV(filePath, cb){
fs.readFile(filePath, function (err, data) {
if (err) {
return cb(err);
}
return cb(null, data);
});
}
// parse airports csv
function parseCSV(data, cb){
parse(data, function (err, parsedData) {
if (err) {
return cb(err);
}
return cb(null, parsedData);
});
}
// Create objects for airports in the US with scheduled service and save them to the DB.
// Includes heliports, seaplane bases, balloonports, and airports of all sizes.
function saveAirports(data, cb){
var count = 0;
var q = async.queue(function (data, callback) {
var airport = new db.Airport(data);
airport.save(function (err) {
if (err) {
return callback(err);
}
return callback(null, data);
})
}, 5);
for (var i = 1; i < data.length; i++) {
var country = data[i][8];
var scheduledService = data[i][11];
var status = data[i][2];
if(country === 'US' && scheduledService === 'yes' && status !== 'closed'){
var airportObj = createAirportObj(data[i]);
q.push(airportObj, function (err) {
if (err) {
fs.appendFile('../logs/create-db-errors.txt', err.toString(), function (writeErr) {
if (writeErr) {
console.log('error writing to error log', writeErr);
}
});
}
count++;
});
}
}
// quick way to check successfully saved records from command line against
// expected value
q.drain = function(){
console.log(count);
cb(null);
}
}
// I like this here to cut down on the function size of saveAirports
function createAirportObj(row) {
var airportObj = {
id: Number(row[0]) || 0,
ident: row[1] || null,
type: row[2] || null,
name: row[3] || null,
latitude: Number(row[4]) || 0, // latitude in degrees
longitude: Number(row[5]) || 0, // longitude in degrees
eleveation: Number(row[6]) || 0, // elevation in feet
continent: row[7] || null,
iso_country: row[8] || null,
iso_region: row[9] || null,
city: row[10] || null, // because why not? space is cheap
scheduled: row[11] || null,
gps_code: row[12] || null,
iata: row[13] || null,
local_code: row[14] || null,
home_link: row[15] || null,
wikipedia_link: row[16] || null,
keywords: row[17] || null
};
return airportObj;
}