I wrote a script for reading a CSV, parsing it and saving data to MongoDB. I would love some input on how I'm handling errors and dealing with callbacks. I wrote it with nested without using async and the nesting wasn't too bad, but I tried this way thinking it might be better practice. It's a one-off script.

var fs = require('fs');
var parse = require('csv').parse;
var db = require('../models');
var async = require('async');

// take file path argument from command line (absolute or relative path)
// or default to the file path I used
var airportCSVPath = process.argv[2] || '../public/files/airports.csv';

// Read in airport data from csv file and write it to database.
async.waterfall([
    async.apply(readCSV, airportCSVPath),
    parseCSV,
    saveAirports
], function(err){
    if(err){
        console.log(err);
        return;
    }
    console.log('airports saved to db');
    process.exit(0);
});

// read in airports csv
function readCSV(filePath, cb){
    fs.readFile(filePath, function (err, data) {
        if (err) {
            return cb(err);
        }
        return cb(null, data);
    });
}

// parse airports csv
function parseCSV(data, cb){
    parse(data, function (err, parsedData) {
        if (err) {
            return cb(err);
        }
        return cb(null, parsedData);
    });
}

// Create objects for airports in the US with scheduled service and save them to the DB.
// Includes heliports, seaplane bases, balloonports, and airports of all sizes.
function saveAirports(data, cb){
    var count = 0;
    var q = async.queue(function (data, callback) {
        var airport = new db.Airport(data);
        airport.save(function (err) {
            if (err) {
                return callback(err);
            }
            return callback(null, data);
        })
    }, 5);

    for (var i = 1; i < data.length; i++) {
        var country = data[i][8];
        var scheduledService = data[i][11];
        var status = data[i][2];
        if(country === 'US' && scheduledService === 'yes' && status !== 'closed'){
            var airportObj = createAirportObj(data[i]);

            q.push(airportObj, function (err) {
                if (err) {
                    fs.appendFile('../logs/create-db-errors.txt', err.toString(), function (writeErr) {
                        if (writeErr) {
                            console.log('error writing to error log', writeErr);
                        }
                    });
                }
                count++;
            });
        }
    }

    // quick way to check successfully saved records from command line against
    // expected value
    q.drain = function(){
        console.log(count);
        cb(null);
    }
}

// I like this here to cut down on the function size of saveAirports
function createAirportObj(row) {
    var airportObj = {
                        id: Number(row[0]) || 0,
                        ident: row[1] || null,
                        type: row[2] || null,
                        name: row[3] || null,
                        latitude: Number(row[4]) || 0, // latitude in degrees
                        longitude: Number(row[5]) || 0, // longitude in degrees
                        eleveation: Number(row[6]) || 0, // elevation in feet
                        continent: row[7] || null,
                        iso_country: row[8] || null,
                        iso_region: row[9] || null,
                        city: row[10] || null, // because why not? space is cheap
                        scheduled: row[11] || null,
                        gps_code: row[12] || null,
                        iata: row[13] || null,
                        local_code: row[14] || null,
                        home_link: row[15] || null,
                        wikipedia_link: row[16] || null,
                        keywords: row[17] || null
                    };

    return airportObj;
}
share|improve this question

Your Answer

 
discard

By posting your answer, you agree to the privacy policy and terms of service.

Browse other questions tagged or ask your own question.