Take the 2-minute tour ×
Code Review Stack Exchange is a question and answer site for peer programmer code reviews. It's 100% free, no registration required.

I have a database with 4 tables filled with millions of rows. I have my program run on several computers computing data and then returning it to the database. The huge bottleneck in my program design is that for each calculation, it has to download the data and then perform calculations on it and then save the results to the database. When I had the data on the local network it performed with crazy speed so I realize that the resources to download the data from a remote server is the problem.

What are some ways I can save data from the remote database either before or after my code runs so I can make my program more efficient? These calculations are done once and aren't needed again and I have 24 computers running this same program.

static void Main(string[] args)
    {
        try
        {
            List<StockData> stockData = new List<StockData>();
            List<StockMarketCompare> stockCompareData = new List<StockMarketCompare>();
            List<StockData> sandpInfo = new List<StockData>();
            List<StockData> sandpDateInfo = new List<StockData>();
            List<StockData> globalList = new List<StockData>();
            List<StockData> amexList = new List<StockData>();
            List<StockData> nasdaqList = new List<StockData>();
            List<StockData> nyseList = new List<StockData>();
            List<DateTime> completedDates = new List<DateTime>();
            SymbolInfo symbolClass = new SymbolInfo();
            bool isGoodToGo = false;
            string symbol, market;
            int activeSymbolsCount = 0;
            int rowCount = 0, completedRowCount = 0;
            DateTime date = new DateTime();
            DateTime searchDate = new DateTime();

            // get the data here
            using (StockRatingsTableAdapter stockRatingsAdapter = new StockRatingsTableAdapter())
            using (OoplesDataSet.StockRatingsDataTable stockRatingsTable = new OoplesDataSet.StockRatingsDataTable())
            using (SymbolsTableAdapter symbolAdapter = new SymbolsTableAdapter())
            using (OoplesDataSet.SymbolsDataTable symbolTable = new OoplesDataSet.SymbolsDataTable())
            using (DailyAmexDataTableAdapter dailyAmexAdapter = new DailyAmexDataTableAdapter())
            using (OoplesDataSet.DailyAmexDataDataTable dailyAmexTable = new OoplesDataSet.DailyAmexDataDataTable())
            using (OoplesDataSet.OldStockRatingsDataTable historicalRatingsTable = new OoplesDataSet.OldStockRatingsDataTable())
            using (OldStockRatingsTableAdapter historicalRatingsAdapter = new OldStockRatingsTableAdapter())
            using (OoplesDataSet.OldStockRatingsDataTable historicalRatingSymbolTable = new OoplesDataSet.OldStockRatingsDataTable())
            using (OldStockRatingsTableAdapter historicalRatingSymbolAdapter = new OldStockRatingsTableAdapter())
            using (OoplesDataSet.DailyGlobalDataDataTable sandp500Table = new OoplesDataSet.DailyGlobalDataDataTable())
            using (OoplesDataSet.CurrentSymbolsDataTable currentSymbolTable = new OoplesDataSet.CurrentSymbolsDataTable())
            using (CurrentSymbolsTableAdapter currentSymbolAdapter = new CurrentSymbolsTableAdapter())
            {
                // fill the s&p500 info first
                dailyGlobalAdapter.ClearBeforeFill = true;
                dailyGlobalAdapter.FillBySymbol(sandp500Table, Calculations.sp500);

                var sandpQuery = from c in sandp500Table
                                 select new StockData { Close = c.Close, Date = c.Date, High = c.High, Low = c.Low, Volume = c.Volume };
                sandpInfo = sandpQuery.AsParallel().ToList();

                // set the settings for the historical ratings adapter
                historicalRatingsAdapter.ClearBeforeFill = true;

                // fill the stock ratings info
                stockRatingsAdapter.Fill(stockRatingsTable);

                // get all symbols in the stock ratings table
                var symbolsAmountQuery = from c in stockRatingsTable
                                         select new SymbolMarket { Symbol = c.Symbol, Market = c.Market };
                List<SymbolMarket> ratingSymbols = symbolsAmountQuery.AsParallel().ToList();

                if (ratingSymbols != null)
                {
                    activeSymbolsCount = ratingSymbols.AsParallel().Count();
                }

                for (int i = 0; i < activeSymbolsCount; i++)
                {
                    symbol = ratingSymbols.AsParallel().ElementAtOrDefault(i).Symbol;
                    market = ratingSymbols.AsParallel().ElementAtOrDefault(i).Market;

   dailyAmexAdapter.FillBySymbol(dailyAmexTable, symbol);
                            historicalRatingSymbolAdapter.FillBySymbolMarket(historicalRatingSymbolTable, market, symbol);

                            if (dailyAmexTable != null)
                            {
                                var amexFillQuery = from c in dailyAmexTable
                                                 select new StockData { Close = c.Close, Date = c.Date, High = c.High, Low = c.Low, Volume = c.Volume };
                                amexList = amexFillQuery.AsParallel().ToList();

                                rowCount = amexList.AsParallel().Count();
                            }
                            if (historicalRatingSymbolTable != null)
                            {
                                completedRowCount = historicalRatingSymbolTable.AsParallel().Count();
                                completedDates = historicalRatingSymbolTable.AsParallel().Select(d => d.Date).ToList();
                            }

                    currentSymbolAdapter.Fill(currentSymbolTable);
                    var currentSymbolQuery = from c in currentSymbolTable
                                             where c.Symbol == symbol && c.Market == market
                                             select c;
                    List<OoplesDataSet.CurrentSymbolsRow> currentSymbolRow = currentSymbolQuery.AsParallel().ToList();

                    // if the rows don't match up and if no other computer is working on the same symbol
                    if (rowCount - 30 != completedRowCount && currentSymbolRow.Count == 0)
                    {
                        // update the table to let the other computers know that we are working on this symbol
                        var computerQuery = from c in currentSymbolTable
                                            where c.ComputerName == Environment.MachineName
                                            select c;
                        List<OoplesDataSet.CurrentSymbolsRow> currentComputerRow = computerQuery.AsParallel().ToList();

                        if (currentComputerRow.Count > 0)
                        {
                            // update
                            currentComputerRow.AsParallel().ElementAtOrDefault(0).Symbol = symbol;
                            currentComputerRow.AsParallel().ElementAtOrDefault(0).Market = market;

                            OoplesDataSet.CurrentSymbolsDataTable tempCurrentTable = new OoplesDataSet.CurrentSymbolsDataTable();
                            tempCurrentTable = (OoplesDataSet.CurrentSymbolsDataTable)currentSymbolTable.GetChanges();

                            if (tempCurrentTable != null)
                            {
                                currentSymbolAdapter.Adapter.UpdateCommand.UpdatedRowSource = System.Data.UpdateRowSource.None;
                                currentSymbolAdapter.Update(tempCurrentTable);
                                tempCurrentTable.AcceptChanges();
                                tempCurrentTable.Dispose();
                                Console.WriteLine(Environment.MachineName + " has claimed dominion over " + symbol + " in the " + market + " market!");
                            }
                        }
                        else
                        {
                            // insert
                            currentSymbolAdapter.Insert(symbol, market, Environment.MachineName);
                            Console.WriteLine(Environment.MachineName + " has claimed dominion over " + symbol + " in the " + market + " market!");
                        }

                        Parallel.For(0, rowCount - 30, new ParallelOptions
                            {
                                MaxDegreeOfParallelism = Environment.ProcessorCount
                            }, j =>
                            {
                                        if (amexList.AsParallel().Count() > 0)
                                        {
                                            date = amexList.AsParallel().ElementAtOrDefault(j).Date;
                                            searchDate = date.Subtract(TimeSpan.FromDays(60));

                                            if (completedDates.Contains(date) == false)
                                            {
                                                var amexQuery = from c in sandpInfo
                                                                where c.Date >= searchDate && c.Date <= date
                                                                join d in amexList on c.Date equals d.Date
                                                                select new StockMarketCompare { stockClose = d.Close, marketClose = c.Close };

                                                var amexStockDataQuery = from c in amexList
                                                                            where c.Date >= searchDate && c.Date <= date
                                                                            select new StockData { Close = c.Close, High = c.High, Low = c.Low, Volume = c.Volume, Date = c.Date };

                                                stockCompareData = amexQuery.AsParallel().ToList();
                                                stockData = amexStockDataQuery.AsParallel().ToList();
                                                isGoodToGo = true;
                                            }
                                            else
                                            {
                                                isGoodToGo = false;
                                            }
                                        }

                                if (completedDates.Contains(date) == false)
                                {
                                    var sandpDateQuery = from c in sandpInfo
                                                        where c.Date >= searchDate && c.Date <= date
                                                        select c;
                                    sandpDateInfo = sandpDateQuery.AsParallel().ToList();
                                    symbolClass = new SymbolInfo(symbol, market);
                                    isGoodToGo = true;
                                }
                                else
                                {
                                    isGoodToGo = false;
                                }

                                if (isGoodToGo)
                                {
                                    sendMessage(sandpInfo, date, symbolClass, stockData, stockCompareData);
                                }
                            });
                    }
                }
            }
        }
        catch (Exception ex)
        {
            Console.WriteLine(ex.Message);
            Console.WriteLine(ex.StackTrace);
        }
    }
share|improve this question
1  
I see four usings that aren't actually used: symbolAdapter, symbolTable, historicalRatingsTable and historicalRatingsAdapter. –  BCdotWEB Feb 3 at 16:12

1 Answer 1

up vote 3 down vote accepted

Your code formatting is off.


If the bottleneck is really the retrieval of the data from the database, you should check your queries. These queries can often be improved for faster retrieval.


ratingSymbols is a List<SymbolMarket> so you won't gain any speed improvement by calling ratingSymbols.AsParallel().Count(); this could reduce the speed because it is creating some threads.

You should just use ratingSymbols.Count; <- the property.


This

symbol = ratingSymbols.AsParallel().ElementAtOrDefault(i).Symbol;
market = ratingSymbols.AsParallel().ElementAtOrDefault(i).Market;  

can be reduced to

symbol = ratingSymbols[i].Symbol;
market = ratingSymbols[i].Market;  

which will maybe speed things a little bit. This applies to each of this usages of ElementAtOrDefault().


rowCount = amexList.AsParallel().Count();
completedRowCount = historicalRatingSymbolTable.AsParallel().Count();

again, use the Count property without the AsParallel().


if (amexList.AsParallel().Count() > 0)`

again, use the Count property without the AsParallel().

share|improve this answer

Your Answer

 
discard

By posting your answer, you agree to the privacy policy and terms of service.

Not the answer you're looking for? Browse other questions tagged or ask your own question.