Remove dynamically appended HTML from DOM with ng-click. AngularJS

I built a directive that dynamically appends html. I used the $compile service to add my ng-click function to the scope so I can remove items from my collection, however, it does not remove it from the DOM. It only removes it from the collection. Any thoughts on how to remove an individual item from the collection and have it update the DOM accordingly?

Here is my code:

app.directive('appendCategory', function(){
 return {
    restrict: 'E',
    controller: function($scope, $http, $compile, $filter){
        $scope.addedCategories = [];
        $scope.categoryList = "";

        $http.get('/api/categories/all-categories').success(function(response){
            $scope.categories = response.map(function(category){
                return {
                    name: category.name
                }
            })
        })

        $scope.onSelect = function ($item) {
            $scope.nameObject = [$item.name];
            $scope.addedCategories.push($scope.nameObject);
        };

        $scope.addCategory = function(){
            var myElement = angular.element(document.querySelector('#appended'));
            $scope.categoryList = ("<span class='label label-default'>" + $scope.selectedCategory + "<i class='fa fa-times' ng-click='removeCategory()'></i></span>");
            var compiled = $compile($scope.categoryList)($scope);
            myElement.append(compiled);

            $scope.addedCategories.push({category: $scope.categoryList && $scope.selectedCategory, deleted:false});
            console.log($scope.addedCategories);
            $scope.selectedCategory = "";
        }

        $scope.removeCategory = function(index){
            $scope.addedCategories.splice(index,1);
            console.log($scope.addedCategories);
        }
    },
    templateUrl: '/category/category.html'
 }
})

Mark-Up: It is appended here.

 <div class="margin-top-1" id="appended"></div>