JavaScript on page load and on scroll functions

Would it serve much purpose to work at refactoring the latter parts of this code?

// This part is OK
function isScrolledIntoView(elem, trigger_height) {

  var docViewTop = $(window).scrollTop();
  var docViewBottom = docViewTop + $(window).height();
  var elemTop = elem.offset().top;

  if (trigger_height == false) {
    trigger_height = elem.height()
  }
  var elemBottom = elemTop + trigger_height;

  return ((elemBottom <= docViewBottom) && (elemTop >= docViewTop));
}

// This function happens when the page first loads
$("p").addClass('hidden').each(function() {
  $this = $(this);
  if (isScrolledIntoView($this, false)) {
    $this.removeClass("hidden");
  }
});

// This function happens on scroll, but basically repeats itself    
$(window).off('scroll').scroll(function() {
  $("p.hidden").each(function() {
    $this = $(this);
    if (isScrolledIntoView($this, false)) {
      $this.removeClass("hidden")
    }
  });
});

To me, it seems like there isn't – but at the same time its repetitive:

$this = $(this);
if (isScrolledIntoView($this, false)) {
  $this.removeClass("hidden")
}

Is there a general rule that anyone follows such as "only refactor if you're looking at > 5 lines of repeated code", or should you be refactoring absolutely everything as you go?