diff --git a/src/annotations.js b/src/annotations.js
index 8193ed9..475cb67 100644
--- a/src/annotations.js
+++ b/src/annotations.js
@@ -81,6 +81,7 @@ class Annotations {
 
 			let views = this.rendition.views();
 			views.each( (view) => {
+				this._removeFromAnnotationBySectionIndex(annotation.sectionIndex, hash);
 				if (annotation.sectionIndex === view.index) {
 					annotation.detach(view);
 				}
@@ -90,6 +91,14 @@ class Annotations {
 		}
 	}
 
+	_removeFromAnnotationBySectionIndex (sectionIndex, hash) {
+		this._annotationsBySectionIndex[sectionIndex] = this._annotationsAt(sectionIndex).filter(h => h !== hash);
+	}
+
+	_annotationsAt (index) {
+		return this._annotationsBySectionIndex[index];
+	}
+
 	highlight (cfiRange, data, cb) {
 		this.add("highlight", cfiRange, data, cb);
 	}